build-images.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: "Build Images"
run-name: >
  Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }}
on:  # yamllint disable-line rule:truthy
  pull_request_target:
permissions:
  # all other permissions are set to none
  contents: read
  pull-requests: read
  packages: read
env:
  ANSWER: "yes"
  DB_RESET: "true"
  VERBOSE: "true"
  GITHUB_REPOSITORY: ${{ github.repository }}
  GITHUB_USERNAME: ${{ github.actor }}
  # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
  # Airflow one is going to be used
  CONSTRAINTS_GITHUB_REPOSITORY: >-
    ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
        secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
  # This token is WRITE one - pull_request_target type of events always have the WRITE token
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
  USE_SUDO: "true"
  INCLUDE_SUCCESS_OUTPUTS: "true"

concurrency:
  group: build-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  build-info:
    timeout-minutes: 10
    name: "Build Info"
    runs-on: 'ubuntu-22.04'
    env:
      TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
    outputs:
      python-versions: "${{ steps.selective-checks.python-versions }}"
      upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
      all-python-versions-list-as-string: >-
        ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
      default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
      run-tests: ${{ steps.selective-checks.outputs.run-tests }}
      run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
      image-build: ${{ steps.selective-checks.outputs.image-build }}
      cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
      default-branch: ${{ steps.selective-checks.outputs.default-branch }}
      default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
      runs-on: ${{ steps.selective-checks.outputs.runs-on }}
      target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
          github.event.pull_request.head.sha ||
          github.sha
        }}"
    if: github.repository == 'apache/airflow'
    steps:
      - name: Discover PR merge commit
        id: discover-pr-merge-commit
        run: |
          TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
          echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> ${GITHUB_ENV}
          echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT}
        if: github.event_name == 'pull_request_target'
      # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get
      # up-to-date values
      - name: Get latest PR labels
        id: get-latest-pr-labels
        run: |
          echo -n "pull-request-labels=" >> ${GITHUB_OUTPUT}
          gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query='
            query($node_id: ID!, $endCursor: String) {
              node(id:$node_id) {
                ... on PullRequest {
                  labels(first: 100, after: $endCursor) {
                    nodes { name }
                    pageInfo { hasNextPage endCursor }
                  }
                }
              }
            }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT}
        if: github.event_name == 'pull_request_target'
      # Retrieve it to be able to determine which files has changed in the incoming commit of the PR
      # we checkout the target commit and its parent to be able to compare them
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - uses: actions/checkout@v3
        with:
          ref: ${{ env.TARGET_COMMIT_SHA }}
          persist-credentials: false
          fetch-depth: 2
      - name: "Setup python"
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: "Retrieve defaults from branch_defaults.py"
        # We cannot "execute" the branch_defaults.py python code here because that would be
        # a security problem (we cannot run any code that comes from the sources coming from the PR.
        # Therefore, we extract the branches via embedded Python code
        # we need to do it before next step replaces checked-out breeze and scripts code coming from
        # the PR, because the PR defaults have to be retrieved here.
        id: defaults
        run: |
          python - <<EOF >> ${GITHUB_ENV}
          from pathlib import Path
          import re
          import sys

          DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
          BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
          CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$'

          branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
          constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)

          output = f"""
          DEFAULT_BRANCH={branch}
          DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}
          """.strip()

          print(output)
          # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log
          print(output, file=sys.stderr)
          EOF
      - name: "Get target branch build scripts"
        uses: ./.github/actions/get-target-branch-build-scripts
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: Selective checks
        id: selective-checks
        env:
          PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}"
          COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}"
          VERBOSE: "false"
        run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
      - name: env
        run: printenv
        env:
          PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}
          GITHUB_CONTEXT: ${{ toJson(github) }}

  build-ci-images:
    permissions:
      contents: read
      packages: write
    timeout-minutes: 80
    name: >
      Build CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}
    runs-on: ${{ needs.build-info.outputs.runs-on }}
    needs: [build-info]
    if: |
      needs.build-info.outputs.image-build == 'true' &&
      github.event.pull_request.head.repo.full_name != 'apache/airflow' &&
      github.repository == 'apache/airflow'
    env:
      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
      RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
      BACKEND: sqlite
      VERSION_SUFFIX_FOR_PYPI: "dev0"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - uses: actions/checkout@v3
        with:
          ref: ${{ needs.build-info.outputs.target-commit-sha }}
          persist-credentials: false
          submodules: recursive
      - name: "Get target branch build scripts"
        uses: ./.github/actions/get-target-branch-build-scripts
      - name: >
          Build CI Images ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/build-ci-images
        env:
          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
          DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
          BUILD_TIMEOUT_MINUTES: 70

  build-prod-images:
    permissions:
      contents: read
      packages: write
    timeout-minutes: 80
    name: >
      Build PROD images
      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
    runs-on: ${{ needs.build-info.outputs.runs-on }}
    needs: [build-info, build-ci-images]
    if: |
      needs.build-info.outputs.image-build == 'true' &&
      github.event.pull_request.head.repo.full_name != 'apache/airflow' &&
      github.repository == 'apache/airflow'
    env:
      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
      RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
      BACKEND: sqlite
      VERSION_SUFFIX_FOR_PYPI: "dev0"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - uses: actions/checkout@v3
        with:
          ref: ${{ needs.build-info.outputs.target-commit-sha }}
          persist-credentials: false
          submodules: recursive
      - name: "Get target branch build scripts"
        uses: ./.github/actions/get-target-branch-build-scripts
      - name: >
          Build PROD Images
          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/build-prod-images
        with:
          build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
        env:
          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
          DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}

  build-ci-images-arm:
    timeout-minutes: 50
    name: "Build ARM CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}"
    runs-on: ${{ needs.build-info.outputs.runs-on }}
    needs: [build-info, build-prod-images]
    if: |
      needs.build-info.outputs.image-build == 'true' &&
      needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
      github.event.pull_request.head.repo.full_name != 'apache/airflow' &&
      needs.build-info.outputs.runs-on == 'self-hosted' &&
      github.repository == 'apache/airflow'
    env:
      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
      RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
      BACKEND: sqlite
      outputs: ${{toJSON(needs.build-info.outputs) }}
      VERSION_SUFFIX_FOR_PYPI: "dev0"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - uses: actions/checkout@v3
        with:
          ref: ${{ needs.build-info.outputs.target-commit-sha }}
          persist-credentials: false
          submodules: recursive
      - name: "Get target branch build scripts"
        uses: ./.github/actions/get-target-branch-build-scripts
      - name: "Start ARM instance"
        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: >
          Build ARM CI images ${{ env.IMAGE_TAG }}
          ${{needs.build-info.outputs.all-python-versions-list-as-string}}
        run: >
          breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
        env:
          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
      - name: "Stop ARM instance"
        run: ./scripts/ci/images/ci_stop_arm_instance.sh
        if: always()
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()