Migrate Dataproc to new system tests design

Change-Id: Id14c3bcf51c2e6077c994d8aa3a65c9b69d74902

.codespellignorelines

@@ -1,2 +1,3 @@
             f"DELETE {source_table} FROM { ', '.join(_from_name(tbl) for tbl in stmt.froms) }"
         for frm in source_query.selectable.froms:
+    roles = relationship("Role", secondary=assoc_user_role, backref="user", lazy="selectin")

.github/workflows/build-images.yml

@@ -39,7 +39,7 @@ env:
         secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
   # This token is WRITE one - pull_request_target type of events always have the WRITE token
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  AIRFLOW_LOGIN_TO_GITHUB_REGISTRY: "true"
+  LOGIN_TO_GITHUB_REGISTRY: "true"
   IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}"
 
 concurrency:
@@ -224,24 +224,19 @@ jobs:
       - run: python -m pip install --editable ./dev/breeze/
       - name: "Free space"
         run: airflow-freespace
-      - name: "Build CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
-        run: Breeze2 build-image
-        env:
-          GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
-          GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
-          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
-          DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
-      - name: "Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
-        run: ./scripts/ci/images/ci_push_ci_images.sh
-        env:
-          GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+      - name: "Build/Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
+        run: >
+          breeze build-image --push-image
+          --upgrade-to-newer-dependencies "${{ needs.build-info.outputs.upgradeToNewerDependencies }}"
+          --docker-cache "${{ needs.build-info.outputs.cacheDirective }}"
+          --image-tag "${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
       - name: >
           Push empty CI images to finish waiting jobs:
           ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
         if: failure() || cancelled()
-        run: ./scripts/ci/images/ci_push_empty_ci_images.sh
-        env:
-          GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+        run: >
+          breeze build-image --push-image --empty-image
+          --github-registry-push-image-tag "${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
       - name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}"
         if: failure() || cancelled()
         run: airflow-find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}"
@@ -339,6 +334,5 @@ jobs:
           Push empty PROD images to finish waiting jobs:
           ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
         if: failure() || cancelled()
-        run: ./scripts/ci/images/ci_push_empty_prod_images.sh
-        env:
-          GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+        run: >
+          breeze build-prod-image --push-image --empty-image --image-tag "${{ env.IMAGE_TAG_FOR_THE_BUILD }}"

.github/workflows/ci.yml

@@ -1,5 +1,4 @@
 # Licensed to the Apache Software Foundation (ASF) under one
-# Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
@@ -22,6 +21,7 @@ on:  # yamllint disable-line rule:truthy
   schedule:
     - cron: '28 0 * * *'
   push:
+    branches: ['main', 'v[0-9]+-[0-9]+-test']
   pull_request:
     branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable']
 permissions:
@@ -44,7 +44,7 @@ env:
         secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
   # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  AIRFLOW_LOGIN_TO_GITHUB_REGISTRY: "true"
+  LOGIN_TO_GITHUB_REGISTRY: "true"
   ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}"
   IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}"
 
@@ -327,19 +327,13 @@ jobs:
       - name: "Free space"
         run: airflow-freespace
         if: needs.build-info.outputs.inWorkflowBuild == 'true'
-      - name: "Build CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
-        run: Breeze2 build-image
-        if: needs.build-info.outputs.inWorkflowBuild == 'true'
-        env:
-          GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
-          GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
-          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
-          DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
-      - name: "Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
-        run: ./scripts/ci/images/ci_push_ci_images.sh
+      - name: "Build/Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
+        run: >
+          breeze build-image --push-image
+          --upgrade-to-newer-dependencies "${{ needs.build-info.outputs.upgradeToNewerDependencies }}"
+          --docker-cache "${{ needs.build-info.outputs.cacheDirective }}"
+          --image-tag "${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
         if: needs.build-info.outputs.inWorkflowBuild == 'true'
-        env:
-          GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
       - name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}"
         if: failure() || cancelled()
         run: airflow-find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}"
@@ -416,7 +410,7 @@ jobs:
 
   run-new-breeze-tests:
     timeout-minutes: 10
-    name: Breeze2 tests
+    name: breeze tests
     runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
     needs: [build-info]
     steps:
@@ -680,6 +674,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         uses: actions/setup-python@v2
         with:
           python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+          cache: 'pip'
+          cache-dependency-path: ./dev/breeze/setup*
+      - run: python -m pip install --editable ./dev/breeze/
       - name: >
           Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v2

.gitignore

@@ -222,5 +222,5 @@ pip-wheel-metadata
 # Generated UI licenses
 licenses/LICENSES-ui.txt
 
-# Packaged Breeze2 on Windows
-/Breeze2.exe
+# Packaged breeze on Windows
+/breeze.exe

.gitpod.yml

@@ -20,7 +20,7 @@
 # Docs: https://www.gitpod.io/docs/config-gitpod-file/
 
 tasks:
-  - init: ./breeze -y
+  - init: ./breeze-legacy -y
   - name: Install pre-commit
     openMode: split-right
     command: |

.pre-commit-config.yaml

@@ -95,7 +95,7 @@ repos:
       - id: insert-license
         name: Add license for all shell files
         exclude: ^\.github/.*$|^airflow/_vendor/|^dev/breeze/autocomplete/.*$
-        files: ^breeze$|^breeze-complete$|\.bash$|\.sh$
+        files: ^breeze-legacy$|^breeze-complete$|\.bash$|\.sh$
         args:
           - --comment-style
           - "|#|"
@@ -266,7 +266,7 @@ repos:
           The word(s) should be in lowercase." && exec codespell "$@"' --
         language: python
         types: [text]
-        exclude: ^airflow/_vendor/|^CHANGELOG\.txt$|^airflow/www/static/css/material-icons\.css$
+        exclude: ^airflow/_vendor/|^CHANGELOG\.txt$|^airflow/www/static/css/material-icons\.css$|^images/.*$
         args:
           - --ignore-words=docs/spelling_wordlist.txt
           - --skip=docs/*/commits.rst,airflow/providers/*/*.rst,*.lock,INTHEWILD.md,*.min.js,docs/apache-airflow/pipeline_example.csv
@@ -320,11 +320,12 @@ repos:
         files: ^airflow/providers/.*\.py$
         additional_dependencies: ['rich']
       - id: update-breeze-file
-        name: Update output of breeze command in BREEZE.rst
-        entry: ./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
-        language: system
-        files: ^BREEZE\.rst$|^breeze$|^breeze-complete$|^Dockerfile$
+        name: Update output of breeze commands in BREEZE.rst
+        entry: ./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
+        language: python
+        files: ^BREEZE\.rst$|^dev/breeze/.*$
         pass_filenames: false
+        additional_dependencies: ['rich']
       - id: update-local-yml-file
         name: Update mounts in the local yml file
         entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
@@ -364,6 +365,11 @@ repos:
         files: ^scripts/ci/pre_commit/supported_versions\.py$|^README\.md$|^docs/apache-airflow/supported-versions\.rst$
         pass_filenames: false
         additional_dependencies: ['tabulate']
+      - id: check-revision-heads-map
+        name: Check that the REVISION_HEADS_MAP is up-to-date
+        language: python
+        entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py
+        pass_filenames: false
       - id: update-version
         name: Update version to the latest version in the documentation
         entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py
@@ -438,9 +444,6 @@ repos:
         pass_filenames: true
         files: >
           (?x)
-          ^airflow/hooks/.*$|
-          ^airflow/operators/.*$|
-          ^airflow/sensors/.*$|
           ^airflow/providers/.*\.py$
         exclude: ^airflow/_vendor/
       - id: provide-create-sessions
@@ -523,7 +526,7 @@ repos:
         name: Check Shell scripts syntax correctness
         language: docker_image
         entry: koalaman/shellcheck:v0.7.2 -x -a
-        files: ^breeze$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$
+        files: ^breeze-legacy$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$
         exclude: ^dev/breeze/autocomplete/.*$
       - id: stylelint
         name: stylelint
@@ -582,6 +585,21 @@ repos:
         pass_filenames: false
         require_serial: true
         additional_dependencies: ['rich']
+      - id: update-breeze-config-hash
+        name: Update Breeze README.md with config files hash
+        language: python
+        entry: ./scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py
+        files: ^dev/breeze/setup.*$|^dev/breeze/pyproject.toml$|^dev/breeze/README.md$
+        pass_filenames: false
+        require_serial: true
+      - id: limit-breeze-dependencies
+        name: Breeze should have small number of top-level dependencies
+        language: python
+        entry: ./scripts/tools/check_if_limited_dependencies.py
+        files: ^dev/breeze/.*$
+        pass_filenames: false
+        require_serial: true
+        additional_dependencies: ['click', 'rich']
       - id: check-system-tests
         name: Check if system tests have required segments of code
         entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests.py