Merge pull request #14 from Teradata/main

Merge latest main to td_develop
Teradata · Dec 19, 2023 · 14ee26b · 14ee26b
2 parents 90d5d43 + 26179d7
commit 14ee26b
Show file tree

Hide file tree

Showing 147 changed files with 1,799 additions and 1,845 deletions.
diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml
@@ -23,9 +23,7 @@ runs:
   steps:
     - name: "Install Breeze"
       uses: ./.github/actions/breeze
-    - name: "Regenerate dependencies"
-      # This is done in case some someone updated provider dependencies and did not generate
-      # dependencies - in which case build image might fail because of lack of new dependencies
+    - name: "Check if dependencies are properly regenerated"
       shell: bash
       run: |
         pip install rich>=12.4.4 pyyaml

diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
@@ -183,21 +183,16 @@ labelPRBasedOnFilePath:
     - tests/providers/common/sql/**/*
     - tests/system/providers/common/sql/**/*
 
-  provider:daskexecutor:
-    - airflow/providers/daskexecutor/**/*
-    - docs/apache-airflow-providers-daskexecutor/**/*
-    - tests/providers/daskexecutor/**/*
-
   provider:databricks:
     - airflow/providers/databricks/**/*
     - docs/apache-airflow-providers-databricks/**/*
     - tests/providers/databricks/**/*
     - tests/system/providers/databricks/**/*
 
   provider:datadog:
-    - airflow/providers/daskexecutor/**/*
-    - docs/apache-airflow-providers-daskexecutor/**/*
-    - tests/providers/daskexecutor/**/*
+    - airflow/providers/datadog/**/*
+    - docs/apache-airflow-providers-datadog/**/*
+    - tests/providers/datadog/**/*
 
   provider:dbt-cloud:
     - airflow/providers/dbt/cloud/**/*

diff --git a/.github/workflows/ci-teradata-breeze.yml b/.github/workflows/ci-teradata-breeze.yml
@@ -30,7 +30,7 @@
         strategy:
           max-parallel: 1
           matrix:
-            python-version: ["3.8"]
+            python-version: ["3.10"]
         steps:
           - name: Cleanup repo
             run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -50,6 +50,6 @@
           - name: Run breeze
             run: pipx install -e ./dev/breeze
           - name: Run Unit Tests
-            run: breeze testing tests --test-type "Providers[teradata]"
+            run: breeze --python 3.10 testing tests --test-type "Providers[teradata]"
 
 
diff --git a/.github/workflows/ci-teradata.yml b/.github/workflows/ci-teradata.yml
@@ -17,8 +17,12 @@
 #
 ---
 name: airflow-teradata-tests
-on:  # yamllint disable-line rule:truthy
-  workflow_dispatch
+on:
+  push:
+    branches:
+      - ${{ vars.BRANCH }}
+  schedule:
+  - cron: "0 6 * * *"
 permissions:
   contents: write
   pages: write
@@ -27,22 +31,18 @@ jobs:
   teradata-tests:
     name: "Run Teradata Provider Unit and System tests"
     runs-on: "ubuntu-22.04"
-    strategy:
-      max-parallel: 1
-      matrix:
-        python-version: ["3.8"]
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v4
         with:
-          ref: develop_test
+          ref: ${{ vars.BRANCH }}
           persist-credentials: false      
       - name: "Install Python"
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.8
           cache: 'pip'
           cache-dependency-path: ./dev/breeze/setup*
       - name: Display Python version
@@ -87,20 +87,4 @@ jobs:
         uses: actions/upload-pages-artifact@v2
       - name: Deploy teradata provider test HTML report to GITHUB page
         id: deployment
-        uses: actions/deploy-pages@v2
-      - name: Run breeze
-        run: pipx install -e ./dev/breeze
-      - name: build docucmentation
-        run: breeze ci-image build --upgrade-to-newer-dependencies; breeze build-docs --package-filter apache-airflow-providers-teradata
-      - name: Setup GITHUB Page to publish teradata provider documentation
-        uses: actions/configure-pages@v3
-      - name: Build GITHUB Page for teradata provider documentation
-        uses: actions/jekyll-build-pages@v1
-        with:
-         source: ./docs/_build
-         destination: ./_docsite
-      - name: Upload documentation
-        uses: actions/upload-pages-artifact@v2
-      - name: Deploy teradata provider documentation
-        id: deploymentDocs
-        uses: actions/deploy-pages@v2 
+        uses: actions/deploy-pages@v2      
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -395,32 +395,6 @@ jobs:
       - name: "Generate client codegen diff"
         run: ./scripts/ci/openapi/client_codegen_diff.sh
 
-  test-examples-of-prod-image-building:
-    timeout-minutes: 60
-    name: "Test examples of production image building"
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
-    needs: [build-info]
-    if: needs.build-info.outputs.ci-image-build == 'true'
-    steps:
-      - name: Cleanup repo
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-          persist-credentials: false
-      - name: "Setup python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: "${{needs.build-info.outputs.default-python-version}}"
-          cache: 'pip'
-          cache-dependency-path: ./dev/requirements.txt
-      - name: "Test examples of PROD image building"
-        run: >
-          cd ./docker_tests &&
-          python -m pip install -r requirements.txt &&
-          python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes
-
   test-git-clone-on-windows:
     timeout-minutes: 5
     name: "Test git clone on Windows"
@@ -906,7 +880,7 @@ jobs:
           path: old-airflow
       - name: "Prepare airflow package: wheel"
         run: |
-          pip install pip==23.3.1 wheel==0.36.2 gitpython==3.1.40
+          pip install pip==23.3.2 wheel==0.36.2 gitpython==3.1.40
           python setup.py egg_info --tag-build ".dev0" bdist_wheel -d ../dist
         working-directory: ./old-airflow
       - name: >
@@ -1734,6 +1708,63 @@ jobs:
           # TODO: improve caching for that build
           IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
 
+  build-prod-images-mysql-client:
+    timeout-minutes: 80
+    name: >
+      Build MysQL Client PROD images (main)
+      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    needs: [build-info, build-ci-images]
+    if: needs.build-info.outputs.canary-run == 'true'
+    env:
+      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+      BACKEND: sqlite
+      VERSION_SUFFIX_FOR_PYPI: "dev0"
+      DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      # Force more parallelism for build even on public images
+      PARALLELISM: 6
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.build-info.outputs.targetCommitSha }}
+          persist-credentials: false
+          submodules: recursive
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
+      - name: >
+          Build Bullseye PROD Images
+          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+        uses: ./.github/actions/build-prod-images
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
+        with:
+          build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+          chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
+          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+          DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
+          INSTALL_MYSQL_CLIENT_TYPE: "mysql"
+          # Do not override the "mariadb" (original) image - just push a new mysql image
+          # TODO: improve caching for that build
+          IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
+
+
   build-prod-images-release-branch:
     timeout-minutes: 80
     name: >
@@ -1840,6 +1871,62 @@ jobs:
           # TODO: improve caching for that build
           IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
 
+  build-prod-images-mysql-release-branch:
+    timeout-minutes: 80
+    name: >
+      Build MySQL PROD images (v2_*_test)
+      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    needs: [build-info, generate-constraints]
+    if: needs.build-info.outputs.canary-run == 'true'
+    env:
+      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+      BACKEND: sqlite
+      VERSION_SUFFIX_FOR_PYPI: "dev0"
+      DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      # Force more parallelism for build even on public images
+      PARALLELISM: 6
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch != 'main'
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.build-info.outputs.targetCommitSha }}
+          persist-credentials: false
+          submodules: recursive
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch != 'main'
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch != 'main'
+      - name: >
+          Build Bullseye PROD Images
+          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+        uses: ./.github/actions/build-prod-images
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch != 'main'
+        with:
+          build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+          chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
+          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+          DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
+          INSTALL_MYSQL_CLIENT_TYPE: "mysql"
+          # Do not override the "mariadb" image - just push a new mysql image
+          # TODO: improve caching for that build
+          IMAGE_TAG: "mysql-${{ github.event.pull_request.head.sha || github.sha }}"
+
   wait-for-prod-images:
     timeout-minutes: 80
     name: "Wait for PROD images"
@@ -1875,6 +1962,43 @@ jobs:
           DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
         if: needs.build-info.outputs.in-workflow-build == 'false'
 
+  test-examples-of-prod-image-building:
+    timeout-minutes: 60
+    name: "Test examples of production image building"
+    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    needs: [build-info, wait-for-prod-images]
+    if: needs.build-info.outputs.prod-image-build == 'true'
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+      - name: Pull PROD image "${{needs.build-info.outputs.default-python-version}}":${{ env.IMAGE_TAG }}
+        run: breeze prod-image pull --tag-as-latest
+        env:
+          PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
+      - name: "Setup python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{needs.build-info.outputs.default-python-version}}
+          cache: 'pip'
+          cache-dependency-path: ./dev/requirements.txt
+      - name: "Test examples of PROD image building"
+        # yamllint disable-line rule:line-length
+        run: >
+          cd ./docker_tests &&
+          python -m pip install -r requirements.txt &&
+          TEST_IMAGE="ghcr.io/${{env.GITHUB_REPOSITORY}}/${{needs.build-info.outputs.default-branch}}/prod/python${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
+          python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes
+        env:
+          PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
+
+
   test-docker-compose-quick-start:
     timeout-minutes: 60
     name: "Docker-compose quick start with PROD image verifying"

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -174,7 +174,7 @@ repos:
         pass_filenames: false
         require_serial: true
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.7
+    rev: v0.1.8
     hooks:
       # Since ruff makes use of multiple cores we _purposefully_ don't run this in docker so it can use the
       # host CPU to it's fullest
@@ -441,7 +441,7 @@ repos:
         name: Update version to the latest version in the documentation
         entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py
         language: python
-        files: ^docs
+        files: ^docs|^airflow/__init__.py$
         pass_filenames: false
         additional_dependencies: ['rich>=12.4.4']
       - id: check-pydevd-left-in-code

diff --git a/CI_DIAGRAMS.md b/CI_DIAGRAMS.md
@@ -60,8 +60,6 @@ sequenceDiagram
         Note over Tests: OpenAPI client gen
     and
         Note over Tests: React WWW tests
-    and
-        Note over Tests: Test examples<br>PROD image building
     and
         Note over Tests: Test git clone on Windows
     and
@@ -152,6 +150,11 @@ sequenceDiagram
         end
     end
     par
+        opt
+            GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
+            Note over Tests: Test examples<br>PROD image building
+        end
+    and
         opt
             GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
             Note over Tests: Run Kubernetes <br>tests
@@ -343,8 +346,6 @@ sequenceDiagram
         Note over Tests: OpenAPI client gen
     and
         Note over Tests: React WWW tests
-    and
-        Note over Tests: Test examples<br>PROD image building
     and
         Note over Tests: Test git clone on Windows
     end
@@ -400,6 +401,9 @@ sequenceDiagram
         Artifacts ->> Tests: Download source,pypi,no-providers constraints
         Note over Tests: Display constraints diff
         Tests ->> Airflow Repo: Push constraints if changed to 'constraints-BRANCH'
+    and
+        GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
+        Note over Tests: Test examples<br>PROD image building
     and
         GitHub Registry ->> Tests: Pull PROD Image<br>[COMMIT_SHA]
         Note over Tests: Run Kubernetes <br>tests

diff --git a/CONTRIBUTORS_QUICK_START_GITPOD.rst b/CONTRIBUTORS_QUICK_START_GITPOD.rst
@@ -62,8 +62,8 @@ Gitpod default image have all the required packages installed.
 
   If you see below warning - it means that you hit `known issue <https://github.com/pypa/pipx/issues/1092>`_
   with ``packaging`` version 23.2:
-  ⚠️ Ignoring --editable install option. pipx disallows it for anything but a local path,
-  to avoid having to create a new src/ directory.
+  ⚠️ Ignoring ``--editable install`` option. ``pipx`` disallows it for anything but a local path,
+  to avoid having to create a new ``src/`` directory.
 
   The workaround is to downgrade packaging to 23.1 and re-running the ``pipx install`` command. for example
   by running ``pip install "packaging<23.2"``.