From 71931df4b21a0c3add16ddcdefc7f966b0e7a255 Mon Sep 17 00:00:00 2001
From: dave <shrps@posteo.net>
Date: Tue, 12 Nov 2024 12:58:50 +0100
Subject: [PATCH] update workflows

---
 .github/workflows/test_common.yml    | 17 ++++++
 .github/workflows/test_pyarrow17.yml | 80 ----------------------------
 2 files changed, 17 insertions(+), 80 deletions(-)
 delete mode 100644 .github/workflows/test_pyarrow17.yml

diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml
index 72c3825383..a3a3c1fbb4 100644
--- a/.github/workflows/test_common.yml
+++ b/.github/workflows/test_common.yml
@@ -118,6 +118,9 @@ jobs:
       - name: Install pyarrow
         run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk
 
+      - name: Ensure pyarrow 15
+        run: poetry run pip install pyarrow==15.0.2
+
       - run: |
           poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow
         if: runner.os != 'Windows'
@@ -155,6 +158,20 @@ jobs:
         name: Run extract tests Windows
         shell: cmd
 
+      # here we upgrade pyarrow to 17 and run the libs tests again
+      - name: Install pyarrow 17
+        run: poetry run pip install pyarrow==17.0.0
+
+      - run: |
+          poetry run pytest tests/libs
+        if: runner.os != 'Windows'
+        name: Run libs tests Linux/MAC
+      - run: |
+          poetry run pytest tests/libs
+        if: runner.os == 'Windows'
+        name: Run libs tests Windows
+        shell: cmd
+
       # - name: Install Pydantic 1.0
       #   run: pip install "pydantic<2"
 
diff --git a/.github/workflows/test_pyarrow17.yml b/.github/workflows/test_pyarrow17.yml
deleted file mode 100644
index 941469bd4e..0000000000
--- a/.github/workflows/test_pyarrow17.yml
+++ /dev/null
@@ -1,80 +0,0 @@
-
-name: tests marked as needspyarrow17
-
-on:
-  pull_request:
-    branches:
-      - master
-      - devel
-  workflow_dispatch:
-  schedule:
-    - cron:  '0 2 * * *'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-
-  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
-
-  # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752
-  RUNTIME__LOG_LEVEL: ERROR
-  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
-
-  ACTIVE_DESTINATIONS: "[\"filesystem\"]"
-  ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\", \"r2\", \"s3\", \"gs\", \"az\", \"abfss\", \"gdrive\"]" #excludes sftp
-
-jobs:
-  get_docs_changes:
-    name: docs changes
-    uses: ./.github/workflows/get_docs_changes.yml
-    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
-
-  run_pyarrow17:
-    name: needspyarrow17 tests
-    needs: get_docs_changes
-    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    defaults:
-      run:
-        shell: bash
-    runs-on: "ubuntu-latest"
-
-    steps:
-
-      - name: Check out
-        uses: actions/checkout@master
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10.x"
-
-      - name: Install Poetry
-        uses: snok/install-poetry@v1.3.2
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
-
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v3
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-pyarrow17
-
-      - name: Install dependencies
-        run: poetry install --no-interaction --with sentry-sdk --with pipeline -E deltalake -E duckdb -E filesystem -E gs -E s3 -E az
-
-
-      - name: Upgrade pyarrow
-        run: poetry run pip install pyarrow==17.0.0
-
-      - name: create secrets.toml
-        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
-
-      - name: Run needspyarrow17 tests Linux
-        run: |
-          poetry run pytest tests/libs -m "needspyarrow17"
-          poetry run pytest tests/load -m "needspyarrow17"