disable sftp + delta test (#2052)

* fix delta tests

* update workflows

.github/workflows/test_common.yml

@@ -116,7 +116,7 @@ jobs:
         shell: cmd
 
       - name: Install pyarrow
-        run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk
+        run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk && poetry run pip install pyarrow==15.0.2
 
       - run: |
           poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow
@@ -129,7 +129,7 @@ jobs:
         shell: cmd
 
       - name: Install pipeline and sources dependencies
-        run: poetry install --no-interaction -E duckdb -E cli -E parquet -E deltalake -E sql_database --with sentry-sdk,pipeline,sources
+        run: poetry install --no-interaction -E duckdb -E cli -E parquet -E deltalake -E sql_database --with sentry-sdk,pipeline,sources && poetry run pip install pyarrow==15.0.2
 
       - run: |
           poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources
@@ -155,6 +155,20 @@ jobs:
         name: Run extract tests Windows
         shell: cmd
 
+      # here we upgrade pyarrow to 17 and run the libs tests again
+      - name: Install pyarrow 17
+        run: poetry run pip install pyarrow==17.0.0
+
+      - run: |
+          poetry run pytest tests/libs
+        if: runner.os != 'Windows'
+        name: Run libs tests Linux/MAC
+      - run: |
+          poetry run pytest tests/libs
+        if: runner.os == 'Windows'
+        name: Run libs tests Windows
+        shell: cmd
+
       # - name: Install Pydantic 1.0
       #   run: pip install "pydantic<2"
 

tests/load/pipeline/test_filesystem_pipeline.py

@@ -33,6 +33,7 @@
     MEMORY_BUCKET,
     FILE_BUCKET,
     AZ_BUCKET,
+    SFTP_BUCKET,
 )
 
 from tests.pipeline.utils import load_table_counts, assert_load_info, load_tables_to_dicts
@@ -222,6 +223,9 @@ def some_source():
         assert table.column("value").to_pylist() == [1, 2, 3, 4, 5]
 
 
+@pytest.mark.skip(
+    reason="pyarrow version check not needed anymore, since we have 17 as a dependency"
+)
 def test_delta_table_pyarrow_version_check() -> None:
     """Tests pyarrow version checking for `delta` table format.
 
@@ -255,7 +259,7 @@ def foo():
     destinations_configs(
         table_format_filesystem_configs=True,
         with_table_format="delta",
-        bucket_exclude=(MEMORY_BUCKET),
+        bucket_exclude=(MEMORY_BUCKET, SFTP_BUCKET),
     ),
     ids=lambda x: x.name,
 )
@@ -982,7 +986,7 @@ def parent_delta():
     destinations_configs(
         table_format_filesystem_configs=True,
         with_table_format="delta",
-        bucket_subset=(FILE_BUCKET,),
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )

tests/load/pipeline/test_merge_disposition.py

@@ -40,6 +40,7 @@
     DestinationTestConfiguration,
     FILE_BUCKET,
     AZ_BUCKET,
+    SFTP_BUCKET,
 )
 
 

tests/load/utils.py

@@ -1004,7 +1004,7 @@ def prepare_load_package(
 def sequence_generator() -> Generator[List[Dict[str, str]], None, None]:
     count = 1
     while True:
-        yield [{"content": str(count + i)} for i in range(2000)]
+        yield [{"content": str(count + i)} for i in range(3)]
         count += 3