CI Fixes

.github/workflows/build.yml

@@ -97,9 +97,15 @@ jobs:
           name: python-wheel-license
           path: .
       - run: cat LICENSE.txt
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          version: '3.x'
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
+          env:
+            RUST_BACKTRACE: 1
           rust-toolchain: nightly
           target: x86_64
           manylinux: auto

.github/workflows/test.yaml

@@ -53,18 +53,10 @@ jobs:
           toolchain: ${{ matrix.toolchain }}
           override: true
 
-      - name: Install protobuf compiler
-        shell: bash
-        run: |
-          mkdir -p $HOME/d/protoc
-          cd $HOME/d/protoc
-          export PROTO_ZIP="protoc-21.4-linux-x86_64.zip"
-          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-          unzip $PROTO_ZIP
-          export PATH=$PATH:$HOME/d/protoc/bin
-          export PROTOC=$HOME/d/protoc/bin
-          sudo chown -R $(whoami) $HOME/d/protoc
-          protoc --version
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          version: '3.x'
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -112,22 +104,11 @@ jobs:
           flake8 --exclude venv --ignore=E501,W503
           black --line-length 79 --diff --check .
 
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          command: build
-          args: --release --out dist
-
       - name: Run tests
+        env:
+          RUST_BACKTRACE: 1
         run: |
           git submodule update --init
-          export PATH=$PATH:$HOME/d/protoc/bin
-          export PROTOC=$HOME/d/protoc/bin
-          sudo chown -R $(whoami) $HOME/d/protoc
-          ls -l $HOME/d/protoc/
-          ls -l $HOME/d/protoc/bin
-          pip install datafusion-python --no-index --find-links dist --force-reinstall
-          pip install pytest
-          cargo clean
-          maturin develop
-          RUST_BACKTRACE=1 pytest -v .
+          source venv/bin/activate
+          pip install -e . -vv
+          pytest -v .

Cargo.toml

@@ -34,11 +34,11 @@ default = ["mimalloc"]
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
 pyo3 = { version = "~0.17.3", features = ["extension-module", "abi3", "abi3-py37"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb", features = ["pyarrow", "avro"] }
-datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb" }
-datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb" }
-datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb", features = ["pyarrow"] }
-datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb" }
+datafusion = { version = "17.0.0", features = ["pyarrow", "avro"] }
+datafusion-expr = "17.0.0"
+datafusion-optimizer = "17.0.0"
+datafusion-common = { version = "17.0.0", features = ["pyarrow"] }
+datafusion-substrait = "17.0.0"
 uuid = { version = "1.2", features = ["v4"] }
 mimalloc = { version = "*", optional = true, default-features = false }
 async-trait = "0.1"

datafusion/tests/test_dataframe.py

@@ -52,11 +52,12 @@ def struct_df():
 
     return ctx.create_dataframe([[batch]])
 
+
 @pytest.fixture
 def aggregate_df():
     ctx = SessionContext()
-    ctx.register_csv('test', 'testing/data/csv/aggregate_test_100.csv')
-    return ctx.sql('select c1, sum(c2) from test group by c1')
+    ctx.register_csv("test", "testing/data/csv/aggregate_test_100.csv")
+    return ctx.sql("select c1, sum(c2) from test group by c1")
 
 
 def test_select(df):
@@ -271,10 +272,11 @@ def test_logical_plan(aggregate_df):
 
     assert expected == plan.display()
 
-    expected = \
-        "Projection: test.c1, SUM(test.c2)\n" \
-        "  Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
+    expected = (
+        "Projection: test.c1, SUM(test.c2)\n"
+        "  Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
         "    TableScan: test"
+    )
 
     assert expected == plan.display_indent()
 
@@ -286,25 +288,29 @@ def test_optimized_logical_plan(aggregate_df):
 
     assert expected == plan.display()
 
-    expected = \
-        "Projection: test.c1, SUM(test.c2)\n" \
-        "  Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
+    expected = (
+        "Projection: test.c1, SUM(test.c2)\n"
+        "  Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
         "    TableScan: test projection=[c1, c2]"
+    )
 
     assert expected == plan.display_indent()
 
 
 def test_execution_plan(aggregate_df):
     plan = aggregate_df.execution_plan()
 
-    expected = "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
+    expected = (
+        "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
+    )
 
     assert expected == plan.display()
 
-    expected = \
-        "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n" \
-        "  Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
+    expected = (
+        "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
+        "  Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
         "    TableScan: test projection=[c1, c2]"
+    )
 
     indent = plan.display_indent()
 
@@ -317,7 +323,6 @@ def test_execution_plan(aggregate_df):
     assert "CsvExec:" in indent
 
 
-
 def test_repartition(df):
     df.repartition(2)
 

datafusion/tests/test_substrait.py

@@ -16,9 +16,8 @@
 # under the License.
 
 import pyarrow as pa
-import pyarrow.dataset as ds
 
-from datafusion import column, literal, SessionContext
+from datafusion import SessionContext
 from datafusion import substrait as ss
 import pytest
 
@@ -39,8 +38,14 @@ def test_substrait_serialization(ctx):
     assert ctx.tables() == {"t"}
 
     # For now just make sure the method calls blow up
-    substrait_plan = ss.substrait.serde.serialize_to_plan("SELECT * FROM t", ctx)
-    substrait_bytes = ss.substrait.serde.serialize_bytes("SELECT * FROM t", ctx)
+    substrait_plan = ss.substrait.serde.serialize_to_plan(
+        "SELECT * FROM t", ctx
+    )
+    substrait_bytes = ss.substrait.serde.serialize_bytes(
+        "SELECT * FROM t", ctx
+    )
     substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes)
-    df_logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan)
+    df_logical_plan = ss.substrait.consumer.from_substrait_plan(
+        ctx, substrait_plan
+    )
     substrait_plan = ss.substrait.producer.to_substrait_plan(df_logical_plan)

pyproject.toml

@@ -43,7 +43,7 @@ classifier = [
     "Programming Language :: Rust",
 ]
 dependencies = [
-    "pyarrow>=1",
+    "pyarrow>=6.0.1",
 ]
 
 [project.urls]