CI Fixes

.github/workflows/build.yml

@@ -97,9 +97,15 @@ jobs:
  name: python-wheel-license
  path: .
  - run: cat LICENSE.txt
+ - name: Install Protoc
+ uses: arduino/setup-protoc@v1
+ with:
+ version: '3.x'
  - name: Build wheels
  uses: PyO3/maturin-action@v1
  with:
+ env:
+ RUST_BACKTRACE: 1
  rust-toolchain: nightly
  target: x86_64
  manylinux: auto

.github/workflows/test.yaml

@@ -53,18 +53,10 @@ jobs:
  toolchain: ${{ matrix.toolchain }}
  override: true
 
- - name: Install protobuf compiler
- shell: bash
- run: |
- mkdir -p $HOME/d/protoc
- cd $HOME/d/protoc
- export PROTO_ZIP="protoc-21.4-linux-x86_64.zip"
- curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
- unzip $PROTO_ZIP
- export PATH=$PATH:$HOME/d/protoc/bin
- export PROTOC=$HOME/d/protoc/bin
- sudo chown -R $(whoami) $HOME/d/protoc
- protoc --version
+ - name: Install Protoc
+ uses: arduino/setup-protoc@v1
+ with:
+ version: '3.x'
 
  - name: Setup Python
  uses: actions/setup-python@v4
@@ -119,15 +111,10 @@ jobs:
  args: --release --out dist
 
  - name: Run tests
+ env:
+ RUST_BACKTRACE: 1
  run: |
  git submodule update --init
- export PATH=$PATH:$HOME/d/protoc/bin
- export PROTOC=$HOME/d/protoc/bin
- sudo chown -R $(whoami) $HOME/d/protoc
- ls -l $HOME/d/protoc/
- ls -l $HOME/d/protoc/bin
- pip install datafusion-python --no-index --find-links dist --force-reinstall
- pip install pytest
- cargo clean
- maturin develop
- RUST_BACKTRACE=1 pytest -v .
+ source venv/bin/activate
+ pip install -e . --vv
+ pytest -v .

Cargo.toml

@@ -34,11 +34,11 @@ default = ["mimalloc"]
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
 pyo3 = { version = "~0.17.3", features = ["extension-module", "abi3", "abi3-py37"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb", features = ["pyarrow", "avro"] }
-datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb" }
-datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb" }
-datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb", features = ["pyarrow"] }
-datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "5238e8c97f998b4d2cb9fab85fb182f325a1a7fb" }
+datafusion = { version = "17.0.0", features = ["pyarrow", "avro"] }
+datafusion-expr = "17.0.0"
+datafusion-optimizer = "17.0.0"
+datafusion-common = { version = "17.0.0", features = ["pyarrow"] }
+datafusion-substrait = "17.0.0"
 uuid = { version = "1.2", features = ["v4"] }
 mimalloc = { version = "*", optional = true, default-features = false }
 async-trait = "0.1"

datafusion/tests/test_dataframe.py

@@ -52,11 +52,12 @@ def struct_df():
 
  return ctx.create_dataframe([[batch]])
 
+
 @pytest.fixture
 def aggregate_df():
  ctx = SessionContext()
- ctx.register_csv('test', 'testing/data/csv/aggregate_test_100.csv')
- return ctx.sql('select c1, sum(c2) from test group by c1')
+ ctx.register_csv("test", "testing/data/csv/aggregate_test_100.csv")
+ return ctx.sql("select c1, sum(c2) from test group by c1")
 
 
 def test_select(df):
@@ -271,10 +272,11 @@ def test_logical_plan(aggregate_df):
 
  assert expected == plan.display()
 
- expected = \
- "Projection: test.c1, SUM(test.c2)\n" \
- " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
+ expected = (
+ "Projection: test.c1, SUM(test.c2)\n"
+ " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
  " TableScan: test"
+ )
 
  assert expected == plan.display_indent()
 
@@ -286,25 +288,29 @@ def test_optimized_logical_plan(aggregate_df):
 
  assert expected == plan.display()
 
- expected = \
- "Projection: test.c1, SUM(test.c2)\n" \
- " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
+ expected = (
+ "Projection: test.c1, SUM(test.c2)\n"
+ " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
  " TableScan: test projection=[c1, c2]"
+ )
 
  assert expected == plan.display_indent()
 
 
 def test_execution_plan(aggregate_df):
  plan = aggregate_df.execution_plan()
 
- expected = "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
+ expected = (
+ "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
+ )
 
  assert expected == plan.display()
 
- expected = \
- "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n" \
- " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
+ expected = (
+ "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
+ " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
  " TableScan: test projection=[c1, c2]"
+ )
 
  indent = plan.display_indent()
 
@@ -317,7 +323,6 @@ def test_execution_plan(aggregate_df):
  assert "CsvExec:" in indent
 
 
-
 def test_repartition(df):
  df.repartition(2)
 

datafusion/tests/test_substrait.py

@@ -16,9 +16,8 @@
 # under the License.
 
 import pyarrow as pa
-import pyarrow.dataset as ds
 
-from datafusion import column, literal, SessionContext
+from datafusion import SessionContext
 from datafusion import substrait as ss
 import pytest
 
@@ -39,8 +38,14 @@ def test_substrait_serialization(ctx):
  assert ctx.tables() == {"t"}
 
  # For now just make sure the method calls blow up
- substrait_plan = ss.substrait.serde.serialize_to_plan("SELECT * FROM t", ctx)
- substrait_bytes = ss.substrait.serde.serialize_bytes("SELECT * FROM t", ctx)
+ substrait_plan = ss.substrait.serde.serialize_to_plan(
+ "SELECT * FROM t", ctx
+ )
+ substrait_bytes = ss.substrait.serde.serialize_bytes(
+ "SELECT * FROM t", ctx
+ )
  substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes)
- df_logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan)
+ df_logical_plan = ss.substrait.consumer.from_substrait_plan(
+ ctx, substrait_plan
+ )
  substrait_plan = ss.substrait.producer.to_substrait_plan(df_logical_plan)

pyproject.toml

@@ -43,7 +43,7 @@ classifier = [
  "Programming Language :: Rust",
 ]
 dependencies = [
- "pyarrow>=1",
+ "pyarrow>=6.0.1",
 ]
 
 [project.urls]