Substrait bindings (#145)

* introduce bindings for substrait and new substrait module

* documention for the substrait module including examples

* add pytest for substrait

* cargo fmt

* Add action for setting up build environment and also install protobuf compiler

* pin to a specific revision instead of master

* pin to specific version, wrong version last time

* Remove --locked since building from github revisions

* add lock file

* adjust path to include protoc

* set PROTOC evn variable

* change permissions of directory containing protobuf compiler since permissions error was encountered

* Try permissions again

* Try permissions again

* Try permissions again

* Try permissions again

* Try permissions again

* Alter permissions

* Alter permissions

* Alter permissions

.github/actions/setup-builder/action.yaml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder
+description: 'Prepare Rust Build Environment'
+inputs:
+ rust-version:
+ description: 'version of rust to install (e.g. stable)'
+ required: true
+ default: 'stable'
+runs:
+ using: "composite"
+ steps:
+ - name: Install Build Dependencies
+ shell: bash
+ run: |
+ apt-get update
+ apt-get install -y protobuf-compiler
+ - name: Setup Rust toolchain
+ shell: bash
+ run: |
+ echo "Installing ${{ inputs.rust-version }}"
+ rustup toolchain install ${{ inputs.rust-version }}
+ rustup default ${{ inputs.rust-version }}
+ rustup component add rustfmt

.github/workflows/build.yml

@@ -60,11 +60,6 @@ jobs:
  with:
  toolchain: stable
 
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install maturin==0.14.2
-
  - run: rm LICENSE.txt
  - name: Download LICENSE.txt
  uses: actions/download-artifact@v3
@@ -73,7 +68,7 @@ jobs:
  path: .
 
  - name: Build Python package
- run: maturin build --release --strip --locked
+ run: maturin build --release --strip
 
  - name: List Windows wheels
  if: matrix.os == 'windows-latest'
@@ -103,12 +98,12 @@ jobs:
  path: .
  - run: cat LICENSE.txt
  - name: Build wheels
- run: |
-  export RUSTFLAGS='-C target-cpu=skylake'
- docker run --rm -v $(pwd):/io \
-  --workdir /io \
-  ghcr.io/pyo3/maturin:v0.14.2 \
-  build --release --manylinux 2014 --locked
+ uses: PyO3/maturin-action@v1
+ with:
+ rust-toolchain: nightly
+ target: x86_64
+ manylinux: auto
+ args: --release --manylinux 2014
  - name: Archive wheels
  uses: actions/upload-artifact@v3
  with:
@@ -129,12 +124,11 @@ jobs:
  path: .
  - run: cat LICENSE.txt
  - name: Build sdist
- run: |
- docker run --rm -v $(pwd):/io \
- --workdir /io \
- ghcr.io/pyo3/maturin:v0.14.2 \
- build --release --sdist --locked
- find target/wheels/
+ uses: PyO3/maturin-action@v1
+ with:
+ rust-toolchain: stable
+ manylinux: auto
+ args: --release --sdist --out dist
  - name: Archive wheels
  uses: actions/upload-artifact@v2
  with:

.github/workflows/test.yaml

@@ -53,6 +53,19 @@ jobs:
  toolchain: ${{ matrix.toolchain }}
  override: true
 
+ - name: Install protobuf compiler
+ shell: bash
+ run: |
+ mkdir -p $HOME/d/protoc
+ cd $HOME/d/protoc
+ export PROTO_ZIP="protoc-21.4-linux-x86_64.zip"
+ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+ unzip $PROTO_ZIP
+ export PATH=$PATH:$HOME/d/protoc/bin
+ export PROTOC=$HOME/d/protoc/bin
+ sudo chown -R $(whoami) $HOME/d/protoc
+ protoc --version
+
  - name: Setup Python
  uses: actions/setup-python@v4
  with:
@@ -99,9 +112,22 @@ jobs:
  flake8 --exclude venv --ignore=E501,W503
  black --line-length 79 --diff --check .
 
+ - name: Build wheels
+ uses: PyO3/maturin-action@v1
+ with:
+ command: build
+ args: --release --out dist
+
  - name: Run tests
  run: |
  git submodule update --init
- source venv/bin/activate
- maturin develop --locked
+ export PATH=$PATH:$HOME/d/protoc/bin
+ export PROTOC=$HOME/d/protoc/bin
+ sudo chown -R $(whoami) $HOME/d/protoc
+ ls -l $HOME/d/protoc/
+ ls -l $HOME/d/protoc/bin
+ pip install datafusion-python --no-index --find-links dist --force-reinstall
+ pip install pytest
+ cargo clean
+ maturin develop
  RUST_BACKTRACE=1 pytest -v .