Support for expressions (#523)

* expr: Rust based expressions

* Integrate expr with filter and assign

* Add eval and typeof to expr and move astype to TypeExpr

* Add eval and typeof to expr and move astype to TypeExpr

* Github actions using maturin

.github/workflows/packagr_publish.yml

@@ -1,86 +1,87 @@
-name: Packagr Publish
+name: Build, Test, and Publish
 
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push but only for the main branch
   push:
     branches: [ main ]
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
-  build:
-    # The type of runner that the job will run on
+  build-and-publish:
     runs-on: ubuntu-latest
+    name: Build, Test, and Publish Package
 
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    name: Run tests and publish latest version if succcessful
     steps:
-      #----------------------------------------------
-      #       check-out repo and set-up python
-      #----------------------------------------------
-      - name: Check out repository
+      # Checkout the Python client repository
+      - name: Check out client repository (Python)
         uses: actions/checkout@v2
+        with:
+          path: 'client'
 
-      - name: Set up python
-        id: setup-python
+      # Checkout the Rust server repository
+      - name: Check out server repository (Rust)
+        uses: actions/checkout@v2
+        with:
+          repository: 'fennel-ai/server'
+          token: ${{ secrets.SERVER_CHECKOUT }}
+          path: 'server'
+          ref: 'main'
+
+      # Set up Python environment
+      - name: Set up Python
         uses: actions/setup-python@v2
         with:
           python-version: 3.12
 
-      #----------------------------------------------
-      #  -----  install & configure poetry  -----
-      #----------------------------------------------
+      # Install Poetry
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
 
-      #----------------------------------------------
-      #       load cached venv if cache exists
-      #----------------------------------------------
+      # Load cached Python virtual environment if it exists
       - name: Load cached venv
         id: cached-poetry-dependencies
         uses: actions/cache@v2
         with:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
-      #----------------------------------------------
-      # install dependencies if cache does not exist
-      #----------------------------------------------
+
+      # Install Python dependencies if the cache does not exist
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction --no-root
 
-      #----------------------------------------------
-      # install your root project, if required
-      #----------------------------------------------
-      - name: Install library
-        run: poetry install --no-interaction
+      # Set up Rust toolchain
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          default: true
+
+      # Install protobuf compiler (protoc)
+      - name: Install protobuf compiler (protoc)
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      # Install Maturin to build the package
+      - name: Install Maturin
+        run: python -m pip install maturin
+
+      # Build the package using Maturin
+      - name: Build the package using Maturin
+        run: maturin build --release --manifest-path ./server/fennel_data_lib/Cargo.toml --out dist --no-sdist
 
-      # Run test suite
+      # Install the built package
+      - name: Install built package
+        run: pip install dist/*.whl
+
+      # Run Python tests
       - name: Run tests
         run: |
           source .venv/bin/activate
-          poetry run pytest . --run_slow
-          poetry run python -m unittest discover fennel
-
+          poetry run pytest fennel --run_slow
 
-      #----------------------------------------------
-      # publish to pypi - this is run only if the previous steps are successful
-      #----------------------------------------------
-      - name: Publish to Pypi
+      # Publish the package to PyPI if all tests pass
+      - name: Publish to PyPI
         run: |
-          rm -rf dist/*
-          pip install setuptools wheel twine
-          poetry build
           twine upload --repository pypi dist/* --verbose -u __token__ -p ${{ secrets.PYPI_PACKAGR_ACCESS_TOKEN }}
-          
-       
-
-

.github/workflows/test_linux.yml

@@ -23,9 +23,15 @@ jobs:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
+
+
     # Steps represent a sequence of tasks that will be executed as part of the job
     name: Unit tests with py-version ${{ matrix.python-version }}
     steps:
+      - name: Test GITHUB_TOKEN permissions
+        run: |
+          curl -H "Authorization: token ${{ secrets.SERVER_CHECKOUT }}" https://api.github.com/repos/fennel-ai/server/contents/
+
       #----------------------------------------------
       #       check-out repo and set-up python
       #----------------------------------------------
@@ -36,6 +42,27 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+
+      #----------------------------------------------
+      #       check-out server repo
+      #----------------------------------------------
+      - name: Check out server repository (Rust)
+        uses: actions/checkout@v2
+        with:
+          repository: 'fennel-ai/server'
+          token: ${{ secrets.SERVER_CHECKOUT }}  
+          path: 'server'
+          ref: 'main'  
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          default: true
+
+      - name: Install protobuf compiler (protoc)
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
       #----------------------------------------------
       #  -----  install & configure poetry  -----
       #----------------------------------------------
@@ -59,6 +86,17 @@ jobs:
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction --no-root
+
+    #----------------------------------------------
+      #       install maturin 
+      #----------------------------------------------     
+      - name: Install maturin
+        run: python -m pip install maturin
+
+      - name: Build and link the package using Maturin Develop
+        run: maturin develop --release --manifest-path ./server/fennel_data_lib/Cargo.toml
+
+
       #----------------------------------------------
       # install your root project, if required
       #----------------------------------------------
@@ -69,5 +107,6 @@ jobs:
       - name: Run tests
         run: |
           source .venv/bin/activate
-          poetry run pytest . --run_slow
+          poetry run pytest fennel --run_slow
+          poetry run pytest docs
           poetry run python -m unittest discover fennel

fennel/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## [1.5.0] - 2024-08-04
+- Rust based expressions
+
 ## [1.4.6] - 2024-07-30
 - Add support for indirections in preproc ref type for Protobuf format
 

fennel/client/client.py

@@ -34,9 +34,9 @@
 # Connection timeout i.e. the time spent by the client to establish a
 # connection to the remote machine. NOTE: This should be slightly larger than
 # a multiple of 3 (this is the default TCP retransmission window)
-_DEFAULT_CONNECT_TIMEOUT = 10
+_DEFAULT_CONNECT_TIMEOUT = 300
 # Default request timeout(s).
-_DEFAULT_TIMEOUT = 180
+_DEFAULT_TIMEOUT = 300
 # Name of the default branch
 _MAIN_BRANCH = "main"
 _BRANCH_HEADER_NAME = "X-FENNEL-BRANCH"

fennel/client_tests/test_complex_struct.py

@@ -64,9 +64,9 @@ class MovieInfo:
     def movie_info(cls, movie: Dataset):
         return (
             movie.assign(
-                name="role",
-                dtype=Role,
-                func=lambda x: x[["role_id", "name", "cost"]].apply(
+                "role",
+                Role,
+                lambda x: x[["role_id", "name", "cost"]].apply(
                     lambda z: Role(
                         **{"role_id": z[0], "name": z[1], "cost": z[2]}
                     ),