Remove ibis version cap (#533)

* Remove ibis version cap

* fix ray

* update

* update

* Fix duckdb tests

* update

* update

* update

.github/workflows/test_win.yml

@@ -34,6 +34,6 @@ jobs:
  - name: Install dependencies
  run: pip install -r requirements.txt
  - name: Install pyarrow
- run: pip install pyarrow==7.0.0
+ run: pip install pyarrow==8.0.0
  - name: Test
  run: python -m pytest --reruns 2 --only-rerun 'Overflow in cast' tests/fugue tests/fugue_dask tests/fugue_ibis tests/fugue_duckdb

.readthedocs.yaml

@@ -0,0 +1,17 @@
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+ os: ubuntu-20.04
+ tools:
+ python: "3.10"
+ jobs:
+ pre_install:
+ - pip install -U pip
+
+sphinx:
+ configuration: docs/conf.py
+
+python:
+ install:
+ - requirements: requirements.txt

fugue_ibis/execution_engine.py

@@ -92,11 +92,11 @@ def join(
  _df2 = self.to_df(df2)
  key_schema, end_schema = get_join_schemas(_df1, _df2, how=how, on=on)
  on_fields = [_df1.native[k] == _df2.native[k] for k in key_schema]
- if ibis.__version__ < "6":
+ if ibis.__version__ < "6": # pragma: no cover
  suffixes: Dict[str, Any] = dict(suffixes=("", _JOIN_RIGHT_SUFFIX))
- else: # pragma: no cover
+ else:
  # breaking change in ibis 6.0
- suffixes = dict(lname="", rname=_JOIN_RIGHT_SUFFIX)
+ suffixes = dict(lname="", rname="{name}" + _JOIN_RIGHT_SUFFIX)
  if how.lower() == "cross":
  tb = _df1.native.cross_join(_df2.native, **suffixes)
  elif how.lower() == "right_outer":

setup.py

@@ -38,7 +38,7 @@ def get_version() -> str:
  keywords="distributed spark dask ray sql dsl domain specific language",
  url="http://github.com/fugue-project/fugue",
  install_requires=[
- "triad>=0.9.3",
+ "triad>=0.9.4",
  "adagio>=0.2.4",
  ],
  extras_require={
@@ -50,14 +50,19 @@ def get_version() -> str:
  "pyarrow>=7.0.0",
  "pandas>=2.0.2",
  ],
- "ray": ["ray[data]>=2.4.0", "duckdb>=0.5.0", "pyarrow>=6.0.1"],
+ "ray": [
+ "ray[data]>=2.4.0",
+ "duckdb>=0.5.0",
+ "pyarrow>=6.0.1",
+ "pandas<2.2",
+ ],
  "duckdb": SQL_DEPENDENCIES
  + [
  "duckdb>=0.5.0",
  "numpy",
  ],
  "polars": ["polars"],
- "ibis": SQL_DEPENDENCIES + ["ibis-framework>=3.2.0,<6"],
+ "ibis": SQL_DEPENDENCIES + ["ibis-framework"],
  "notebook": ["notebook", "jupyterlab", "ipython>=7.10.0"],
  "all": SQL_DEPENDENCIES
  + [
@@ -70,8 +75,8 @@ def get_version() -> str:
  "ipython>=7.10.0",
  "duckdb>=0.5.0",
  "pyarrow>=6.0.1",
- "pandas>=2.0.2",
- "ibis-framework>=3.2.0,<6",
+ "pandas>=2.0.2,<2.2", # because of Ray
+ "ibis-framework",
  "polars",
  ],
  },

tests/fugue_duckdb/test_execution_engine.py

@@ -3,15 +3,13 @@
 import duckdb
 import pandas as pd
 import pyarrow as pa
-import pytest
 from pytest import raises
 
 import fugue.api as fa
 import fugue.test as ft
 from fugue import ArrowDataFrame, DataFrame, FugueWorkflow, fsql
 from fugue.api import engine_context
 from fugue.plugins import infer_execution_engine
-from fugue_duckdb import DuckExecutionEngine
 from fugue_duckdb.dataframe import DuckDataFrame
 from fugue_test.builtin_suite import BuiltInTests
 from fugue_test.execution_suite import ExecutionEngineTests
@@ -109,39 +107,55 @@ def test_builtin_connection():
 
 
 def test_configs():
- dag = FugueWorkflow()
- df = dag.df([[None], [1]], "a:double")
- df = dag.select("SELECT * FROM ", df, "ORDER BY a LIMIT 1")
- df.assert_eq(dag.df([[None]], "a:double"))
-
- dag.run(
- "duckdb",
- {
- "fugue.duckdb.pragma.threads": 2,
- "fugue.duckdb.pragma.default_null_order": "NULLS FIRST",
- },
+ df = fa.as_pandas(
+ fa.fugue_sql(
+ """
+ SELECT name, value FROM duckdb_settings()
+ WHERE name IN ('threads') 
+ """,
+ engine="duckdb",
+ engine_conf={"fugue.duckdb.pragma.threads": 1},
+ )
  )
+ assert df.value.iloc[0] == "1"
 
- dag = FugueWorkflow()
- df = dag.df([[None], [1]], "a:double")
- df = dag.select("SELECT * FROM ", df, "ORDER BY a LIMIT 1")
- df.assert_eq(dag.df([[1]], "a:double"))
-
- dag.run(
- "duckdb",
- {
- "fugue.duckdb.pragma.threads": 2,
- "fugue.duckdb.pragma.default_null_order": "NULLS LAST",
- },
+ df = fa.as_pandas(
+ fa.fugue_sql(
+ """
+ SELECT name, value FROM duckdb_settings()
+ WHERE name IN ('threads') 
+ """,
+ engine="duckdb",
+ engine_conf={"fugue.duckdb.pragma.threads": 3},
+ )
  )
+ assert df.value.iloc[0] == "3"
 
  with raises(ValueError):
  # invalid config format
- dag.run("duckdb", {"fugue.duckdb.pragma.threads;xx": 2})
+ df = fa.as_pandas(
+ fa.fugue_sql(
+ """
+ SELECT name, value FROM duckdb_settings()
+ WHERE name IN ('threads') 
+ """,
+ engine="duckdb",
+ engine_conf={"fugue.duckdb.pragma.threads;xx": 3},
+ )
+ )
 
  with raises(Exception):
  # non-existent config
- dag.run("duckdb", {"fugue.duckdb.pragma.threads_xx": 2})
+ df = fa.as_pandas(
+ fa.fugue_sql(
+ """
+ SELECT name, value FROM duckdb_settings()
+ WHERE name IN ('threads') 
+ """,
+ engine="duckdb",
+ engine_conf={"fugue.duckdb.pragma.threads_xx": 3},
+ )
+ )
 
 
 def test_annotations():