Remove SQLite Support (#443)

* Remove SQLite

* Remove SQLite

RELEASE.md

@@ -4,6 +4,7 @@
 
 -   [430](https://github.com/fugue-project/fugue/issues/430) Support Polars DataFrames
 -   [434](https://github.com/fugue-project/fugue/issues/434) Make Transformations data format aware
+-   [408](https://github.com/fugue-project/fugue/issues/408) Remove SQLite support
 
 ## 0.8.1
 

fugue/__init__.py

@@ -54,7 +54,6 @@
     NativeExecutionEngine,
     PandasMapEngine,
     QPDPandasEngine,
-    SqliteEngine,
 )
 from fugue.extensions.creator import Creator, creator, register_creator
 from fugue.extensions.outputter import Outputter, outputter, register_outputter

fugue/dev.py

@@ -32,7 +32,6 @@
 from fugue.execution.native_execution_engine import (
     PandasMapEngine,
     QPDPandasEngine,
-    SqliteEngine,
 )
 from fugue.rpc import (
     EmptyRPCHandler,

fugue/execution/__init__.py

@@ -10,8 +10,4 @@
     register_execution_engine,
     register_sql_engine,
 )
-from .native_execution_engine import (
-    NativeExecutionEngine,
-    QPDPandasEngine,
-    SqliteEngine,
-)
+from .native_execution_engine import NativeExecutionEngine, QPDPandasEngine

fugue/execution/factory.py

@@ -492,9 +492,6 @@ def make_sql_engine(
 
             # S2(engine)
             make_sql_engine("s2", engine)
-
-            # SqliteEngine(engine)
-            make_sql_engine(SqliteEngine)
     """
     if isinstance(engine, SQLEngine):
         assert_or_throw(
@@ -554,9 +551,6 @@ def parse_sql_engine(
 
             # S2(engine)
             make_sql_engine("s2", engine)
-
-            # SqliteEngine(engine)
-            make_sql_engine(SqliteEngine)
     """
     if engine is None or (isinstance(engine, str) and engine == ""):
         assert_or_throw(

fugue/execution/native_execution_engine.py

@@ -5,7 +5,6 @@
 import pandas as pd
 from qpd_pandas import run_sql_on_pandas
 from qpd_pandas.engine import PandasUtils
-from sqlalchemy import create_engine
 from triad import Schema
 from triad.collections.dict import IndexedOrderedDict
 from triad.collections.fs import FileSystem
@@ -38,29 +37,6 @@
 )
 
 
-class SqliteEngine(SQLEngine):
-    """Sqlite execution implementation.
-
-    :param execution_engine: the execution engine this sql engine will run on
-    """
-
-    @property
-    def is_distributed(self) -> bool:
-        return False
-
-    @property
-    def dialect(self) -> Optional[str]:
-        return "sqlite"
-
-    def select(self, dfs: DataFrames, statement: StructuredRawSQL) -> DataFrame:
-        _dfs, _sql = self.encode(dfs, statement)
-        sql_engine = create_engine("sqlite:///:memory:")
-        for k, v in _dfs.items():
-            v.as_pandas().to_sql(k, sql_engine, if_exists="replace", index=False)
-        df = pd.read_sql_query(_sql, sql_engine)
-        return PandasDataFrame(df)
-
-
 class QPDPandasEngine(SQLEngine):
     """QPD execution implementation.
 

fugue/registry.py

@@ -2,7 +2,6 @@
 from fugue.execution.native_execution_engine import (
     NativeExecutionEngine,
     QPDPandasEngine,
-    SqliteEngine,
 )
 
 
@@ -25,7 +24,6 @@ def _register_engines() -> None:
     register_execution_engine(
         "pandas", lambda conf: NativeExecutionEngine(conf), on_dup="ignore"
     )
-    register_sql_engine("sqlite", lambda engine: SqliteEngine(engine), on_dup="ignore")
     register_sql_engine(
         "qpdpandas", lambda engine: QPDPandasEngine(engine), on_dup="ignore"
     )

setup.py

@@ -35,7 +35,6 @@ def get_version() -> str:
         "adagio>=0.2.4",
         "qpd>=0.4.0",
         "fugue-sql-antlr>=0.1.5",
-        "sqlalchemy",
         "sqlglot",
         "pyarrow>=0.15.1",
         "pandas>=1.0.2",

tests/fugue/execution/test_factory.py

@@ -5,7 +5,6 @@
 
 from fugue import (
     NativeExecutionEngine,
-    SqliteEngine,
     make_execution_engine,
     make_sql_engine,
     register_default_execution_engine,
@@ -20,13 +19,12 @@
     is_pandas_or,
     make_execution_engine,
     make_sql_engine,
-    parse_execution_engine,
-    parse_sql_engine,
     register_default_execution_engine,
     register_default_sql_engine,
     register_execution_engine,
     register_sql_engine,
 )
+from fugue_duckdb import DuckDBEngine
 
 
 class _MockExecutionEngine(NativeExecutionEngine):
@@ -50,7 +48,7 @@ def __init__(self, obj: Any, conf: Any, other: int = 0):
         self.other = other
 
 
-class _MockSQlEngine(SqliteEngine):
+class _MockSQlEngine(DuckDBEngine):
     def __init__(self, execution_engine, other: int = 1):
         super().__init__(execution_engine)
         self.other = other
@@ -189,7 +187,7 @@ def test_global_funcs():
     )
     assert isinstance(make_execution_engine(), _MockExecutionEngine)
 
-    se = SqliteEngine(make_execution_engine())
+    se = DuckDBEngine(make_execution_engine())
     assert make_sql_engine(se) is se
     assert not isinstance(
         make_sql_engine(None, make_execution_engine()), _MockSQlEngine
@@ -229,7 +227,7 @@ def test_make_execution_engine():
     assert e.conf.get_or_throw(FUGUE_CONF_SQL_IGNORE_CASE, bool)
 
     e = make_execution_engine(
-        (NativeExecutionEngine, "sqlite"), {FUGUE_CONF_SQL_IGNORE_CASE: True}
+        (NativeExecutionEngine, "duckdb"), {FUGUE_CONF_SQL_IGNORE_CASE: True}
     )
     assert isinstance(e, NativeExecutionEngine)
     assert e.conf.get_or_throw(FUGUE_CONF_SQL_IGNORE_CASE, bool)
@@ -239,14 +237,14 @@ def test_make_execution_engine():
     assert e.conf.get_or_throw(FUGUE_CONF_SQL_IGNORE_CASE, bool)
 
     e = make_execution_engine(
-        (NativeExecutionEngine({FUGUE_CONF_SQL_IGNORE_CASE: True}), "sqlite")
+        (NativeExecutionEngine({FUGUE_CONF_SQL_IGNORE_CASE: True}), "duckdb")
     )
     e = make_execution_engine(e)
     assert isinstance(e, NativeExecutionEngine)
     assert e.conf.get_or_throw(FUGUE_CONF_SQL_IGNORE_CASE, bool)
-    assert isinstance(e.sql_engine, SqliteEngine)
+    assert isinstance(e.sql_engine, DuckDBEngine)
 
-    assert isinstance(make_sql_engine(None, e), SqliteEngine)
+    assert isinstance(make_sql_engine(None, e), DuckDBEngine)
 
     # MUST HAVE THIS STEP, or other tests will fail
     _reset()

tests/fugue/execution/test_naive_execution_engine.py

@@ -1,54 +1,12 @@
 import pandas as pd
 import pyarrow as pa
 
-from fugue import FugueWorkflow, NativeExecutionEngine, QPDPandasEngine, SqliteEngine
+from fugue import FugueWorkflow, NativeExecutionEngine, QPDPandasEngine
 from fugue.execution.execution_engine import _get_file_threshold
 from fugue_test.builtin_suite import BuiltInTests
 from fugue_test.execution_suite import ExecutionEngineTests
 
 
-class NativeExecutionEngineSqliteTests(ExecutionEngineTests.Tests):
-    def make_engine(self):
-        e = NativeExecutionEngine(dict(test=True))
-        e.set_sql_engine(SqliteEngine(e))
-        return e
-
-    def test_properties(self):
-        assert not self.engine.is_distributed
-        assert not self.engine.map_engine.is_distributed
-        assert not self.engine.sql_engine.is_distributed
-
-    def test_map_with_dict_col(self):
-        # TODO: add back
-        return
-
-
-class NativeExecutionEngineBuiltInSqliteTests(BuiltInTests.Tests):
-    def make_engine(self):
-        e = NativeExecutionEngine(dict(test=True))
-        e.set_sql_engine(SqliteEngine(e))
-        return e
-
-    def test_yield_table(self):
-        pass
-
-    def test_annotation(self):
-        def m_c(engine: NativeExecutionEngine) -> pa.Table:
-            return pa.Table.from_pandas(pd.DataFrame([[0]], columns=["a"]))
-
-        def m_p(engine: NativeExecutionEngine, df: pa.Table) -> pa.Table:
-            return df
-
-        def m_o(engine: NativeExecutionEngine, df: pa.Table) -> None:
-            assert 1 == df.to_pandas().shape[0]
-
-        with FugueWorkflow() as dag:
-            df = dag.create(m_c).process(m_p)
-            df.assert_eq(dag.df([[0]], "a:long"))
-            df.output(m_o)
-        dag.run(self.engine)
-
-
 class NativeExecutionEngineQPDTests(ExecutionEngineTests.Tests):
     def make_engine(self):
         e = NativeExecutionEngine(dict(test=True))

tests/fugue/sql/test_workflow.py

@@ -114,7 +114,7 @@ def test_jinja_keyword_in_sql():
         OUTPUT df, df2 USING assert_eq
         """
         )
-    dag.run(("", "sqlite"))
+    dag.run(("", "duckdb"))
 
 
 def test_use_df(tmpdir):

tests/fugue/sql/test_workflow_parse.py

@@ -1,4 +1,3 @@
-import json
 from typing import Any, Iterable, List
 
 from fugue_sql_antlr import FugueSQLParser
@@ -8,21 +7,20 @@
 from triad.utils.convert import get_caller_global_local_vars
 
 from fugue import (
-    DataFrame,
     DataFrames,
     FugueWorkflow,
     LocalDataFrame,
     OutputTransformer,
     PartitionSpec,
-    SqliteEngine,
     WorkflowDataFrame,
     WorkflowDataFrames,
     module,
     register_sql_engine,
 )
 from fugue.extensions.transformer.convert import _to_output_transformer
-from fugue.sql._visitors import FugueSQLHooks, _Extensions, _VisitorBase
+from fugue.sql._visitors import FugueSQLHooks, _Extensions
 from fugue.exceptions import FugueSQLError
+from fugue.execution.native_execution_engine import QPDPandasEngine
 
 
 def test_create_data():
@@ -421,7 +419,7 @@ def test_select_with():
 
 
 def test_select_plus_engine():
-    class MockEngine(SqliteEngine):
+    class MockEngine(QPDPandasEngine):
         def __init__(self, execution_engine, p: int = 0):
             super().__init__(execution_engine)
             self.p = p

tests/fugue_ibis/test_dataframe.py

@@ -24,7 +24,8 @@ def setUpClass(cls):
     def df(self, data: Any = None, schema: Any = None) -> DuckDataFrame:
         df = ArrowDataFrame(data, schema)
         name = f"_{id(df.native)}"
-        self._con.con.execute("register", (name, df.native))
+        # self._con.con.execute("register", (name, df.native))
+        self._con.register(df.native, name)
         return MockDuckDataFrame(self._con.table(name), schema=schema)
 
     def test_init_df(self):
@@ -71,12 +72,14 @@ def setUpClass(cls):
     def df(self, data: Any = None, schema: Any = None):
         df = ArrowDataFrame(data, schema)
         name = f"_{id(df.native)}"
-        self._con.con.execute("register", (name, df.native))
+        # self._con.con.execute("register", (name, df.native))
+        self._con.register(df.native, name)
         return MockDuckDataFrame(self._con.table(name), schema=schema).native
 
     def to_native_df(self, pdf: pd.DataFrame) -> Any:
         name = f"_{id(pdf)}"
-        self._con.con.execute("register", (name, pa.Table.from_pandas(pdf)))
+        # self._con.con.execute("register", (name, pa.Table.from_pandas(pdf)))
+        self._con.register(pa.Table.from_pandas(pdf), name)
         return self._con.table(name)
 
     def test_is_local(self):