diff --git a/ci/310-oldest.yaml b/ci/310-oldest.yaml index 5ffe7f1a6..60e5cc745 100644 --- a/ci/310-oldest.yaml +++ b/ci/310-oldest.yaml @@ -27,6 +27,7 @@ dependencies: - numba=0.55 - pyarrow>=7.0 - scikit-learn=1.1 + - sqlalchemy=2.0 - zstd - pip - pip: diff --git a/ci/310.yaml b/ci/310.yaml index 54b4edb41..702dd50ec 100644 --- a/ci/310.yaml +++ b/ci/310.yaml @@ -27,4 +27,5 @@ dependencies: - numba - pyarrow - scikit-learn + - sqlalchemy - zstd diff --git a/ci/311.yaml b/ci/311.yaml index 00988bc1d..43a91a140 100644 --- a/ci/311.yaml +++ b/ci/311.yaml @@ -20,11 +20,12 @@ dependencies: - pytest-cov - pytest-mpl - pytest-xdist - # optional + # optional - geodatasets - joblib - networkx - numba - pyarrow - scikit-learn + - sqlalchemy - zstd diff --git a/ci/312-dev.yaml b/ci/312-dev.yaml index 7a40eb72f..7b0c65dbc 100644 --- a/ci/312-dev.yaml +++ b/ci/312-dev.yaml @@ -25,6 +25,7 @@ dependencies: - packaging - pyarrow - pyproj + - sqlalchemy - zstd - pip - pip: diff --git a/ci/312.yaml b/ci/312.yaml index 72c0bdd97..6063157bb 100644 --- a/ci/312.yaml +++ b/ci/312.yaml @@ -27,6 +27,7 @@ dependencies: # - numba # follow up when numba is ready for 3.12 - pyarrow - scikit-learn + - sqlalchemy - zstd # for docs build action (this env only) - mkdocs-jupyter diff --git a/libpysal/io/iohandlers/__init__.py b/libpysal/io/iohandlers/__init__.py index 83f71eae2..ba4fcc68b 100644 --- a/libpysal/io/iohandlers/__init__.py +++ b/libpysal/io/iohandlers/__init__.py @@ -22,4 +22,4 @@ try: from . import db except: - warnings.warn("SQLAlchemy and Geomet not installed, database I/O disabled") + warnings.warn("SQLAlchemy not installed, database I/O disabled") diff --git a/libpysal/io/iohandlers/db.py b/libpysal/io/iohandlers/db.py index b64eb82c4..a252455fe 100644 --- a/libpysal/io/iohandlers/db.py +++ b/libpysal/io/iohandlers/db.py @@ -1,17 +1,8 @@ from .. import fileio +from shapely import wkb errmsg = "" -try: - try: - from geomet import wkb - except ImportError: - from shapely import wkb -except ImportError: - wkb = None - errmsg += "No WKB parser found. Please install one of the following packages " - errmsg += "to enable this functionality: [geomet, shapely].\n" - try: from sqlalchemy.ext.automap import automap_base from sqlalchemy import create_engine @@ -20,13 +11,14 @@ nosql_mode = False except ImportError: nosql_mode = True - errmsg += "No module named sqlalchemy. Please install" - errmsg += " sqlalchemy to enable this functionality." + errmsg += ( + "No module named sqlalchemy. Please install" + " sqlalchemy to enable this functionality." + ) class SQLConnection(fileio.FileIO): - """Reads an SQL mappable. - """ + """Reads an SQL mappable.""" FORMATS = ["sqlite", "db"] MODES = ["r"] @@ -41,7 +33,7 @@ def __init__(self, *args, **kwargs): self.dbname = args[0] self.Base = automap_base() self._engine = create_engine(self.dbname) - self.Base.prepare(self._engine, reflect=True) + self.Base.prepare(autoload_with=self._engine) self.metadata = self.Base.metadata def read(self, *args, **kwargs): @@ -58,11 +50,9 @@ def close(self): fileio.FileIO.close(self) def _get_gjson(self, tablename: str, geom_column="GEOMETRY"): - gjson = {"type": "FeatureCollection", "features": []} for row in self.session.query(self.metadata.tables[tablename]): - feat = {"type": "Feature", "geometry": {}, "properties": {}} feat["GEOMETRY"] = wkb.loads(getattr(row, geom_column)) @@ -76,7 +66,6 @@ def _get_gjson(self, tablename: str, geom_column="GEOMETRY"): @property def tables(self) -> list: - if not hasattr(self, "_tables"): self._tables = list(self.metadata.tables.keys()) @@ -85,12 +74,12 @@ def tables(self) -> list: @property def session(self): """Create an ``sqlalchemy.orm.Session`` instance. - + Returns ------- self._session : sqlalchemy.orm.Session An ``sqlalchemy.orm.Session`` instance. - + """ # What happens if the session is externally closed? Check for None? diff --git a/libpysal/io/iohandlers/tests/test_db.py b/libpysal/io/iohandlers/tests/test_db.py index cdb219d57..bc6530ffb 100644 --- a/libpysal/io/iohandlers/tests/test_db.py +++ b/libpysal/io/iohandlers/tests/test_db.py @@ -4,6 +4,9 @@ import unittest as ut from .... import examples as pysal_examples +import platform +import shapely + try: import sqlalchemy @@ -11,28 +14,12 @@ except ImportError: missing_sql = True -try: - import geomet - - missing_geomet = False -except ImportError: - missing_geomet = True - - -def to_wkb_point(c): - """Super quick hack that does not actually belong in here.""" - point = {"type": "Point", "coordinates": [c[0], c[1]]} +windows = platform.system() == "Windows" - return geomet.wkb.dumps(point) - -@ut.skipIf( - missing_sql or missing_geomet, - "Missing dependencies: Geomet ({}) & SQLAlchemy ({}).".format( - missing_geomet, missing_sql - ), -) +@ut.skipIf(windows, "Skipping Windows due to `PermissionError`.") +@ut.skipIf(missing_sql, f"Missing dependency: SQLAlchemy ({missing_sql}).") class Test_sqlite_reader(ut.TestCase): def setUp(self): path = pysal_examples.get_path("new_haven_merged.dbf") @@ -40,14 +27,15 @@ def setUp(self): pysal_examples.load_example("newHaven") path = pysal_examples.get_path("new_haven_merged.dbf") df = pdio.read_files(path) - df["GEOMETRY"] = df["geometry"].apply(to_wkb_point) + df["GEOMETRY"] = shapely.to_wkb(shapely.points(df["geometry"].values.tolist())) # This is a hack to not have to worry about a custom point type in the DB del df["geometry"] - engine = sqlalchemy.create_engine("sqlite:///test.db") - conn = engine.connect() + self.dbf = "iohandlers_test_db.db" + engine = sqlalchemy.create_engine(f"sqlite:///{self.dbf}") + self.conn = engine.connect() df.to_sql( "newhaven", - conn, + self.conn, index=True, dtype={ "date": sqlalchemy.types.UnicodeText, # Should convert the df date into a true date object, just a hack again @@ -60,14 +48,15 @@ def setUp(self): ) # This is converted to TEXT as lowest type common sqlite def test_deserialize(self): - db = psopen("sqlite:///test.db") + db = psopen(f"sqlite:///{self.dbf}") self.assertEqual(db.tables, ["newhaven"]) gj = db._get_gjson("newhaven") self.assertEqual(gj["type"], "FeatureCollection") - def tearDown(self): - os.remove("test.db") + self.conn.close() + + os.remove(self.dbf) if __name__ == "__main__": diff --git a/libpysal/weights/tests/test_user.py b/libpysal/weights/tests/test_user.py index 7d6ed1aaa..4a840030c 100644 --- a/libpysal/weights/tests/test_user.py +++ b/libpysal/weights/tests/test_user.py @@ -17,6 +17,7 @@ def test_build_lattice_shapefile(self): user.build_lattice_shapefile(20, 20, of) w = Rook.from_shapefile(of) self.assertEqual(w.n, 400) + os.remove("lattice.dbf") os.remove("lattice.shp") os.remove("lattice.shx") diff --git a/pyproject.toml b/pyproject.toml index dfe9823c9..0bcd487e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ plus = [ "numba>=0.55", "pyarrow>=7.0", "scikit-learn>=1.1", + "sqlalchemy>=2.0", "zstd", ] dev = [