diff --git a/followthemoney/mapping/sql.py b/followthemoney/mapping/sql.py index d94d8b9c2..37a40adc1 100644 --- a/followthemoney/mapping/sql.py +++ b/followthemoney/mapping/sql.py @@ -55,10 +55,7 @@ def __init__(self, query: "QueryMapping", data: Dict[str, Any]) -> None: if database is None: raise InvalidMapping("No database in SQL mapping!") self.database_uri = cast(str, os.path.expandvars(database)) - kwargs = {} - if self.database_uri.lower().startswith("postgres"): - kwargs["server_side_cursors"] = True - self.engine = create_engine(self.database_uri, poolclass=NullPool, **kwargs) # type: ignore + self.engine = create_engine(self.database_uri, poolclass=NullPool) self.meta = MetaData() tables = keys_values(data, "table", "tables") @@ -104,7 +101,7 @@ def records(self) -> Generator[Record, None, None]: q = self.compose_query() log.info("Query: %s", q) with self.engine.connect() as conn: - rp = conn.execute(q) + rp = conn.execution_options(stream_results=True).execute(q) while True: rows = rp.fetchmany(size=DATA_PAGE) if not len(rows): @@ -124,4 +121,4 @@ def __len__(self) -> int: q = self.apply_filters(q) with self.engine.connect() as conn: rp = conn.execute(q) - return int(rp.scalar()) + return int(rp.scalar() or 0) diff --git a/setup.py b/setup.py index 9a140de34..9c1fc65e6 100644 --- a/setup.py +++ b/setup.py @@ -34,13 +34,13 @@ "pyyaml >= 5.0.0, < 7.0.0", "types-PyYAML", "sqlalchemy2-stubs", - "banal >= 1.0.1, < 1.1.0", + "banal >= 1.0.6, < 1.1.0", "click >= 8.0, < 8.1.7", "stringcase >= 1.2.0, < 2.0.0", "requests >= 2.21.0, < 3.0.0", "python-levenshtein >= 0.12.0, < 1.0.0", - "normality >= 2.1.1, < 3.0.0", - "sqlalchemy >= 1.2.0, < 3.0.0", + "normality >= 2.4.0, < 3.0.0", + "sqlalchemy >= 1.4.49, < 3.0.0", "countrynames >= 1.13.0, < 2.0.0", "languagecodes >= 1.1.0, < 2.0.0", "prefixdate >= 0.4.0, < 1.0.0",