From 4874aa81acea3367536dedf3a8e3833323c783a3 Mon Sep 17 00:00:00 2001 From: Alex Stefanescu Date: Fri, 21 Jul 2023 17:06:47 +0200 Subject: [PATCH 1/6] Set server side cursors (compat. 1/4+) --- followthemoney/mapping/sql.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/followthemoney/mapping/sql.py b/followthemoney/mapping/sql.py index d94d8b9c2..a8d31bb99 100644 --- a/followthemoney/mapping/sql.py +++ b/followthemoney/mapping/sql.py @@ -55,10 +55,7 @@ def __init__(self, query: "QueryMapping", data: Dict[str, Any]) -> None: if database is None: raise InvalidMapping("No database in SQL mapping!") self.database_uri = cast(str, os.path.expandvars(database)) - kwargs = {} - if self.database_uri.lower().startswith("postgres"): - kwargs["server_side_cursors"] = True - self.engine = create_engine(self.database_uri, poolclass=NullPool, **kwargs) # type: ignore + self.engine = create_engine(self.database_uri, poolclass=NullPool) # type: ignore self.meta = MetaData() tables = keys_values(data, "table", "tables") @@ -104,7 +101,7 @@ def records(self) -> Generator[Record, None, None]: q = self.compose_query() log.info("Query: %s", q) with self.engine.connect() as conn: - rp = conn.execute(q) + rp = conn.execution_options(stream_results=True).execute(q) while True: rows = rp.fetchmany(size=DATA_PAGE) if not len(rows): From 9cf5829c1fa79d42c585ba38070e35902dc07ae6 Mon Sep 17 00:00:00 2001 From: Alex Stefanescu Date: Thu, 27 Jul 2023 16:15:58 +0300 Subject: [PATCH 2/6] Fixes mypy error --- followthemoney/mapping/sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/followthemoney/mapping/sql.py b/followthemoney/mapping/sql.py index a8d31bb99..37a40adc1 100644 --- a/followthemoney/mapping/sql.py +++ b/followthemoney/mapping/sql.py @@ -55,7 +55,7 @@ def __init__(self, query: "QueryMapping", data: Dict[str, Any]) -> None: if database is None: raise InvalidMapping("No database in SQL mapping!") self.database_uri = cast(str, os.path.expandvars(database)) - self.engine = create_engine(self.database_uri, poolclass=NullPool) # type: ignore + self.engine = create_engine(self.database_uri, poolclass=NullPool) self.meta = MetaData() tables = keys_values(data, "table", "tables") @@ -121,4 +121,4 @@ def __len__(self) -> int: q = self.apply_filters(q) with self.engine.connect() as conn: rp = conn.execute(q) - return int(rp.scalar()) + return int(rp.scalar() or 0) From b3753c941fb031302fac0c198af6060faf43fb55 Mon Sep 17 00:00:00 2001 From: Alex Stefanescu Date: Tue, 8 Aug 2023 16:27:49 +0300 Subject: [PATCH 3/6] Bump versions for normality and banal --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ec68698f5..10ef59616 100644 --- a/setup.py +++ b/setup.py @@ -34,12 +34,12 @@ "pyyaml >= 5.0.0, < 7.0.0", "types-PyYAML", "sqlalchemy2-stubs", - "banal >= 1.0.1, < 1.1.0", + "banal >= 1.0.6, < 1.1.0", "click >= 8.0, < 8.1.4", "stringcase >= 1.2.0, < 2.0.0", "requests >= 2.21.0, < 3.0.0", "python-levenshtein >= 0.12.0, < 1.0.0", - "normality >= 2.1.1, < 3.0.0", + "normality >= 2.4.0, < 3.0.0", "sqlalchemy >= 1.2.0, < 3.0.0", "countrynames >= 1.13.0, < 2.0.0", "languagecodes >= 1.1.0, < 2.0.0", From f5efc759a03445f16ae5035d2af3b987aafb757a Mon Sep 17 00:00:00 2001 From: Alex Stefanescu Date: Tue, 8 Aug 2023 16:33:16 +0300 Subject: [PATCH 4/6] Set minimum version for SQLAlchemy to be the same as for dataset --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 10ef59616..7f814cfad 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ "requests >= 2.21.0, < 3.0.0", "python-levenshtein >= 0.12.0, < 1.0.0", "normality >= 2.4.0, < 3.0.0", - "sqlalchemy >= 1.2.0, < 3.0.0", + "sqlalchemy >= 1.3.2, < 3.0.0", "countrynames >= 1.13.0, < 2.0.0", "languagecodes >= 1.1.0, < 2.0.0", "prefixdate >= 0.4.0, < 1.0.0", From 33c53e13fec91bf0623b6c079d38758aef6b4c4e Mon Sep 17 00:00:00 2001 From: Alex Stefanescu Date: Wed, 9 Aug 2023 17:06:52 +0300 Subject: [PATCH 5/6] Restore max version for click to a version beyond the latest one published on PyPI --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d76482185..c5344d4a0 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "types-PyYAML", "sqlalchemy2-stubs", "banal >= 1.0.6, < 1.1.0", - "click >= 8.0, < 8.1.4", + "click >= 8.0, < 8.1.7", "stringcase >= 1.2.0, < 2.0.0", "requests >= 2.21.0, < 3.0.0", "python-levenshtein >= 0.12.0, < 1.0.0", From 0c6402dc57efa8deab8fa87aaa34a3ee3cd67352 Mon Sep 17 00:00:00 2001 From: Alex Stefanescu Date: Thu, 10 Aug 2023 15:02:58 +0300 Subject: [PATCH 6/6] Increase minimum version of SQLAlchemy to 1.4.49 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c5344d4a0..9c1fc65e6 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ "requests >= 2.21.0, < 3.0.0", "python-levenshtein >= 0.12.0, < 1.0.0", "normality >= 2.4.0, < 3.0.0", - "sqlalchemy >= 1.3.2, < 3.0.0", + "sqlalchemy >= 1.4.49, < 3.0.0", "countrynames >= 1.13.0, < 2.0.0", "languagecodes >= 1.1.0, < 2.0.0", "prefixdate >= 0.4.0, < 1.0.0",