diff --git a/changelog.d/6954.misc b/changelog.d/6954.misc new file mode 100644 index 000000000000..8b84ce2f1925 --- /dev/null +++ b/changelog.d/6954.misc @@ -0,0 +1 @@ +Minor perf fixes to `get_auth_chain_ids`. diff --git a/synapse/storage/data_stores/main/event_federation.py b/synapse/storage/data_stores/main/event_federation.py index 60c67457b4d1..d75ab0767580 100644 --- a/synapse/storage/data_stores/main/event_federation.py +++ b/synapse/storage/data_stores/main/event_federation.py @@ -15,7 +15,6 @@ import itertools import logging -from six.moves import range from six.moves.queue import Empty, PriorityQueue from twisted.internet import defer @@ -27,6 +26,7 @@ from synapse.storage.data_stores.main.signatures import SignatureWorkerStore from synapse.storage.database import Database from synapse.util.caches.descriptors import cached +from synapse.util.iterutils import batch_iter logger = logging.getLogger(__name__) @@ -71,14 +71,12 @@ def _get_auth_chain_ids_txn(self, txn, event_ids, include_given): front = set(event_ids) while front: new_front = set() - front_list = list(front) - chunks = [front_list[x : x + 100] for x in range(0, len(front), 100)] - for chunk in chunks: + for chunk in batch_iter(front, 100): clause, args = make_in_list_sql_clause( txn.database_engine, "event_id", chunk ) - txn.execute(base_sql + clause, list(args)) - new_front.update([r[0] for r in txn]) + txn.execute(base_sql + clause, args) + new_front.update(r[0] for r in txn) new_front -= results diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 3eeb2f7c04fc..6dcb5c04dacf 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -1504,7 +1504,7 @@ def simple_search_list_txn(cls, txn, table, term, col, retcols): def make_in_list_sql_clause( database_engine, column: str, iterable: Iterable -) -> Tuple[str, Iterable]: +) -> Tuple[str, list]: """Returns an SQL clause that checks the given column is in the iterable. On SQLite this expands to `column IN (?, ?, ...)`, whereas on Postgres