diff --git a/CHANGES.rst b/CHANGES.rst index a7ed49e105a5..bac78d3c1c5e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,69 @@ -Unreleased -========== +Changes in synapse v0.27.0-rc2 (2018-03-19) +=========================================== + +Pulls in v0.26.1 + +Bug fixes: + +* Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005) + + +Changes in synapse v0.26.1 (2018-03-15) +======================================= + +Bug fixes: + +* Fix bug where an invalid event caused server to stop functioning correctly, + due to parsing and serializing bugs in ujson library (PR #3008) + -synctl no longer starts the main synapse when using ``-a`` option with workers. -A new worker file should be added with ``worker_app: synapse.app.homeserver``. +Changes in synapse v0.27.0-rc1 (2018-03-14) +=========================================== + +The common case for running Synapse is not to run separate workers, but for those that do, be aware that synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``. This release also begins the process of renaming a number of the metrics reported to prometheus. See `docs/metrics-howto.rst `_. +Note that the v0.28.0 release will remove the deprecated metric names. + +Features: + +* Add ability for ASes to override message send time (PR #2754) +* Add support for custom storage providers for media repository (PR #2867, #2777, #2783, #2789, #2791, #2804, #2812, #2814, #2857, #2868, #2767) +* Add purge API features, see `docs/admin_api/purge_history_api.rst `_ for full details (PR #2858, #2867, #2882, #2946, #2962, #2943) +* Add support for whitelisting 3PIDs that users can register. (PR #2813) +* Add ``/room/{id}/event/{id}`` API (PR #2766) +* Add an admin API to get all the media in a room (PR #2818) Thanks to @turt2live! +* Add ``federation_domain_whitelist`` option (PR #2820, #2821) + + +Changes: + +* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst `_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784) +* Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) +* Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! +* No longer require a specific version of saml2 (PR #2695) Thanks to @okurz! +* Remove ``verbosity``/``log_file`` from generated config (PR #2755) +* Add and improve metrics and logging (PR #2770, #2778, #2785, #2786, #2787, #2793, #2794, #2795, #2809, #2810, #2833, #2834, #2844, #2965, #2927, #2975, #2790, #2796, #2838) +* When using synctl with workers, don't start the main synapse automatically (PR #2774) +* Minor performance improvements (PR #2773, #2792) +* Use a connection pool for non-federation outbound connections (PR #2817) +* Make it possible to run unit tests against postgres (PR #2829) +* Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp! +* Remove ability for AS users to call /events and /sync (PR #2948) +* Use bcrypt.checkpw (PR #2949) Thanks to @krombel! + +Bug fixes: + +* Fix broken ``ldap_config`` config option (PR #2683) Thanks to @seckrv! +* Fix error message when user is not allowed to unban (PR #2761) Thanks to @turt2live! +* Fix publicised groups GET API (singular) over federation (PR #2772) +* Fix user directory when using ``user_directory_search_all_users`` config option (PR #2803, #2831) +* Fix error on ``/publicRooms`` when no rooms exist (PR #2827) +* Fix bug in quarantine_media (PR #2837) +* Fix url_previews when no Content-Type is returned from URL (PR #2845) +* Fix rare race in sync API when joining room (PR #2944) +* Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848) Changes in synapse v0.26.0 (2018-01-05) diff --git a/README.rst b/README.rst index b18a35506198..d09964c9393b 100644 --- a/README.rst +++ b/README.rst @@ -354,6 +354,10 @@ https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one Fedora ------ +Synapse is in the Fedora repositories as ``matrix-synapse``:: + + sudo dnf install matrix-synapse + Oleg Girko provides Fedora RPMs at https://obs.infoserver.lv/project/monitor/matrix-synapse @@ -899,6 +903,17 @@ This should end with a 'PASSED' result:: PASSED (successes=143) +Running the Integration Tests +============================= + +Synapse is accompanied by `SyTest `_, +a Matrix homeserver integration testing suite, which uses HTTP requests to +access the API as a Matrix client would. It is able to run Synapse directly from +the source tree, so installation of the server is not required. + +Testing with SyTest is recommended for verifying that changes related to the +Client-Server API are functioning correctly. See the `installation instructions +`_ for details. Building Internal API Documentation =================================== diff --git a/docs/admin_api/purge_history_api.rst b/docs/admin_api/purge_history_api.rst index ea2922da5c7c..2da833c82762 100644 --- a/docs/admin_api/purge_history_api.rst +++ b/docs/admin_api/purge_history_api.rst @@ -16,9 +16,11 @@ including an ``access_token`` of a server admin. By default, events sent by local users are not deleted, as they may represent the only copies of this content in existence. (Events sent by remote users are -deleted, and room state data before the cutoff is always removed). +deleted.) -To delete local events as well, set ``delete_local_events`` in the body: +Room state data (such as joins, leaves, topic) is always preserved. + +To delete local message events as well, set ``delete_local_events`` in the body: .. code:: json diff --git a/docs/workers.rst b/docs/workers.rst index dee04bbf3e47..80f8d2181ae2 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -230,7 +230,7 @@ file. For example:: ``synapse.app.event_creator`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Handles non-state event creation. It can handle REST endpoints matching: +Handles non-state event creation. It can handle REST endpoints matching:: ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send diff --git a/synapse/__init__.py b/synapse/__init__.py index ef8853bd2423..71b409ba8b50 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.26.0" +__version__ = "0.27.0-rc2" diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 2ca9b17f48cb..dc1e7e4c4943 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -17,7 +17,7 @@ from synapse.types import UserID, RoomID from twisted.internet import defer -import ujson as json +import simplejson as json import jsonschema from jsonschema import FormatChecker diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 336959094b3e..c5384b3ad400 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -77,7 +77,9 @@ def default_config(self, **kwargs): # Set the number of bcrypt rounds used to generate password hash. # Larger numbers increase the work factor needed to generate the hash. - # The default number of rounds is 12. + # The default number is 12 (which equates to 2^12 rounds). + # N.B. that increasing this will exponentially increase the time required + # to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. bcrypt_rounds: 12 # Allows users to register as guests without a password/email/etc, and diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 31b1ece13e54..80b359b2e706 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ujson as json +import simplejson as json import logging from canonicaljson import encode_canonical_json diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4f97c8db79d2..5a8ddc253eeb 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -38,7 +38,7 @@ import logging import random -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -678,8 +678,8 @@ def handle_new_client_event( # Ensure that we can round trip before trying to persist in db try: - dump = ujson.dumps(unfreeze(event.content)) - ujson.loads(dump) + dump = simplejson.dumps(unfreeze(event.content)) + simplejson.loads(dump) except Exception: logger.exception("Failed to encode content: %r", event.content) raise diff --git a/synapse/http/server.py b/synapse/http/server.py index e64aa927299c..1551db239d14 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -461,7 +461,6 @@ def respond_with_json(request, code, json_object, send_cors=False, if canonical_json or synapse.events.USE_FROZEN_DICTS: json_bytes = encode_canonical_json(json_object) else: - # ujson doesn't like frozen_dicts. json_bytes = simplejson.dumps(json_object) return respond_with_json_bytes( diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index 171227cce2ef..9633404f736f 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -19,7 +19,7 @@ """ import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f8999d64d737..70d788deea66 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -30,7 +30,7 @@ import logging import urllib -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 31fe7aa75c89..0fc21540c625 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -23,7 +23,7 @@ import shutil import sys import traceback -import ujson as json +import simplejson as json import urlparse from twisted.web.server import NOT_DONE_YET diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index e70c9423e375..f83ff0454a17 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -23,7 +23,7 @@ from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks import abc -import ujson as json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index c88759bf2c72..8af325a9f5a7 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -19,7 +19,7 @@ from twisted.internet import defer -import ujson as json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py index 548e795dafe1..a879e5bfc1a5 100644 --- a/synapse/storage/deviceinbox.py +++ b/synapse/storage/deviceinbox.py @@ -14,7 +14,7 @@ # limitations under the License. import logging -import ujson +import simplejson from twisted.internet import defer @@ -85,7 +85,7 @@ def add_messages_txn(txn, now_ms, stream_id): ) rows = [] for destination, edu in remote_messages_by_destination.items(): - edu_json = ujson.dumps(edu) + edu_json = simplejson.dumps(edu) rows.append((destination, stream_id, now_ms, edu_json)) txn.executemany(sql, rows) @@ -177,7 +177,7 @@ def _add_messages_to_local_device_inbox_txn(self, txn, stream_id, " WHERE user_id = ?" ) txn.execute(sql, (user_id,)) - message_json = ujson.dumps(messages_by_device["*"]) + message_json = simplejson.dumps(messages_by_device["*"]) for row in txn: # Add the message for all devices for this user on this # server. @@ -199,7 +199,7 @@ def _add_messages_to_local_device_inbox_txn(self, txn, stream_id, # Only insert into the local inbox if the device exists on # this server device = row[0] - message_json = ujson.dumps(messages_by_device[device]) + message_json = simplejson.dumps(messages_by_device[device]) messages_json_for_user[device] = message_json if messages_json_for_user: @@ -253,7 +253,7 @@ def get_new_messages_for_device_txn(txn): messages = [] for row in txn: stream_pos = row[0] - messages.append(ujson.loads(row[1])) + messages.append(simplejson.loads(row[1])) if len(messages) < limit: stream_pos = current_stream_id return (messages, stream_pos) @@ -389,7 +389,7 @@ def get_new_messages_for_remote_destination_txn(txn): messages = [] for row in txn: stream_pos = row[0] - messages.append(ujson.loads(row[1])) + messages.append(simplejson.loads(row[1])) if len(messages) < limit: stream_pos = current_stream_id return (messages, stream_pos) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index bd2effdf34a6..712106b83a65 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import ujson as json +import simplejson as json from twisted.internet import defer diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 2cebb203c6f1..ff8538ddf81e 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -17,7 +17,7 @@ from synapse.util.caches.descriptors import cached from canonicaljson import encode_canonical_json -import ujson as json +import simplejson as json from ._base import SQLBaseStore diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 01f833982525..e78f8d0114c3 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -22,7 +22,7 @@ from .stream import lower_bound import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 9fc65229fd80..85ce6bea1abd 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -56,7 +56,6 @@ def encode_json(json_object): if USE_FROZEN_DICTS: - # ujson doesn't like frozen_dicts return encode_canonical_json(json_object) else: return json.dumps(json_object, ensure_ascii=False) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index eac8694e0f98..63997ed4491c 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -23,7 +23,7 @@ import abc import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 34ed84ea22b8..908551d6d93e 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -22,7 +22,7 @@ import collections import logging -import ujson as json +import simplejson as json import re logger = logging.getLogger(__name__) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 52e19e16b07f..d662d1cfc02f 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -28,7 +28,7 @@ from synapse.types import get_domain_from_id import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/storage/schema/delta/25/fts.py b/synapse/storage/schema/delta/25/fts.py index 4269ac69ad10..e7351c3ae6b8 100644 --- a/synapse/storage/schema/delta/25/fts.py +++ b/synapse/storage/schema/delta/25/fts.py @@ -17,7 +17,7 @@ from synapse.storage.prepare_database import get_statements from synapse.storage.engines import PostgresEngine, Sqlite3Engine -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -66,7 +66,7 @@ def run_create(cur, database_engine, *args, **kwargs): "max_stream_id_exclusive": max_stream_id + 1, "rows_inserted": 0, } - progress_json = ujson.dumps(progress) + progress_json = simplejson.dumps(progress) sql = ( "INSERT into background_updates (update_name, progress_json)" diff --git a/synapse/storage/schema/delta/27/ts.py b/synapse/storage/schema/delta/27/ts.py index 71b12a27312c..6df57b5206f0 100644 --- a/synapse/storage/schema/delta/27/ts.py +++ b/synapse/storage/schema/delta/27/ts.py @@ -16,7 +16,7 @@ from synapse.storage.prepare_database import get_statements -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -45,7 +45,7 @@ def run_create(cur, database_engine, *args, **kwargs): "max_stream_id_exclusive": max_stream_id + 1, "rows_inserted": 0, } - progress_json = ujson.dumps(progress) + progress_json = simplejson.dumps(progress) sql = ( "INSERT into background_updates (update_name, progress_json)" diff --git a/synapse/storage/schema/delta/31/search_update.py b/synapse/storage/schema/delta/31/search_update.py index 470ae0c00559..fe6b7d196d40 100644 --- a/synapse/storage/schema/delta/31/search_update.py +++ b/synapse/storage/schema/delta/31/search_update.py @@ -16,7 +16,7 @@ from synapse.storage.prepare_database import get_statements import logging -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def run_create(cur, database_engine, *args, **kwargs): "rows_inserted": 0, "have_added_indexes": False, } - progress_json = ujson.dumps(progress) + progress_json = simplejson.dumps(progress) sql = ( "INSERT into background_updates (update_name, progress_json)" diff --git a/synapse/storage/schema/delta/33/event_fields.py b/synapse/storage/schema/delta/33/event_fields.py index 83066cccc99d..1e002f9db249 100644 --- a/synapse/storage/schema/delta/33/event_fields.py +++ b/synapse/storage/schema/delta/33/event_fields.py @@ -15,7 +15,7 @@ from synapse.storage.prepare_database import get_statements import logging -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -44,7 +44,7 @@ def run_create(cur, database_engine, *args, **kwargs): "max_stream_id_exclusive": max_stream_id + 1, "rows_inserted": 0, } - progress_json = ujson.dumps(progress) + progress_json = simplejson.dumps(progress) sql = ( "INSERT into background_updates (update_name, progress_json)" diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 2755acff4022..984643b05780 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -16,7 +16,7 @@ from collections import namedtuple import logging import re -import ujson as json +import simplejson as json from twisted.internet import defer diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py index fc46bf7bb33f..13bff9f0551a 100644 --- a/synapse/storage/tags.py +++ b/synapse/storage/tags.py @@ -19,7 +19,7 @@ from synapse.util.caches.descriptors import cached from twisted.internet import defer -import ujson as json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 8f61f7ffae02..f825264ea9cc 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -23,7 +23,7 @@ from collections import namedtuple import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index a4bf8fa6ae6d..1c5a982094a0 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -154,11 +154,18 @@ def cache_get(key, default=None, callbacks=[]): def cache_set(key, value, callbacks=[]): node = cache.get(key, None) if node is not None: + # We sometimes store large objects, e.g. dicts, which cause + # the inequality check to take a long time. So let's only do + # the check if we have some callbacks to call. if node.callbacks and value != node.value: for cb in node.callbacks: cb() node.callbacks.clear() + # We don't bother to protect this by value != node.value as + # generally size_callback will be cheap compared with equality + # checks. (For example, taking the size of two dicts is quicker + # than comparing them for equality.) if size_callback: cached_cache_len[0] -= size_callback(node.value) cached_cache_len[0] += size_callback(value)