diff --git a/big_tests/tests/amp_big_SUITE.erl b/big_tests/tests/amp_big_SUITE.erl index 452fced4c40..59b35b18654 100644 --- a/big_tests/tests/amp_big_SUITE.erl +++ b/big_tests/tests/amp_big_SUITE.erl @@ -302,7 +302,7 @@ notify_deliver_to_online_user_bare_jid_test(Config) -> end). notify_deliver_to_online_user_recipient_privacy_test(Config) -> - case is_module_loaded(mod_mam) of + case is_module_loaded(mod_mam_pm) of true -> {skip, "MAM does not support privacy lists"}; false -> do_notify_deliver_to_online_user_recipient_privacy_test(Config) end. @@ -398,7 +398,7 @@ is_offline_storage_working(Config) -> Status == mam orelse Status == offline. notify_deliver_to_offline_user_recipient_privacy_test(Config) -> - case is_module_loaded(mod_mam) of + case is_module_loaded(mod_mam_pm) of true -> {skip, "MAM does not support privacy lists"}; false -> do_notify_deliver_to_offline_user_recipient_privacy_test(Config) end. @@ -735,14 +735,14 @@ user_has_no_incoming_offline_messages(FreshConfig, UserName) -> FreshConfig, [{UserName, 1}], fun(User) -> client_receives_nothing(User), - case is_module_loaded(mod_mam) of + case is_module_loaded(mod_mam_pm) of true -> client_has_no_mam_messages(User); false -> ok end end). user_has_incoming_offline_message(FreshConfig, UserName, MsgText) -> - true = is_module_loaded(mod_mam) orelse is_module_loaded(mod_offline), + true = is_module_loaded(mod_mam_pm) orelse is_module_loaded(mod_offline), {ok, Client} = escalus_client:start(FreshConfig, UserName, <<"new-session">>), escalus:send(Client, escalus_stanza:presence(<<"available">>)), case is_module_loaded(mod_offline) of @@ -751,7 +751,7 @@ user_has_incoming_offline_message(FreshConfig, UserName, MsgText) -> end, Presence = escalus:wait_for_stanza(Client), escalus:assert(is_presence, Presence), - case is_module_loaded(mod_mam) of + case is_module_loaded(mod_mam_pm) of true -> client_has_mam_message(Client); false -> ok end, @@ -988,10 +988,9 @@ required_modules(_) -> []. mam_modules(on) -> - [{mod_mam_meta, mam_helper:config_opts(#{pm => #{}, - async_writer => #{enabled => false}})}]; + [{mod_mam, mam_helper:config_opts(#{pm => #{}, async_writer => #{enabled => false}})}]; mam_modules(off) -> - [{mod_mam_meta, stopped}]. + [{mod_mam, stopped}]. offline_modules(on) -> [{mod_offline, config_parser_helper:mod_config(mod_offline, diff --git a/big_tests/tests/domain_removal_SUITE.erl b/big_tests/tests/domain_removal_SUITE.erl index 2406c27013c..dc1a6163f12 100644 --- a/big_tests/tests/domain_removal_SUITE.erl +++ b/big_tests/tests/domain_removal_SUITE.erl @@ -84,10 +84,10 @@ group_to_modules(auth_removal) -> []; group_to_modules(cache_removal) -> [{mod_cache_users, config_parser_helper:default_mod_config(mod_cache_users)}, - {mod_mam_meta, mam_helper:config_opts(#{pm => #{}})}]; + {mod_mam, mam_helper:config_opts(#{pm => #{}})}]; group_to_modules(mam_removal) -> MucHost = subhost_pattern(muc_light_helper:muc_host_pattern()), - [{mod_mam_meta, mam_helper:config_opts(#{pm => #{}, muc => #{host => MucHost}})}, + [{mod_mam, mam_helper:config_opts(#{pm => #{}, muc => #{host => MucHost}})}, {mod_muc_light, mod_config(mod_muc_light, #{backend => rdbms})}]; group_to_modules(muc_light_removal) -> [{mod_muc_light, mod_config(mod_muc_light, #{backend => rdbms})}]; diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 92b52bf05a9..7d494e5b4dd 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -354,33 +354,33 @@ groupchat_module(muclight) -> mam_required_modules(CN, Backend) when CN =:= remove_mam_pm; CN =:= retrieve_mam_pm -> - [{mod_mam_meta, mam_helper:config_opts(#{backend => Backend, pm => #{}})}]; + [{mod_mam, mam_helper:config_opts(#{backend => Backend, pm => #{}})}]; mam_required_modules(CN, Backend) when CN =:= retrieve_mam_pm_and_muc_light_dont_interfere; CN =:= retrieve_mam_muc_light -> HostPattern = subhost_pattern(muc_light_helper:muc_host_pattern()), - [{mod_mam_meta, mam_helper:config_opts(#{backend => Backend, - pm => #{}, - muc => #{host => HostPattern}})}, + [{mod_mam, mam_helper:config_opts(#{backend => Backend, + pm => #{}, + muc => #{host => HostPattern}})}, {mod_muc_light, default_mod_config(mod_muc_light)}]; mam_required_modules(retrieve_mam_pm_and_muc_light_interfere, Backend) -> HostPattern = subhost_pattern(muc_light_helper:muc_host_pattern()), - [{mod_mam_meta, mam_helper:config_opts(#{backend => Backend, - db_message_format => mam_message_xml, - pm => #{archive_groupchats => true}, - muc => #{host => HostPattern}})}, + [{mod_mam, mam_helper:config_opts(#{backend => Backend, + db_message_format => mam_message_xml, + pm => #{archive_groupchats => true}, + muc => #{host => HostPattern}})}, {mod_muc_light, default_mod_config(mod_muc_light)}]; mam_required_modules(CN, Backend) when CN =:= retrieve_mam_muc_private_msg; CN =:= retrieve_mam_muc -> HostPattern = subhost_pattern(muc_helper:muc_host_pattern()), - [{mod_mam_meta, mam_helper:config_opts(#{backend => Backend, - pm => #{}, - muc => #{host => HostPattern}})}, + [{mod_mam, mam_helper:config_opts(#{backend => Backend, + pm => #{}, + muc => #{host => HostPattern}})}, {mod_muc, muc_helper:make_opts(#{host => HostPattern})}]; mam_required_modules(retrieve_mam_muc_store_pm, Backend) -> HostPattern = subhost_pattern(muc_helper:muc_host_pattern()), - [{mod_mam_meta, mam_helper:config_opts(#{backend => Backend, - pm => #{archive_groupchats => true}, - muc => #{host => HostPattern}})}, + [{mod_mam, mam_helper:config_opts(#{backend => Backend, + pm => #{archive_groupchats => true}, + muc => #{host => HostPattern}})}, {mod_muc, muc_helper:make_opts(#{host => HostPattern})}]. pick_enabled_backend() -> @@ -866,7 +866,7 @@ retrieve_mam_pm_and_muc_light_dont_interfere(Config) -> [mam_helper:wait_for_archive_size(User, 2) || User <- [Alice, Bob]], false = mongoose_helper:successful_rpc(gen_mod, get_module_opt, - [host_type(), mod_mam, archive_groupchats, undefined]), + [host_type(), mod_mam_pm, archive_groupchats, undefined]), AliceDir = retrieve_all_personal_data(Alice, Config), BobDir = retrieve_all_personal_data(Bob, Config), @@ -914,7 +914,7 @@ retrieve_mam_pm_and_muc_light_interfere(Config) -> mam_helper:wait_for_archive_size(Kate, 3), true = mongoose_helper:successful_rpc(gen_mod, get_module_opt, - [host_type(), mod_mam, archive_groupchats, undefined]), + [host_type(), mod_mam_pm, archive_groupchats, undefined]), AliceDir = retrieve_all_personal_data(Alice, Config), BobDir = retrieve_all_personal_data(Bob, Config), diff --git a/big_tests/tests/mam_SUITE.erl b/big_tests/tests/mam_SUITE.erl index f4735f588e8..f2af5271d9b 100644 --- a/big_tests/tests/mam_SUITE.erl +++ b/big_tests/tests/mam_SUITE.erl @@ -647,19 +647,19 @@ required_modules_for_group(C, muc_light, Config) -> Opts = config_opts(Extra#{pm => #{}, muc => #{host => MUCHost}}), Config1 = maybe_set_wait(C, [muc, pm], [{mam_meta_opts, Opts} | Config]), {[{mod_muc_light, default_mod_config(mod_muc_light)}, - {mod_mam_meta, Opts}], Config1}; + {mod_mam, Opts}], Config1}; required_modules_for_group(C, BG, Config) when BG =:= muc_all; BG =:= muc_disabled_retraction -> Extra = maps:merge(mam_opts_for_conf(C), mam_opts_for_base_group(BG)), MUCHost = subhost_pattern(muc_domain(Config)), Opts = config_opts(Extra#{muc => #{host => MUCHost}}), Config1 = maybe_set_wait(C, [muc], [{mam_meta_opts, Opts} | Config]), - {[{mod_mam_meta, Opts}], Config1}; + {[{mod_mam, Opts}], Config1}; required_modules_for_group(C, BG, Config) -> Extra = maps:merge(mam_opts_for_conf(C), mam_opts_for_base_group(BG)), Opts = config_opts(Extra#{pm => #{}}), Config1 = maybe_set_wait(C, [pm], [{mam_meta_opts, Opts} | Config]), - {[{mod_mam_meta, Opts}], Config1}. + {[{mod_mam, Opts}], Config1}. maybe_set_wait(C, Types, Config) when C =:= rdbms_async_pool; C =:= rdbms_async_cache -> @@ -980,31 +980,31 @@ end_per_testcase(CaseName, Config) -> required_modules(muc_light_stored_in_pm_if_allowed_to, Config) -> Opts = #{pm := PM} = ?config(mam_meta_opts, Config), NewOpts = Opts#{pm := PM#{archive_groupchats => true}}, - [{mod_mam_meta, NewOpts}]; + [{mod_mam, NewOpts}]; required_modules(muc_light_chat_markers_are_archived_if_enabled, Config) -> Opts = #{muc := MUC} = ?config(mam_meta_opts, Config), NewOpts = Opts#{muc := MUC#{archive_chat_markers => true}}, - [{mod_mam_meta, NewOpts}]; + [{mod_mam, NewOpts}]; required_modules(muc_no_elements, Config) -> Opts = #{muc := MUC} = ?config(mam_meta_opts, Config), NewOpts = Opts#{muc := MUC#{no_stanzaid_element => true}}, - [{mod_mam_meta, NewOpts}]; + [{mod_mam, NewOpts}]; required_modules(muc_only_stanzaid, Config) -> Opts = ?config(mam_meta_opts, Config), - [{mod_mam_meta, Opts}]; + [{mod_mam, Opts}]; % configurable_archiveid basic group required_modules(no_elements, Config) -> Opts = #{pm := PM} = ?config(mam_meta_opts, Config), NewOpts = Opts#{pm := PM#{no_stanzaid_element => true}}, - [{mod_mam_meta, NewOpts}]; + [{mod_mam, NewOpts}]; required_modules(CaseName, Config) when CaseName =:= same_stanza_id; CaseName =:= retract_message_on_stanza_id -> Opts = #{pm := PM} = ?config(mam_meta_opts, Config), NewOpts = Opts#{pm := PM#{same_mam_id_for_peers => true}}, - [{mod_mam_meta, NewOpts}]; + [{mod_mam, NewOpts}]; required_modules(_, Config) -> Opts = ?config(mam_meta_opts, Config), - [{mod_mam_meta, Opts}]. + [{mod_mam, Opts}]. %%-------------------------------------------------------------------- %% Group name helpers @@ -2850,7 +2850,7 @@ metric_incremented_on_archive_request(ConfigIn) -> end, HostType = domain_helper:host_type(mim), HostTypePrefix = domain_helper:make_metrics_prefix(HostType), - MongooseMetrics = [{[HostTypePrefix, backends, mod_mam, lookup], changed}], + MongooseMetrics = [{[HostTypePrefix, backends, mod_mam_pm, lookup], changed}], Config = [{mongoose_metrics, MongooseMetrics} | ConfigIn], escalus_fresh:story(Config, [{alice, 1}], F). diff --git a/big_tests/tests/mam_helper.erl b/big_tests/tests/mam_helper.erl index 26cdd3f6d5a..4dd47cea2a9 100644 --- a/big_tests/tests/mam_helper.erl +++ b/big_tests/tests/mam_helper.erl @@ -125,15 +125,15 @@ config_opts(ExtraOpts) -> lists:foldl(fun set_opts/2, ExtraOpts, [defaults, backend, pm, muc, async_writer]). set_opts(defaults, Opts) -> - mod_config(mod_mam_meta, Opts); + mod_config(mod_mam, Opts); set_opts(backend, #{backend := riak} = Opts) -> - Opts#{riak => config([modules, mod_mam_meta, riak], maps:get(riak, Opts, #{}))}; + Opts#{riak => config([modules, mod_mam, riak], maps:get(riak, Opts, #{}))}; set_opts(pm, #{pm := PMExtra} = Opts) -> - Opts#{pm := config([modules, mod_mam_meta, pm], PMExtra)}; + Opts#{pm := config([modules, mod_mam, pm], PMExtra)}; set_opts(muc, #{muc := MUCExtra} = Opts) -> - Opts#{muc := config([modules, mod_mam_meta, muc], MUCExtra)}; + Opts#{muc := config([modules, mod_mam, muc], MUCExtra)}; set_opts(async_writer, #{async_writer := AsyncExtra} = Opts) -> - Opts#{async_writer := config([modules, mod_mam_meta, async_writer], AsyncExtra)}; + Opts#{async_writer := config([modules, mod_mam, async_writer], AsyncExtra)}; set_opts(_, Opts) -> Opts. @@ -801,16 +801,16 @@ wait_for_room_archive_size(Server, Username, ExpectedSize) -> archive_size(Server, Username) -> - rpc_apply(mod_mam, archive_size, [Server, Username]). + rpc_apply(mod_mam_pm, archive_size, [Server, Username]). archive_size_with_host_type(HostType, Server, Username) -> - rpc_apply(mod_mam, archive_size_with_host_type, [HostType, Server, Username]). + rpc_apply(mod_mam_pm, archive_size_with_host_type, [HostType, Server, Username]). room_archive_size(Server, Username) -> rpc_apply(mod_mam_muc, archive_size, [Server, Username]). delete_archive(Server, Username) -> - rpc_apply(mod_mam, delete_archive, [Server, Username]). + rpc_apply(mod_mam_pm, delete_archive, [Server, Username]). delete_room_archive(Server, Username) -> rpc_apply(mod_mam_muc, delete_archive, [Server, Username]). @@ -967,7 +967,7 @@ put_msg({{MsgIdOwner, MsgIdRemote}, archive_message(Map2). archive_message(#{} = Map) -> - ok = rpc_apply(mod_mam, archive_message_from_ct, [Map]). + ok = rpc_apply(mod_mam_pm, archive_message_from_ct, [Map]). muc_bootstrap_archive(Config) -> Room = ?config(room, Config), @@ -1017,7 +1017,7 @@ archive_muc_msg({{MsgID, _}, packet => Packet}]). get_archive_id(Server, User) -> - rpc_apply(mod_mam, archive_id, [Server, User]). + rpc_apply(mod_mam_pm, archive_id, [Server, User]). get_muc_archive_id(MucHost, Room) -> rpc_apply(mod_mam_muc, archive_id, [MucHost, Room]). diff --git a/big_tests/tests/mam_proper_SUITE.erl b/big_tests/tests/mam_proper_SUITE.erl index b751a2f6cc2..ee0d56d9cf0 100644 --- a/big_tests/tests/mam_proper_SUITE.erl +++ b/big_tests/tests/mam_proper_SUITE.erl @@ -34,7 +34,7 @@ init_per_group(G, _C) -> end. required_modules(_G) -> - [{mod_mam_meta, mam_helper:config_opts(#{pm => #{}})}]. + [{mod_mam, mam_helper:config_opts(#{pm => #{}})}]. end_per_group(_G, C) -> C. diff --git a/big_tests/tests/mam_send_message_SUITE.erl b/big_tests/tests/mam_send_message_SUITE.erl index 3aefec40a60..22cc57ec83f 100644 --- a/big_tests/tests/mam_send_message_SUITE.erl +++ b/big_tests/tests/mam_send_message_SUITE.erl @@ -73,9 +73,9 @@ end_per_group(_Groupname, Config) -> group_to_modules(send_message) -> MH = subhost_pattern(muc_light_helper:muc_host_pattern()), - [{mod_mam_meta, mam_helper:config_opts(#{pm => #{}, - muc => #{host => MH}, - send_message => mam_send_message_example})}, + [{mod_mam, mam_helper:config_opts(#{pm => #{}, + muc => #{host => MH}, + send_message => mam_send_message_example})}, {mod_muc_light, mod_config(mod_muc_light, #{backend => mongoose_helper:mnesia_or_rdbms_backend()})}, {mam_send_message_example, []}]. diff --git a/big_tests/tests/muc_SUITE.erl b/big_tests/tests/muc_SUITE.erl index 042cf0576d3..01bbf737701 100644 --- a/big_tests/tests/muc_SUITE.erl +++ b/big_tests/tests/muc_SUITE.erl @@ -529,7 +529,7 @@ setup_mam(disabled) -> ok; setup_mam(Backend) -> HostPattern = subhost_pattern(muc_helper:muc_host_pattern()), dynamic_modules:ensure_modules( - host_type(), [{mod_mam_meta, + host_type(), [{mod_mam, mam_helper:config_opts(#{backend => Backend, muc => #{host => HostPattern}})}]). diff --git a/big_tests/tests/muc_light_SUITE.erl b/big_tests/tests/muc_light_SUITE.erl index fb07b9aa58e..2ef2699184c 100644 --- a/big_tests/tests/muc_light_SUITE.erl +++ b/big_tests/tests/muc_light_SUITE.erl @@ -259,12 +259,12 @@ end_per_testcase(CaseName, Config) -> %% Module configuration per test case required_modules(CaseName, MAMBackend) -> - [{mod_mam_meta, mam_helper:config_opts(#{backend => MAMBackend, - muc => #{host => subhost_pattern(?MUCHOST)}})} | + [{mod_mam, mam_helper:config_opts(#{backend => MAMBackend, + muc => #{host => subhost_pattern(?MUCHOST)}})} | common_required_modules(CaseName)]. required_modules(CaseName) -> - [{mod_mam_meta, stopped} | common_required_modules(CaseName)]. + [{mod_mam, stopped} | common_required_modules(CaseName)]. common_required_modules(CaseName) -> BasicOpts = maps:merge(common_muc_light_opts(), muc_light_opts(CaseName)), diff --git a/big_tests/tests/muc_light_legacy_SUITE.erl b/big_tests/tests/muc_light_legacy_SUITE.erl index a2e05065855..5e864b54b5e 100644 --- a/big_tests/tests/muc_light_legacy_SUITE.erl +++ b/big_tests/tests/muc_light_legacy_SUITE.erl @@ -191,12 +191,12 @@ end_per_testcase(CaseName, Config) -> %% Module configuration per test case required_modules(CaseName, MAMBackend) -> - [{mod_mam_meta, mam_helper:config_opts(#{backend => MAMBackend, - muc => #{host => subhost_pattern(?MUCHOST)}})} | + [{mod_mam, mam_helper:config_opts(#{backend => MAMBackend, + muc => #{host => subhost_pattern(?MUCHOST)}})} | common_required_modules(CaseName)]. required_modules(CaseName) -> - [{mod_mam_meta, stopped} | common_required_modules(CaseName)]. + [{mod_mam, stopped} | common_required_modules(CaseName)]. common_required_modules(CaseName) -> Opts = maps:merge(common_muc_light_opts(), muc_light_opts(CaseName)), diff --git a/big_tests/tests/rest_helper.erl b/big_tests/tests/rest_helper.erl index 37fbf3b7229..b1a1961bffa 100644 --- a/big_tests/tests/rest_helper.erl +++ b/big_tests/tests/rest_helper.erl @@ -329,11 +329,11 @@ maybe_enable_mam(_, _, C) -> required_mam_modules(Backend) -> MucPattern = subhost_pattern(muc_light_helper:muc_host_pattern()), - [{mod_mam_meta, mam_helper:config_opts(#{backend => Backend, - pm => #{}, - muc => #{host => MucPattern}, - async_writer => #{enabled => false}, - archive_chat_markers => true})}]. + [{mod_mam, mam_helper:config_opts(#{backend => Backend, + pm => #{}, + muc => #{host => MucPattern}, + async_writer => #{enabled => false}, + archive_chat_markers => true})}]. maybe_skip_mam_test_cases(CaseName, CasesRequireingMAM, Config) -> case lists:member(CaseName, CasesRequireingMAM) of @@ -377,7 +377,7 @@ make_arc_id(Client) -> Server = escalus_client:server(Client), Bin = escalus_client:short_jid(Client), Jid = mongoose_helper:make_jid(User, Server, <<>>), - {Bin, Jid, mam_helper:rpc_apply(mod_mam, archive_id, [Server, User])}. + {Bin, Jid, mam_helper:rpc_apply(mod_mam_pm, archive_id, [Server, User])}. fill_room_archive(RoomID, Users, AlreadyArchivedCount) -> {TodayDate, _} = calendar:local_time(), diff --git a/big_tests/tests/service_domain_db_SUITE.erl b/big_tests/tests/service_domain_db_SUITE.erl index 0a80e739750..2bc9b9ebc29 100644 --- a/big_tests/tests/service_domain_db_SUITE.erl +++ b/big_tests/tests/service_domain_db_SUITE.erl @@ -259,7 +259,7 @@ end_per_testcase(TestcaseName, Config) -> init_per_testcase2(TestcaseName, Config) when TestcaseName =:= rest_delete_domain_cleans_data_from_mam -> HostType = dummy_auth_host_type(), - Mods = [{mod_mam_meta, mam_helper:config_opts(#{pm => #{}})}], + Mods = [{mod_mam, mam_helper:config_opts(#{pm => #{}})}], dynamic_modules:ensure_modules(HostType, Mods), escalus:init_per_testcase(TestcaseName, Config); init_per_testcase2(_, Config) -> diff --git a/big_tests/tests/service_mongoose_system_metrics_SUITE.erl b/big_tests/tests/service_mongoose_system_metrics_SUITE.erl index 57da0fca4ce..108b9476b3b 100644 --- a/big_tests/tests/service_mongoose_system_metrics_SUITE.erl +++ b/big_tests/tests/service_mongoose_system_metrics_SUITE.erl @@ -237,7 +237,7 @@ rdbms_module_opts_are_reported(_Config) -> mongoose_helper:wait_until(fun are_modules_reported/0, true), check_module_backend(mod_auth_token, rdbms), check_module_backend(mod_inbox, rdbms), - check_module_backend(mod_mam_meta, rdbms). + check_module_backend(mod_mam, rdbms). check_module_backend(Module, Backend) -> check_module_opt(Module, backend, atom_to_binary(Backend)). @@ -356,7 +356,7 @@ modules_to_test(module_opts_are_reported) -> modules_to_test(rdbms_module_opts_are_reported) -> [required_module(mod_auth_token), required_module(mod_inbox), - required_module(mod_mam_meta)]. + required_module(mod_mam)]. required_module(Module) -> required_module(Module, #{}). diff --git a/doc/migrations/5.1.0_6.0.0.md b/doc/migrations/5.1.0_6.0.0.md new file mode 100644 index 00000000000..5fbbba012a3 --- /dev/null +++ b/doc/migrations/5.1.0_6.0.0.md @@ -0,0 +1,7 @@ +## Configuration + +The `mod_mam_meta` module is now named `mod_mam` for simplicity, so if you are using this module, you need to update the module name in `mongooseim.toml`. + +## Metrics + +The `mod_mam` backend module is now named `mod_mam_pm` for consistency with `mod_mam_muc`. As a result, the backend metrics have updated names, i.e. each `[backends, mod_mam, Metric]` name is changed to `[backends, mod_mam_pm, Metric]`, where `Metric` can be `lookup` or `archive`. diff --git a/doc/modules/mod_mam.md b/doc/modules/mod_mam.md index 260d4220062..c5b10ecb390 100644 --- a/doc/modules/mod_mam.md +++ b/doc/modules/mod_mam.md @@ -14,7 +14,7 @@ Configure MAM with different storage backends: * Cassandra (NoSQL) * ElasticSearch (NoSQL) -`mod_mam_meta` is a meta-module that ensures all relevant `mod_mam_*` modules are loaded and properly configured. +`mod_mam` is a meta-module that ensures all relevant `mod_mam_*` modules are loaded and properly configured. ### Message retraction This module supports [XEP-0424: Message Retraction](http://xmpp.org/extensions/xep-0424.html) with RDBMS storage backends. When a [retraction message](https://xmpp.org/extensions/xep-0424.html#example-4) is received, the MAM module finds the message to retract and replaces it with a tombstone. @@ -64,28 +64,28 @@ Also note that the default separator for the search query is `AND` (which roughl ## Options -### `modules.mod_mam_meta.backend` +### `modules.mod_mam.backend` * **Syntax:** string, one of `"rdbms"`, `"riak"`, `"cassandra"` and `"elasticsearch"` * **Default:** `"rdbms"` * **Example:** `backend = "riak"` Database backend to use. -### `modules.mod_mam_meta.no_stanzaid_element` +### `modules.mod_mam.no_stanzaid_element` * **Syntax:** boolean * **Default:** `false` * **Example:** `no_stanzaid_element = true` Do not add a `` element from MAM v0.6. -### `modules.mod_mam_meta.is_archivable_message` +### `modules.mod_mam.is_archivable_message` * **Syntax:** non-empty string * **Default:** `"mod_mam_utils"` * **Example:** `is_archivable_message = "mod_mam_utils"` Name of a module implementing [`is_archivable_message/3` callback](#is_archivable_message) that determines if the message should be archived. -### `modules.mod_mam_meta.send_message` +### `modules.mod_mam.send_message` * **Syntax:** non-empty string * **Default:** `"mod_mam_utils"` * **Example:** `send_message = "mod_mam_utils"` @@ -96,7 +96,7 @@ Consult with `mod_mam_utils:send_message/4` code for more information. Check `big_tests/tests/mam_send_message_SUITE_data/mam_send_message_example.erl` file in the MongooseIM repository for the usage example. -### `modules.mod_mam_meta.archive_chat_markers` +### `modules.mod_mam.archive_chat_markers` * **Syntax:** boolean * **Default:** `false` * **Example:** `archive_chat_markers = true` @@ -104,7 +104,7 @@ in the MongooseIM repository for the usage example. If set to true, XEP-0333 chat markers will be archived. See more details [here](#archiving-chat-markers). -### `modules.mod_mam_meta.message_retraction` +### `modules.mod_mam.message_retraction` * **Syntax:** boolean * **Default:** `true` * **Example:** `message_retraction = false` @@ -119,17 +119,17 @@ This functionality is currently implemented only for the `rdbms` backend. Archive for one-to-one messages can be enabled in one of two ways: -* Specify `[mod_mam_meta.pm]` section +* Specify `[mod_mam.pm]` section ```toml -[modules.mod_mam_meta] -[modules.mod_mam_meta.pm] # defining this section enables PM support +[modules.mod_mam] +[modules.mod_mam.pm] # defining this section enables PM support ``` * Define any PM related option ```toml -[modules.mod_mam_meta] +[modules.mod_mam] pm.backend = "rdbms" # enables PM support and overrides its backend ``` @@ -139,17 +139,17 @@ To disable archive for one-to-one messages please remove PM section or any PM re ### PM-specific options -#### `modules.mod_mam_meta.pm.archive_groupchats` +#### `modules.mod_mam.pm.archive_groupchats` * **Syntax:** boolean * **Default:** `false` -* **Example:** `modules.mod_mam_meta.pm.archive_groupchats = true` +* **Example:** `modules.mod_mam.pm.archive_groupchats = true` When enabled, MAM will store groupchat messages in recipients' individual archives. **USE WITH CAUTION!** May increase archive size significantly. Disabling this option for existing installation will neither remove such messages from MAM storage, nor will filter out them from search results. -#### `modules.mod_mam_meta.pm.same_mam_id_for_peers` +#### `modules.mod_mam.pm.same_mam_id_for_peers` * **Syntax:** boolean * **Default:** `false` -* **Example:** `modules.mod_mam_meta.pm.same_mam_id_for_peers = true` +* **Example:** `modules.mod_mam.pm.same_mam_id_for_peers = true` When enabled, MAM will set the same MAM ID for both sender and recipient. This can be useful in combination with [retraction on the stanza-id](#retraction-on-the-stanza-id). Note that this might not work with clients across federation, as the recipient might not implement the same retraction, nor the same IDs. @@ -157,17 +157,17 @@ When enabled, MAM will set the same MAM ID for both sender and recipient. This c Archive for MUC messages can be enabled in one of two ways: -* Specify `[mod_mam_meta.muc]` section +* Specify `[mod_mam.muc]` section ```toml -[modules.mod_mam_meta] -[modules.mod_mam_meta.muc] # defining this section enables MUC support +[modules.mod_mam] +[modules.mod_mam.muc] # defining this section enables MUC support ``` * Define any MUC related option ```toml -[modules.mod_mam_meta] +[modules.mod_mam] muc.backend = "rdbms" # enables MUC support and overrides its backend ``` @@ -177,10 +177,10 @@ To disable archive for MUC messages please remove MUC section or any MUC related ### MUC-specific options -#### `modules.mod_mam_meta.muc.host` +#### `modules.mod_mam.muc.host` * **Syntax:** string * **Default:** `"conference.@HOST@"` -* **Example:** `modules.mod_mam_meta.muc.host = "conference.@HOST@"` +* **Example:** `modules.mod_mam.muc.host = "conference.@HOST@"` The MUC host that will be archived if MUC archiving is enabled. @@ -193,7 +193,7 @@ The example below presents how to override common option for `muc` module specif Please note that you can override all common options (except `cache`) in a similar way. ```toml -[modules.mod_mam_meta] +[modules.mod_mam] backend = "rdbms" async_writer.enabled = true # this option enables async writer for RDBMS backend muc.async_writer.enabled = false # disable async writer for MUC archive only @@ -203,64 +203,64 @@ Please note that you can override all common options (except `cache`) in a simil These options will only have effect when the `rdbms` backend is used: -#### `modules.mod_mam_meta.cache_users` +#### `modules.mod_mam.cache_users` * **Syntax:** boolean * **Default:** `true` -* **Example:** `modules.mod_mam_meta.cache_users = false` +* **Example:** `modules.mod_mam.cache_users = false` Enables Archive ID to integer mappings cache. If caching is enabled, by default it will spawn its own [segmented cache](https://github.com/esl/segmented_cache) cache, with defaults as in [`mod_cache_users`](./mod_cache_users.md). To change these defaults, the same config can be accessed within the `cache` key. To see details about the meaning of each flag, see [`mod_cache_users`](./mod_cache_users.md). To reuse the cache already created by `mod_cache_users`, see the option below. ```toml -modules.mod_mam_meta.cache.strategy -modules.mod_mam_meta.cache.time_to_live -modules.mod_mam_meta.cache.number_of_segments +modules.mod_mam.cache.strategy +modules.mod_mam.cache.time_to_live +modules.mod_mam.cache.number_of_segments ``` -#### `modules.mod_mam_meta.cache.module` +#### `modules.mod_mam.cache.module` * **Syntax:** string, one of `"mod_cache_users"` or `"internal"` * **Default:** `internal` -* **Example:** `modules.mod_mam_meta.cache.module = "mod_cache_users"` +* **Example:** `modules.mod_mam.cache.module = "mod_cache_users"` Configures which cache to use, either start an internal instance, or reuse the cache created by `mod_cache_users`, if such module was enabled. Note that if reuse is desired – that is, `cache.module = "mod_cache_users"`, other cache configuration parameters will be ignored. -#### `modules.mod_mam_meta.async_writer.enabled` +#### `modules.mod_mam.async_writer.enabled` * **Syntax:** boolean * **Default:** `true` -* **Example:** `modules.mod_mam_meta.async_writer.enabled = false` +* **Example:** `modules.mod_mam.async_writer.enabled = false` Enables an asynchronous writer that is faster than the synchronous one but harder to debug. The async writers store batches of messages that will be flush on a timeout (see **flush_interval**) or when the batch reaches a size (see **batch_size**), so the results of the lookup operations executed right after message routing may be incomplete until the configured time passes or the queue is full. -#### `modules.mod_mam_meta.async_writer.flush_interval` +#### `modules.mod_mam.async_writer.flush_interval` * **Syntax:** non-negative integer * **Default:** `2000` -* **Example:** `modules.mod_mam_meta.async_writer.flush_interval = 2000` +* **Example:** `modules.mod_mam.async_writer.flush_interval = 2000` How often (in milliseconds) the buffered messages are flushed to DB. -#### `modules.mod_mam_meta.async_writer.batch_size` +#### `modules.mod_mam.async_writer.batch_size` * **Syntax:** non-negative integer * **Default:** `30` -* **Example:** `modules.mod_mam_meta.async_writer.batch_size = 30` +* **Example:** `modules.mod_mam.async_writer.batch_size = 30` Max size of the batch for an async writer before the queue is considered full and flushed. If the buffer is full, messages are flushed to a database immediately and the flush timer is reset. -#### `modules.mod_mam_meta.async_writer.pool_size` +#### `modules.mod_mam.async_writer.pool_size` * **Syntax:** non-negative integer * **Default:** `4 * erlang:system_info(schedulers_online)` -* **Example:** `modules.mod_mam_meta.async_writer.pool_size = 32` +* **Example:** `modules.mod_mam.async_writer.pool_size = 32` Number of workers in the pool. More than the number of available schedulers is recommended, to minimise lock contention on the message queues, and more than the number of DB workers, to fully utilise the DB capacity. How much more than these two parameters is then a good fine-tuning for specific deployments. ### Common backend options -#### `modules.mod_mam_meta.user_prefs_store` +#### `modules.mod_mam.user_prefs_store` * **Syntax:** one of `"rdbms"`, `"cassandra"`, `"mnesia"` * **Default:** not set -* **Example:** `modules.mod_mam_meta.user_prefs_store = "rdbms"` +* **Example:** `modules.mod_mam.user_prefs_store = "rdbms"` Leaving this option unset will prevent users from setting their archiving preferences. It will also increase performance. @@ -270,10 +270,10 @@ The possible values are: * `"cassandra"` (Cassandra backend only) - User archiving preferences are saved in Cassandra. * `"mnesia"` (recommended) - User archiving preferences saved in Mnesia and accessed without transactions. Recommended in most deployments, could be overloaded with lots of users updating their preferences at once. There's a small risk of an inconsistent (in a rather harmless way) state of the preferences table. -#### `modules.mod_mam_meta.full_text_search` +#### `modules.mod_mam.full_text_search` * **Syntax:** boolean * **Default:** `true` -* **Example:** `modules.mod_mam_meta.full_text_search = false` +* **Example:** `modules.mod_mam.full_text_search = false` Enables full text search in message archive (see *Full Text Search* paragraph). Please note that the full text search is currently only implemented for `"rdbms"` and `"riak"` backends. @@ -309,17 +309,17 @@ This backend works with Riak KV 2.0 and above, but we recommend version 2.1.1. #### Riak-specific options -#### `modules.mod_mam_meta.riak.bucket_type` +#### `modules.mod_mam.riak.bucket_type` * **Syntax:** non-empty string * **Default:** `"mam_yz"` -* **Example:** `modules.mod_mam_meta.riak.bucket_type = "mam_yz"` +* **Example:** `modules.mod_mam.riak.bucket_type = "mam_yz"` Riak bucket type. -#### `modules.mod_mam_meta.riak.search_index` +#### `modules.mod_mam.riak.search_index` * **Syntax:** non-empty string * **Default:** `"mam"` -* **Example:** `modules.mod_mam_meta.riak.search_index = "mam"` +* **Example:** `modules.mod_mam.riak.search_index = "mam"` Riak index name. @@ -338,63 +338,63 @@ Please consult [Outgoing connections](../configuration/outgoing-connections.md#e These options allow for fine-grained control over MAM behaviour. -#### `modules.mod_mam_meta.default_result_limit` +#### `modules.mod_mam.default_result_limit` * **Syntax:** non-negative integer * **Default:** `50` -* **Example:** `modules.mod_mam_meta.default_result_limit = 100` +* **Example:** `modules.mod_mam.default_result_limit = 100` This sets the default page size of returned results. -#### `modules.mod_mam_meta.max_result_limit` +#### `modules.mod_mam.max_result_limit` * **Syntax:** non-negative integer * **Default:** `50` -* **Example:** `modules.mod_mam_meta.max_result_limit = 100` +* **Example:** `modules.mod_mam.max_result_limit = 100` This sets the maximum page size of returned results. -#### `modules.mod_mam_meta.db_jid_format` +#### `modules.mod_mam.db_jid_format` * **Syntax:** string, one of `"mam_jid_rfc"`, `"mam_jid_rfc_trust"`, `"mam_jid_mini"` or a module implementing `mam_jid` behaviour * **Default:** `"mam_jid_rfc"` for MUC archive, `"mam_jid_mini"` for PM archive -* **Example:** `modules.mod_mam_meta.db_jid_format = "mam_jid_mini"` +* **Example:** `modules.mod_mam.db_jid_format = "mam_jid_mini"` Sets the internal MAM jid encoder/decoder module for RDBMS. !!! Warning Archive MUST be empty to change this option -#### `modules.mod_mam_meta.db_message_format` +#### `modules.mod_mam.db_message_format` * **Syntax:** string, one of `"mam_message_xml"`, `"mam_message_eterm"`, `"mam_message_compressed_eterm"` or a module implementing `mam_message` behaviour * **Default:** `"mam_message_compressed_eterm"` for RDBMS, `"mam_message_xml"` for Riak and Cassandra -* **Example:** `modules.mod_mam_meta.db_message_format = "mam_message_compressed_eterm"` +* **Example:** `modules.mod_mam.db_message_format = "mam_message_compressed_eterm"` Sets the internal MAM message encoder/decoder module. !!! Warning Archive MUST be empty to change this option -#### `modules.mod_mam_meta.extra_fin_element` +#### `modules.mod_mam.extra_fin_element` * **Syntax:** string, a module implementing the `extra_fin_element/3` callback * **Default:** none -* **Example:** `modules.mod_mam_meta.extra_fin_element = "example_mod"` +* **Example:** `modules.mod_mam.extra_fin_element = "example_mod"` This module can be used to add subelements to the `` element of the MAM lookup query response. It can be useful to be able to add information to a mam query, that doesn't belong to any specific message but to all of them. -#### `modules.mod_mam_meta.extra_lookup_params` +#### `modules.mod_mam.extra_lookup_params` * **Syntax:** string, a module implementing the `extra_lookup_params/2` callback * **Default:** none -* **Example:** `modules.mod_mam_meta.extra_lookup_params = "example_mod"` +* **Example:** `modules.mod_mam.extra_lookup_params = "example_mod"` This module can be used to add extra lookup parameters to MAM lookup queries. ## Example configuration ```toml -[modules.mod_mam_meta] +[modules.mod_mam] backend = "rdbms" no_stanzaid_element = true diff --git a/doc/operation-and-maintenance/Logging-&-monitoring.md b/doc/operation-and-maintenance/Logging-&-monitoring.md index 12c30fa8d2d..2105d327200 100644 --- a/doc/operation-and-maintenance/Logging-&-monitoring.md +++ b/doc/operation-and-maintenance/Logging-&-monitoring.md @@ -125,8 +125,8 @@ Authorization time: ..backends.auth.authorize...backends.mongoose_rdbms.query. RDBMS prepared query time: ..backends.mongoose_rdbms.execute. MAM lookups: ..mam_lookup_messages.one -MAM archivization time: ..backends.mod_mam.archive. -MAM lookup time: ..backends.mod_mam.lookup. +MAM archivization time: ..backends.mod_mam_pm.archive. +MAM lookup time: ..backends.mod_mam_pm.lookup. MAM private messages flush time: ..mod_mam_rdbms_async_pool_writer.flush_time. MAM MUC messages flush time: ..mod_mam_muc_rdbms_async_pool_writer.flush_time. ``` diff --git a/doc/operation-and-maintenance/known-issues.md b/doc/operation-and-maintenance/known-issues.md index a4f1add8128..f7577c09686 100644 --- a/doc/operation-and-maintenance/known-issues.md +++ b/doc/operation-and-maintenance/known-issues.md @@ -46,7 +46,7 @@ The problems should now be resolved, and MSSQL is verified to work on Ubuntu 20. ## GDPR retrieval for MAM MUC limitation When the personal data retrieval is executed for a user in a specific domain, Message Archive Management for groupchats must be running for this particular domain. -This is the case for most configurations, but the problem manifests when a MongooseIM operator configures `mod_mam_muc`/`mod_mam_meta` to start only for a subset of domains supported by the cluster (`host_config` option). +This is the case for most configurations, but the problem manifests when a MongooseIM operator configures `mod_mam_muc`/`mod_mam` to start only for a subset of domains supported by the cluster (`host_config` option). In such case, personal data stored by MAM MUC will not be retrieved for this user. diff --git a/mkdocs.yml b/mkdocs.yml index e6831d9b55c..18dfbf90fdd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -186,6 +186,7 @@ nav: - '4.1.0 to 4.2.0': 'migrations/4.1.0_4.2.0.md' - '4.2.0 to 5.0.0': 'migrations/4.2.0_5.0.0.md' - '5.0.0 to 5.1.0': 'migrations/5.0.0_5.1.0.md' + - '5.1.0 to 6.0.0': 'migrations/5.1.0_6.0.0.md' - 'MAM MUC migration helper': 'migrations/jid-from-mam-muc-script.md' - 'Contributions to the Ecosystem': 'Contributions.md' - 'MongooseIM History': 'History.md' diff --git a/src/config/mongoose_config_spec.erl b/src/config/mongoose_config_spec.erl index 26ee7c4dc8d..803a7b411fd 100644 --- a/src/config/mongoose_config_spec.erl +++ b/src/config/mongoose_config_spec.erl @@ -738,7 +738,7 @@ configurable_modules() -> mod_jingle_sip, mod_keystore, mod_last, - mod_mam_meta, + mod_mam, mod_muc, mod_muc_light, mod_muc_log, diff --git a/src/mam/mod_mam.erl b/src/mam/mod_mam.erl index a553cdb2dfa..41cdddfe91f 100644 --- a/src/mam/mod_mam.erl +++ b/src/mam/mod_mam.erl @@ -1,116 +1,33 @@ -%%%------------------------------------------------------------------- -%%% @author Uvarov Michael -%%% @copyright (C) 2013, Uvarov Michael -%%% @doc XEP-0313: Message Archive Management -%%% -%%% The module uses several backend modules: -%%% -%%%
    -%%%
  • Preference manager ({@link mod_mam_muc_rdbms_prefs});
    • -%%%
  • Writer ({@link mod_mam_muc_rdbms_arch} or {@link mod_mam_muc_rdbms_async_pool_writer});
    • -%%%
  • Archive manager ({@link mod_mam_muc_rdbms_arch});
    • -%%%
  • User's ID generator ({@link mod_mam_muc_user}).
    • -%%%
-%%% -%%% Preferences can be also stored in Mnesia ({@link mod_mam_mnesia_prefs}). -%%% This module handles simple archives. -%%% -%%% This module should be started for each host. -%%% Message archivation is not shaped here (use standard support for this). -%%% MAM's IQs are shaped inside {@link shaper_srv}. -%%% -%%% Message identifiers (or UIDs in the spec) are generated based on: -%%% -%%%
    -%%%
  • date (using `timestamp()');
    • -%%%
  • node number (using {@link ejabberd_node_id}).
    • -%%%
-%%% @end -%%%------------------------------------------------------------------- +%%============================================================================== +%% Copyright 2016 Erlang Solutions Ltd. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%============================================================================== + -module(mod_mam). --behavior(gen_mod). +-behaviour(gen_mod). -behaviour(mongoose_module_metrics). --xep([{xep, 313}, {version, "0.4.1"}]). --xep([{xep, 313}, {version, "0.5"}]). --xep([{xep, 313}, {version, "0.6"}]). -%% ---------------------------------------------------------------------- -%% Exports - -%% Client API --export([delete_archive/2, - archive_size/2, - archive_size_with_host_type/3, - archive_id/2]). - -%% gen_mod handlers --export([start/2, stop/1, supported_features/0]). - -%% ejabberd handlers --export([disco_local_features/1, - process_mam_iq/5, - user_send_packet/4, - remove_user/3, - filter_packet/1, - determine_amp_strategy/5, - sm_filter_offline_message/4]). - -%% gdpr callbacks --export([get_personal_data/3]). - -%%private --export([archive_message_from_ct/1]). --export([lookup_messages/2]). --export([archive_id_int/2]). - --ignore_xref([archive_message_from_ct/1, - archive_size/2, archive_size_with_host_type/3, delete_archive/2, - determine_amp_strategy/5, disco_local_features/1, filter_packet/1, - get_personal_data/3, remove_user/3, sm_filter_offline_message/4, - user_send_packet/4]). -%% ---------------------------------------------------------------------- -%% Imports - -%% UID --import(mod_mam_utils, - [get_or_generate_mam_id/1, - decode_compact_uuid/1]). - -%% XML --import(mod_mam_utils, - [maybe_add_arcid_elems/4, - wrap_message/6, - result_set/4, - result_prefs/4, - make_fin_element/7, - parse_prefs/1, - is_mam_result_message/1, - features/2]). - -%% Forms --import(mod_mam_utils, - [message_form/3]). - -%% Other --import(mod_mam_utils, - [mess_id_to_external_binary/1, - is_complete_result_page/4]). - -%% ejabberd --import(mod_mam_utils, - [is_jid_in_user_roster/3]). - - --include("mongoose.hrl"). --include("jlib.hrl"). --include("amp.hrl"). -include("mod_mam.hrl"). +-include("mongoose_config_spec.hrl"). + +-type module_opts() :: gen_mod:module_opts(). +-type module_map() :: gen_mod_deps:module_map(). %% ---------------------------------------------------------------------- %% Datetime types %% Microseconds from 01.01.1970 -type unix_timestamp() :: non_neg_integer(). --type host_type() :: mongooseim:host_type(). %% ---------------------------------------------------------------------- %% Other types @@ -137,8 +54,8 @@ AlwaysJIDs :: [jid:literal_jid()], NeverJIDs :: [jid:literal_jid()]}. --type archive_message_params() :: #{message_id := mod_mam:message_id(), - archive_id := mod_mam:archive_id(), +-type archive_message_params() :: #{message_id := message_id(), + archive_id := archive_id(), local_jid := jid:jid(), remote_jid := jid:jid(), source_jid := jid:jid(), @@ -146,7 +63,7 @@ direction := atom(), packet := exml:element(), %% Only in mod_mam_muc_rdbms_arch:retract_message/2 - sender_id => mod_mam:archive_id()}. + sender_id => archive_id()}. -export_type([rewriter_fun/0, borders/0, @@ -162,625 +79,274 @@ archive_message_params/0 ]). -%% ---------------------------------------------------------------------- -%% API +-export([start/2, stop/1, config_spec/0, supported_features/0, deps/2, config_metrics/1]). --spec get_personal_data(gdpr:personal_data(), mongooseim:host_type(), jid:jid()) -> - gdpr:personal_data(). -get_personal_data(Acc, HostType, ArcJID) -> - Schema = ["id", "from", "message"], - Entries = mongoose_hooks:get_mam_pm_gdpr_data(HostType, ArcJID), - [{mam_pm, Schema, Entries} | Acc]. - --spec delete_archive(jid:server(), jid:user()) -> 'ok'. -delete_archive(Server, User) - when is_binary(Server), is_binary(User) -> - ?LOG_DEBUG(#{what => mam_delete_archive, user => User, server => Server}), - ArcJID = jid:make_bare(User, Server), - HostType = jid_to_host_type(ArcJID), - ArcID = archive_id_int(HostType, ArcJID), - remove_archive_hook(HostType, ArcID, ArcJID), - ok. +-export([remove_unused_backend_opts/1]). --spec archive_size(jid:server(), jid:user()) -> integer(). -archive_size(Server, User) - when is_binary(Server), is_binary(User) -> - ArcJID = jid:make_bare(User, Server), - HostType = jid_to_host_type(ArcJID), - ArcID = archive_id_int(HostType, ArcJID), - archive_size(HostType, ArcID, ArcJID). - --spec archive_size_with_host_type(host_type(), jid:server(), jid:user()) -> integer(). -archive_size_with_host_type(HostType, Server, User) -> - ArcJID = jid:make_bare(User, Server), - ArcID = archive_id_int(HostType, ArcJID), - archive_size(HostType, ArcID, ArcJID). - --spec archive_id(jid:server(), jid:user()) -> integer() | undefined. -archive_id(Server, User) - when is_binary(Server), is_binary(User) -> - ArcJID = jid:make_bare(User, Server), - HostType = jid_to_host_type(ArcJID), - archive_id_int(HostType, ArcJID). - -%% gen_mod callbacks -%% Starting and stopping functions for users' archives - --spec start(host_type(), gen_mod:module_opts()) -> any(). -start(HostType, Opts) -> - ?LOG_INFO(#{what => mam_starting, host_type => HostType}), - ensure_metrics(HostType), - ejabberd_hooks:add(hooks(HostType)), - add_iq_handlers(HostType, Opts), - ok. - --spec stop(host_type()) -> any(). -stop(HostType) -> - ?LOG_INFO(#{what => mam_stopping, host_type => HostType}), - ejabberd_hooks:delete(hooks(HostType)), - remove_iq_handlers(HostType), - ok. +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- -spec supported_features() -> [atom()]. supported_features() -> [dynamic_domains]. -%% ---------------------------------------------------------------------- -%% hooks and handlers - -%% `To' is an account or server entity hosting the archive. -%% Servers that archive messages on behalf of local users SHOULD expose archives -%% to the user on their bare JID (i.e. `From.luser'), -%% while a MUC service might allow MAM queries to be sent to the room's bare JID -%% (i.e `To.luser'). --spec process_mam_iq(Acc :: mongoose_acc:t(), - From :: jid:jid(), To :: jid:jid(), IQ :: jlib:iq(), - _Extra) -> {mongoose_acc:t(), jlib:iq() | ignore}. -process_mam_iq(Acc, From, To, IQ, _Extra) -> - HostType = mongoose_acc:host_type(Acc), - mod_mam_utils:maybe_log_deprecation(IQ), - Action = mam_iq:action(IQ), - case is_action_allowed(HostType, Action, From, To) of - true -> - case mod_mam_utils:wait_shaper(HostType, To#jid.lserver, Action, From) of - ok -> - handle_error_iq(HostType, Acc, To, Action, - handle_mam_iq(Action, From, To, IQ, Acc)); - {error, max_delay_reached} -> - ?LOG_WARNING(#{what => mam_max_delay_reached, - text => <<"Return max_delay_reached error IQ from MAM">>, - action => Action, acc => Acc}), - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), - {Acc, return_max_delay_reached_error_iq(IQ)} - end; - false -> - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), - {Acc, return_action_not_allowed_error_iq(IQ)} - end. - --spec disco_local_features(mongoose_disco:feature_acc()) -> mongoose_disco:feature_acc(). -disco_local_features(Acc = #{host_type := HostType, node := <<>>}) -> - mongoose_disco:add_features(features(?MODULE, HostType), Acc); -disco_local_features(Acc) -> - Acc. - -%% @doc Handle an outgoing message. -%% -%% Note: for outgoing messages, the server MUST use the value of the 'to' -%% attribute as the target JID. --spec user_send_packet(Acc :: mongoose_acc:t(), From :: jid:jid(), - To :: jid:jid(), - Packet :: exml:element()) -> mongoose_acc:t(). -user_send_packet(Acc, From, To, Packet) -> - ?LOG_DEBUG(#{what => mam_user_send_packet, acc => Acc}), - {_, Acc2} = handle_package(outgoing, true, From, To, From, Packet, Acc), - Acc2. - -%% @doc Handle an incoming message. -%% -%% Note: For incoming messages, the server MUST use the value of the -%% 'from' attribute as the target JID. -%% -%% Return drop to drop the packet, or the original input to let it through. -%% From and To are jid records. --type fpacket() :: {From :: jid:jid(), - To :: jid:jid(), - Acc :: mongoose_acc:t(), - Packet :: exml:element()}. --spec filter_packet(Value :: fpacket() | drop) -> fpacket() | drop. -filter_packet(drop) -> - drop; -filter_packet({From, To, Acc1, Packet}) -> - ?LOG_DEBUG(#{what => mam_user_receive_packet, acc => Acc1}), - HostType = mongoose_acc:host_type(Acc1), - Type = mongoose_lib:get_message_type(Acc1), - {AmpEvent, PacketAfterArchive, Acc3} = - case mongoose_lib:does_local_user_exist(HostType, To, Type) of - false -> - {mam_failed, Packet, Acc1}; - true -> - case process_incoming_packet(From, To, Packet, Acc1) of - {undefined, Acc2} -> - {mam_failed, Packet, Acc2}; - {MessID, Acc2} -> - Packet2 = maybe_add_arcid_elems( - To, MessID, Packet, - mod_mam_params:add_stanzaid_element(?MODULE, HostType)), - {archived, Packet2, Acc2} - end - end, - Acc4 = mongoose_acc:update_stanza(#{ element => PacketAfterArchive, - from_jid => From, - to_jid => To }, Acc3), - Acc5 = mod_amp:check_packet(Acc4, AmpEvent), - {From, To, Acc5, mongoose_acc:element(Acc5)}. - -process_incoming_packet(From, To, Packet, Acc) -> - handle_package(incoming, true, To, From, From, Packet, Acc). - -%% hook handler --spec remove_user(mongoose_acc:t(), jid:user(), jid:server()) -> mongoose_acc:t(). -remove_user(Acc, User, Server) -> - delete_archive(Server, User), - Acc. - -sm_filter_offline_message(_Drop=false, _From, _To, Packet) -> - %% If ... - is_mam_result_message(Packet); - %% ... than drop the message -sm_filter_offline_message(Other, _From, _To, _Packet) -> - Other. - -%% ---------------------------------------------------------------------- -%% Internal functions - --spec jid_to_host_type(jid:jid()) -> host_type(). -jid_to_host_type(#jid{lserver=LServer}) -> - lserver_to_host_type(LServer). - -lserver_to_host_type(LServer) -> - case mongoose_domain_api:get_domain_host_type(LServer) of - {ok, HostType} -> - HostType; - {error, not_found} -> - error({get_domain_host_type_failed, LServer}) - end. - --spec acc_to_host_type(mongoose_acc:t()) -> host_type(). -acc_to_host_type(Acc) -> - case mongoose_acc:host_type(Acc) of - undefined -> - lserver_to_host_type(mongoose_acc:lserver(Acc)); - HostType -> - HostType - end. - --spec is_action_allowed(HostType :: host_type(), - Action :: mam_iq:action(), From :: jid:jid(), - To :: jid:jid()) -> boolean(). -is_action_allowed(HostType, Action, From, To) -> - case acl:match_rule(HostType, To#jid.lserver, Action, From, default) of - allow -> true; - deny -> false; - default -> is_action_allowed_by_default(Action, From, To) - end. - --spec is_action_allowed_by_default(Action :: mam_iq:action(), From :: jid:jid(), - To :: jid:jid()) -> boolean(). -is_action_allowed_by_default(_Action, From, To) -> - compare_bare_jids(From, To). - --spec compare_bare_jids(jid:simple_jid() | jid:jid(), - jid:simple_jid() | jid:jid()) -> boolean(). -compare_bare_jids(JID1, JID2) -> - jid:to_bare(JID1) =:= jid:to_bare(JID2). - --spec handle_mam_iq(mam_iq:action(), From :: jid:jid(), To :: jid:jid(), - IQ :: jlib:iq(), Acc :: mongoose_acc:t()) -> - jlib:iq() | {error, term(), jlib:iq()}. -handle_mam_iq(Action, From, To, IQ, Acc) -> - case Action of - mam_get_prefs -> - handle_get_prefs(To, IQ, Acc); - mam_set_prefs -> - handle_set_prefs(To, IQ, Acc); - mam_set_message_form -> - handle_set_message_form(From, To, IQ, Acc); - mam_get_message_form -> - handle_get_message_form(From, To, IQ, Acc) - end. - --spec handle_set_prefs(jid:jid(), jlib:iq(), mongoose_acc:t()) -> - jlib:iq() | {error, term(), jlib:iq()}. -handle_set_prefs(ArcJID=#jid{}, IQ=#iq{sub_el = PrefsEl}, Acc) -> - {DefaultMode, AlwaysJIDs, NeverJIDs} = parse_prefs(PrefsEl), - ?LOG_DEBUG(#{what => mam_set_prefs, default_mode => DefaultMode, - always_jids => AlwaysJIDs, never_jids => NeverJIDs, iq => IQ}), - HostType = acc_to_host_type(Acc), - ArcID = archive_id_int(HostType, ArcJID), - Res = set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs), - handle_set_prefs_result(Res, DefaultMode, AlwaysJIDs, NeverJIDs, IQ). - -handle_set_prefs_result(ok, DefaultMode, AlwaysJIDs, NeverJIDs, IQ) -> - Namespace = IQ#iq.xmlns, - ResultPrefsEl = result_prefs(DefaultMode, AlwaysJIDs, NeverJIDs, Namespace), - IQ#iq{type = result, sub_el = [ResultPrefsEl]}; -handle_set_prefs_result({error, Reason}, - _DefaultMode, _AlwaysJIDs, _NeverJIDs, IQ) -> - return_error_iq(IQ, Reason). - --spec handle_get_prefs(jid:jid(), IQ :: jlib:iq(), Acc :: mongoose_acc:t()) -> - jlib:iq() | {error, term(), jlib:iq()}. -handle_get_prefs(ArcJID=#jid{}, IQ=#iq{}, Acc) -> - HostType = acc_to_host_type(Acc), - ArcID = archive_id_int(HostType, ArcJID), - Res = get_prefs(HostType, ArcID, ArcJID, always), - handle_get_prefs_result(Res, IQ). - -handle_get_prefs_result({DefaultMode, AlwaysJIDs, NeverJIDs}, IQ) -> - ?LOG_DEBUG(#{what => mam_get_prefs_result, default_mode => DefaultMode, - always_jids => AlwaysJIDs, never_jids => NeverJIDs, iq => IQ}), - Namespace = IQ#iq.xmlns, - ResultPrefsEl = result_prefs(DefaultMode, AlwaysJIDs, NeverJIDs, Namespace), - IQ#iq{type = result, sub_el = [ResultPrefsEl]}; -handle_get_prefs_result({error, Reason}, IQ) -> - return_error_iq(IQ, Reason). - --spec handle_set_message_form(From :: jid:jid(), ArcJID :: jid:jid(), - IQ :: jlib:iq(), Acc :: mongoose_acc:t()) -> - jlib:iq() | ignore | {error, term(), jlib:iq()}. -handle_set_message_form(#jid{} = From, #jid{} = ArcJID, #iq{} = IQ, Acc) -> - HostType = acc_to_host_type(Acc), - ArcID = archive_id_int(HostType, ArcJID), - try iq_to_lookup_params(HostType, IQ) of - Params0 -> - do_handle_set_message_form(Params0, From, ArcID, ArcJID, IQ, HostType) - catch _C:R:S -> - report_issue({R, S}, mam_lookup_failed, ArcJID, IQ), - return_error_iq(IQ, R) - end. - - --spec do_handle_set_message_form(Params :: mam_iq:lookup_params(), - From :: jid:jid(), - ArcId :: archive_id(), - ArcJID :: jid:jid(), - IQ :: jlib:iq(), - HostType :: mongooseim:host_type()) -> - jlib:iq() | ignore | {error, term(), jlib:iq()}. -do_handle_set_message_form(Params0, From, ArcID, ArcJID, - #iq{xmlns=MamNs, sub_el = QueryEl} = IQ, - HostType) -> - QueryID = exml_query:attr(QueryEl, <<"queryid">>, <<>>), - Params = mam_iq:lookup_params_with_archive_details(Params0, ArcID, ArcJID, From), - case lookup_messages(HostType, Params) of - {error, Reason} -> - report_issue(Reason, mam_lookup_failed, ArcJID, IQ), - return_error_iq(IQ, Reason); - {ok, {TotalCount, Offset, MessageRows}} -> - %% Forward messages - {FirstMessID, LastMessID} = forward_messages(HostType, From, ArcJID, MamNs, - QueryID, MessageRows, true), - %% Make fin iq - IsComplete = is_complete_result_page(TotalCount, Offset, MessageRows, Params), - IsStable = true, - ResultSetEl = result_set(FirstMessID, LastMessID, Offset, TotalCount), - ExtFinMod = mod_mam_params:extra_fin_element_module(?MODULE, HostType), - FinElem = make_fin_element(HostType, Params, IQ#iq.xmlns, IsComplete, IsStable, ResultSetEl, ExtFinMod), - IQ#iq{type = result, sub_el = [FinElem]} - end. - -iq_to_lookup_params(HostType, IQ) -> - Max = mod_mam_params:max_result_limit(?MODULE, HostType), - Def = mod_mam_params:default_result_limit(?MODULE, HostType), - Ext = mod_mam_params:extra_params_module(?MODULE, HostType), - mam_iq:form_to_lookup_params(IQ, Max, Def, Ext). - -forward_messages(HostType, From, ArcJID, MamNs, QueryID, MessageRows, SetClientNs) -> - %% Forward messages - {FirstMessID, LastMessID} = - case MessageRows of - [] -> {undefined, undefined}; - [_|_] -> {message_row_to_ext_id(hd(MessageRows)), - message_row_to_ext_id(lists:last(MessageRows))} - end, - SendModule = mod_mam_params:send_message_mod(?MODULE, HostType), - [send_message(SendModule, Row, ArcJID, From, - message_row_to_xml(MamNs, Row, QueryID, SetClientNs)) - || Row <- MessageRows], - {FirstMessID, LastMessID}. - -send_message(SendModule, Row, ArcJID, From, Packet) -> - mam_send_message:call_send_message(SendModule, Row, ArcJID, From, Packet). - --spec handle_get_message_form(jid:jid(), jid:jid(), jlib:iq(), mongoose_acc:t()) -> - jlib:iq(). -handle_get_message_form(_From=#jid{}, _ArcJID=#jid{}, IQ=#iq{}, Acc) -> - HostType = acc_to_host_type(Acc), - return_message_form_iq(HostType, IQ). - -determine_amp_strategy(Strategy = #amp_strategy{deliver = Deliver}, - FromJID, ToJID, Packet, initial_check) -> - HostType = jid_to_host_type(ToJID), - ShouldBeStored = is_archivable_message(HostType, incoming, Packet) - andalso is_interesting(ToJID, FromJID) - andalso ejabberd_auth:does_user_exist(ToJID), - case ShouldBeStored of - true -> Strategy#amp_strategy{deliver = amp_deliver_strategy(Deliver)}; - false -> Strategy - end; -determine_amp_strategy(Strategy, _, _, _, _) -> - Strategy. - -amp_deliver_strategy([none]) -> [stored, none]; -amp_deliver_strategy([direct, none]) -> [direct, stored, none]. - --spec handle_package(Dir :: incoming | outgoing, ReturnMessID :: boolean(), - LocJID :: jid:jid(), RemJID :: jid:jid(), SrcJID :: jid:jid(), - Packet :: exml:element(), Acc :: mongoose_acc:t()) -> - {MaybeMessID :: binary() | undefined, Acc :: mongoose_acc:t()}. -handle_package(Dir, ReturnMessID, - LocJID = #jid{}, RemJID = #jid{}, SrcJID = #jid{}, Packet, Acc) -> - HostType = acc_to_host_type(Acc), - case is_archivable_message(HostType, Dir, Packet) - andalso should_archive_if_groupchat(HostType, exml_query:attr(Packet, <<"type">>)) of - true -> - ArcID = archive_id_int(HostType, LocJID), - OriginID = mod_mam_utils:get_origin_id(Packet), - case is_interesting(HostType, LocJID, RemJID, ArcID) of - true -> - MessID = get_or_generate_mam_id(Acc), - Params = #{message_id => MessID, - archive_id => ArcID, - local_jid => LocJID, - remote_jid => RemJID, - source_jid => SrcJID, - origin_id => OriginID, - direction => Dir, - packet => Packet}, - Result = archive_message(HostType, Params), - ExtMessId = return_external_message_id_if_ok(ReturnMessID, Result, MessID), - {ExtMessId, return_acc_with_mam_id_if_configured(ExtMessId, HostType, Acc)}; - false -> - {undefined, Acc} - end; - false -> - {undefined, Acc} - end. - -should_archive_if_groupchat(HostType, <<"groupchat">>) -> - gen_mod:get_module_opt(HostType, ?MODULE, archive_groupchats); -should_archive_if_groupchat(_, _) -> - true. - --spec return_external_message_id_if_ok(ReturnMessID :: boolean(), - ArchivingResult :: ok | any(), - MessID :: integer()) -> binary() | undefined. -return_external_message_id_if_ok(true, ok, MessID) -> mess_id_to_external_binary(MessID); -return_external_message_id_if_ok(_, _, _MessID) -> undefined. - -return_acc_with_mam_id_if_configured(undefined, _, Acc) -> - Acc; -return_acc_with_mam_id_if_configured(ExtMessId, HostType, Acc) -> - case gen_mod:get_module_opt(HostType, ?MODULE, same_mam_id_for_peers) of - false -> mongoose_acc:set(mam, mam_id, ExtMessId, Acc); - true -> mongoose_acc:set_permanent(mam, mam_id, ExtMessId, Acc) - end. - -is_interesting(LocJID, RemJID) -> - HostType = jid_to_host_type(LocJID), - ArcID = archive_id_int(HostType, LocJID), - is_interesting(HostType, LocJID, RemJID, ArcID). - -is_interesting(HostType, LocJID, RemJID, ArcID) -> - case get_behaviour(HostType, ArcID, LocJID, RemJID) of - always -> true; - never -> false; - roster -> is_jid_in_user_roster(HostType, LocJID, RemJID) - end. - -%% ---------------------------------------------------------------------- -%% Backend wrappers - --spec archive_id_int(host_type(), jid:jid()) -> - non_neg_integer() | undefined. -archive_id_int(HostType, ArcJID=#jid{}) -> - mongoose_hooks:mam_archive_id(HostType, ArcJID). - --spec archive_size(host_type(), archive_id(), jid:jid()) -> integer(). -archive_size(HostType, ArcID, ArcJID=#jid{}) -> - mongoose_hooks:mam_archive_size(HostType, ArcID, ArcJID). - --spec get_behaviour(host_type(), archive_id(), LocJID :: jid:jid(), - RemJID :: jid:jid()) -> atom(). -get_behaviour(HostType, ArcID, LocJID=#jid{}, RemJID=#jid{}) -> - mongoose_hooks:mam_get_behaviour(HostType, ArcID, LocJID, RemJID). - --spec set_prefs(host_type(), archive_id(), ArcJID :: jid:jid(), - DefaultMode :: atom(), AlwaysJIDs :: [jid:literal_jid()], - NeverJIDs :: [jid:literal_jid()]) -> any(). -set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> - mongoose_hooks:mam_set_prefs(HostType, ArcID, ArcJID, DefaultMode, - AlwaysJIDs, NeverJIDs). - -%% @doc Load settings from the database. --spec get_prefs(HostType :: host_type(), ArcID :: archive_id(), - ArcJID :: jid:jid(), GlobalDefaultMode :: archive_behaviour() - ) -> preference() | {error, Reason :: term()}. -get_prefs(HostType, ArcID, ArcJID, GlobalDefaultMode) -> - mongoose_hooks:mam_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID). - --spec remove_archive_hook(host_type(), archive_id(), jid:jid()) -> 'ok'. -remove_archive_hook(HostType, ArcID, ArcJID=#jid{}) -> - mongoose_hooks:mam_remove_archive(HostType, ArcID, ArcJID), +-spec start(mongooseim:host_type(), module_opts()) -> any(). +start(_HostType, _Opts) -> ok. --spec lookup_messages(HostType :: host_type(), Params :: map()) -> - {ok, mod_mam:lookup_result()} - | {error, 'policy-violation'} - | {error, Reason :: term()}. -lookup_messages(HostType, Params) -> - Result = lookup_messages_without_policy_violation_check(HostType, Params), - %% If a query returns a number of stanzas greater than this limit and the - %% client did not specify a limit using RSM then the server should return - %% a policy-violation error to the client. - mod_mam_utils:check_result_for_policy_violation(Params, Result). - -lookup_messages_without_policy_violation_check( - HostType, #{search_text := SearchText} = Params) -> - case SearchText /= undefined andalso - not mod_mam_params:has_full_text_search(?MODULE, HostType) of - true -> %% Use of disabled full text search - {error, 'not-supported'}; - false -> - StartT = erlang:monotonic_time(microsecond), - R = mongoose_hooks:mam_lookup_messages(HostType, Params), - Diff = erlang:monotonic_time(microsecond) - StartT, - mongoose_metrics:update(HostType, [backends, ?MODULE, lookup], Diff), - R - end. - -archive_message_from_ct(Params = #{local_jid := JID}) -> - HostType = jid_to_host_type(JID), - archive_message(HostType, Params). - --spec archive_message(host_type(), mod_mam:archive_message_params()) -> - ok | {error, timeout}. -archive_message(HostType, Params) -> - StartT = erlang:monotonic_time(microsecond), - R = mongoose_hooks:mam_archive_message(HostType, Params), - Diff = erlang:monotonic_time(microsecond) - StartT, - mongoose_metrics:update(HostType, [backends, ?MODULE, archive], Diff), - R. - -%% ---------------------------------------------------------------------- -%% Helpers - --spec message_row_to_xml(binary(), message_row(), QueryId :: binary(), boolean()) -> - exml:element(). -message_row_to_xml(MamNs, #{id := MessID, jid := SrcJID, packet := Packet}, - QueryID, SetClientNs) -> - {Microseconds, _NodeMessID} = decode_compact_uuid(MessID), - TS = calendar:system_time_to_rfc3339(Microseconds, [{offset, "Z"}, {unit, microsecond}]), - BExtMessID = mess_id_to_external_binary(MessID), - Packet1 = mod_mam_utils:maybe_set_client_xmlns(SetClientNs, Packet), - wrap_message(MamNs, Packet1, QueryID, BExtMessID, TS, SrcJID). - --spec message_row_to_ext_id(message_row()) -> binary(). -message_row_to_ext_id(#{id := MessID}) -> - mess_id_to_external_binary(MessID). - -handle_error_iq(HostType, Acc, _To, _Action, {error, _Reason, IQ}) -> - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), - {Acc, IQ}; -handle_error_iq(_Host, Acc, _To, _Action, IQ) -> - {Acc, IQ}. - --spec return_action_not_allowed_error_iq(jlib:iq()) -> jlib:iq(). -return_action_not_allowed_error_iq(IQ) -> - ErrorEl = jlib:stanza_errort(<<"">>, <<"cancel">>, <<"not-allowed">>, - <<"en">>, <<"The action is not allowed.">>), - IQ#iq{type = error, sub_el = [ErrorEl]}. - --spec return_max_delay_reached_error_iq(jlib:iq()) -> jlib:iq(). -return_max_delay_reached_error_iq(IQ) -> - %% Message not found. - ErrorEl = mongoose_xmpp_errors:resource_constraint( - <<"en">>, <<"The action is cancelled because of flooding.">>), - IQ#iq{type = error, sub_el = [ErrorEl]}. - --spec return_error_iq(jlib:iq(), Reason :: term()) -> {error, term(), jlib:iq()}. -return_error_iq(IQ, {Reason, {stacktrace, _Stacktrace}}) -> - return_error_iq(IQ, Reason); -return_error_iq(IQ, timeout) -> - E = mongoose_xmpp_errors:service_unavailable(<<"en">>, <<"Timeout">>), - {error, timeout, IQ#iq{type = error, sub_el = [E]}}; -return_error_iq(IQ, invalid_stanza_id) -> - Text = mongoose_xmpp_errors:not_acceptable(<<"en">>, <<"Invalid stanza id provided">>), - {error, invalid_stanza_id, IQ#iq{type = error, sub_el = [Text]}}; -return_error_iq(IQ, item_not_found) -> - {error, item_not_found, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:item_not_found()]}}; -return_error_iq(IQ, not_implemented) -> - {error, not_implemented, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:feature_not_implemented()]}}; -return_error_iq(IQ, Reason) -> - {error, Reason, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:internal_server_error()]}}. - -return_message_form_iq(HostType, IQ) -> - IQ#iq{type = result, sub_el = [message_form(?MODULE, HostType, IQ#iq.xmlns)]}. - -report_issue({Reason, {stacktrace, Stacktrace}}, Issue, ArcJID, IQ) -> - report_issue(Reason, Stacktrace, Issue, ArcJID, IQ); -report_issue(Reason, Issue, ArcJID, IQ) -> - report_issue(Reason, [], Issue, ArcJID, IQ). - -report_issue(invalid_stanza_id, _Stacktrace, _Issue, _ArcJID, _IQ) -> - expected; -report_issue(item_not_found, _Stacktrace, _Issue, _ArcJID, _IQ) -> - expected; -report_issue(not_implemented, _Stacktrace, _Issue, _ArcJID, _IQ) -> - expected; -report_issue(timeout, _Stacktrace, _Issue, _ArcJID, _IQ) -> - expected; -report_issue(Reason, Stacktrace, Issue, #jid{lserver=LServer, luser=LUser}, IQ) -> - ?LOG_ERROR(#{what => mam_error, - issue => Issue, server => LServer, user => LUser, - reason => Reason, iq => IQ, stacktrace => Stacktrace}). - --spec is_archivable_message(HostType :: host_type(), - Dir :: incoming | outgoing, - Packet :: exml:element()) -> boolean(). -is_archivable_message(HostType, Dir, Packet) -> - M = mod_mam_params:is_archivable_message_module(?MODULE, HostType), - ArchiveChatMarkers = mod_mam_params:archive_chat_markers(?MODULE, HostType), - erlang:apply(M, is_archivable_message, [?MODULE, Dir, Packet, ArchiveChatMarkers]). - --spec hooks(jid:lserver()) -> [ejabberd_hooks:hook()]. -hooks(HostType) -> - [{disco_local_features, HostType, ?MODULE, disco_local_features, 99}, - {user_send_packet, HostType, ?MODULE, user_send_packet, 60}, - {rest_user_send_packet, HostType, ?MODULE, user_send_packet, 60}, - {filter_local_packet, HostType, ?MODULE, filter_packet, 60}, - {remove_user, HostType, ?MODULE, remove_user, 50}, - {anonymous_purge_hook, HostType, ?MODULE, remove_user, 50}, - {amp_determine_strategy, HostType, ?MODULE, determine_amp_strategy, 20}, - {sm_filter_offline_message, HostType, ?MODULE, sm_filter_offline_message, 50}, - {get_personal_data, HostType, ?MODULE, get_personal_data, 50} - | mongoose_metrics_mam_hooks:get_mam_hooks(HostType)]. - -add_iq_handlers(HostType, Opts) -> - Component = ejabberd_sm, - %% `parallel' is the only one recommended here. - ExecutionType = gen_mod:get_opt(iqdisc, Opts, parallel), - IQHandlerFn = fun ?MODULE:process_mam_iq/5, - Extra = #{}, - [gen_iq_handler:add_iq_handler_for_domain(HostType, Namespace, - Component, IQHandlerFn, - Extra, ExecutionType) - || Namespace <- [?NS_MAM_04, ?NS_MAM_06]], +-spec stop(mongooseim:host_type()) -> any(). +stop(_HostType) -> ok. -remove_iq_handlers(HostType) -> - Component = ejabberd_sm, - [gen_iq_handler:remove_iq_handler_for_domain(HostType, Namespace, Component) - || Namespace <- [?NS_MAM_04, ?NS_MAM_06]], - ok. +-spec config_spec() -> mongoose_config_spec:config_section(). +config_spec() -> + Items = maps:merge(common_config_items(), root_config_items()), + #section{ + items = Items#{<<"pm">> => pm_config_spec(), + <<"muc">> => muc_config_spec(), + <<"riak">> => riak_config_spec()}, + defaults = #{<<"backend">> => rdbms, + <<"no_stanzaid_element">> => false, + <<"is_archivable_message">> => mod_mam_utils, + <<"send_message">> => mod_mam_utils, + <<"archive_chat_markers">> => false, + <<"message_retraction">> => true, + <<"full_text_search">> => true, + <<"cache_users">> => true, + <<"default_result_limit">> => 50, + <<"max_result_limit">> => 50}, + process = fun ?MODULE:remove_unused_backend_opts/1 + }. + +remove_unused_backend_opts(Opts = #{backend := riak}) -> Opts; +remove_unused_backend_opts(Opts) -> maps:remove(riak, Opts). + +pm_config_spec() -> + #section{items = maps:merge(common_config_items(), pm_config_items()), + defaults = #{<<"archive_groupchats">> => false, + <<"same_mam_id_for_peers">> => false}}. + +muc_config_spec() -> + #section{items = maps:merge(common_config_items(), muc_config_items()), + defaults = #{<<"host">> => mod_muc:default_host()}}. + +root_config_items() -> + Cache = mongoose_user_cache:config_spec(), + AsyncWriter = async_config_spec(), + #{<<"cache">> => Cache#section{include = always}, + <<"async_writer">> => AsyncWriter#section{include = always}}. + +common_config_items() -> + #{%% General options + <<"backend">> => #option{type = atom, + validate = {enum, [rdbms, riak, cassandra, elasticsearch]}}, + <<"no_stanzaid_element">> => #option{type = boolean}, + <<"is_archivable_message">> => #option{type = atom, + validate = module}, + <<"send_message">> => #option{type = atom, + validate = module}, + <<"archive_chat_markers">> => #option{type = boolean}, + <<"message_retraction">> => #option{type = boolean}, + + %% Common backend options + <<"user_prefs_store">> => #option{type = atom, + validate = {enum, [rdbms, cassandra, mnesia]}}, + <<"full_text_search">> => #option{type = boolean}, + + %% RDBMS-specific options + <<"cache_users">> => #option{type = boolean}, + + %% Low-level options + <<"default_result_limit">> => #option{type = integer, + validate = non_negative}, + <<"max_result_limit">> => #option{type = integer, + validate = non_negative}, + <<"db_jid_format">> => #option{type = atom, + validate = module}, + <<"db_message_format">> => #option{type = atom, + validate = module}, + <<"extra_fin_element">> => #option{type = atom, + validate = module}, + <<"extra_lookup_params">> => #option{type = atom, + validate = module} + }. + +pm_config_items() -> + #{<<"async_writer">> => async_config_spec(), + <<"archive_groupchats">> => #option{type = boolean}, + <<"same_mam_id_for_peers">> => #option{type = boolean}}. + +muc_config_items() -> + #{<<"async_writer">> => async_config_spec(), + <<"host">> => #option{type = string, + validate = subdomain_template, + process = fun mongoose_subdomain_utils:make_subdomain_pattern/1}}. + +async_config_spec() -> + #section{ + items = #{<<"enabled">> => #option{type = boolean}, + <<"flush_interval">> => #option{type = integer, validate = non_negative}, + <<"batch_size">> => #option{type = integer, validate = non_negative}, + <<"pool_size">> => #option{type = integer, validate = non_negative}}, + defaults = #{<<"enabled">> => true, + <<"flush_interval">> => 2000, + <<"batch_size">> => 30, + <<"pool_size">> => 4 * erlang:system_info(schedulers_online)} + }. + +riak_config_spec() -> + #section{ + items = #{<<"search_index">> => #option{type = binary, + validate = non_empty}, + <<"bucket_type">> => #option{type = binary, + validate = non_empty}}, + defaults = #{<<"search_index">> => <<"mam">>, + <<"bucket_type">> => <<"mam_yz">>}, + include = always + }. + +-spec deps(mongooseim:host_type(), module_opts()) -> gen_mod_deps:deps(). +deps(_HostType, Opts) -> + DepsWithPm = handle_nested_opts(pm, Opts, #{}), + DepsWithPmAndMuc = handle_nested_opts(muc, Opts, DepsWithPm), + + [{DepMod, DepOpts, hard} || {DepMod, DepOpts} <- maps:to_list(DepsWithPmAndMuc)]. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +-type mam_type() :: pm | muc. +-type mam_backend() :: rdbms | riak | cassandra | elasticsearch. + +-spec handle_nested_opts(mam_type(), module_opts(), module_map()) -> module_map(). +handle_nested_opts(Key, RootOpts, Deps) -> + case maps:find(Key, RootOpts) of + error -> Deps; + {ok, Opts} -> + FullOpts = maps:merge(maps:without([pm, muc], RootOpts), Opts), + parse_opts(Key, FullOpts, Deps) + end. -ensure_metrics(HostType) -> - mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, lookup], histogram), - mongoose_metrics:ensure_metric(HostType, [HostType, modMamLookups, simple], spiral), - mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, archive], histogram), - lists:foreach(fun(Name) -> - mongoose_metrics:ensure_metric(HostType, Name, spiral) - end, - spirals()). - -spirals() -> - [modMamPrefsSets, - modMamPrefsGets, - modMamArchiveRemoved, - modMamLookups, - modMamForwarded, - modMamArchived, - modMamFlushed, - modMamDropped, - modMamDroppedIQ]. +-spec parse_opts(mam_type(), module_opts(), module_map()) -> module_map(). +parse_opts(Type, Opts, Deps) -> + %% Opts are merged root options with options inside pm or muc section + CoreMod = mam_type_to_core_mod(Type), + CoreModOpts = maps:with(valid_core_mod_opts(CoreMod), Opts), + WithCoreDeps = add_dep(CoreMod, CoreModOpts, Deps), + {Backend, BackendOpts} = maps:take(backend, Opts), + WithPrefs = add_prefs_store_module(Backend, Type, Opts, WithCoreDeps), + parse_backend_opts(Backend, Type, BackendOpts, WithPrefs). + +-spec mam_type_to_core_mod(mam_type()) -> module(). +mam_type_to_core_mod(pm) -> mod_mam_pm; +mam_type_to_core_mod(muc) -> mod_mam_muc. + +%% Get a list of options to pass into the two modules. +%% They don't have to be defined in pm or muc sections, the root section is enough. +-spec valid_core_mod_opts(module()) -> [atom()]. +valid_core_mod_opts(mod_mam_pm) -> + [archive_groupchats, same_mam_id_for_peers] ++ common_opts(); +valid_core_mod_opts(mod_mam_muc) -> + [host] ++ common_opts(). + +common_opts() -> + [async_writer, + is_archivable_message, + send_message, + archive_chat_markers, + extra_fin_element, + extra_lookup_params, + full_text_search, + message_retraction, + default_result_limit, + max_result_limit, + no_stanzaid_element]. + +-spec add_prefs_store_module(mam_backend(), mam_type(), module_opts(), module_map()) -> module_map(). +add_prefs_store_module(Backend, Type, #{user_prefs_store := Store}, Deps) -> + PrefsModule = prefs_module(Backend, Store), + add_dep(PrefsModule, #{Type => true}, Deps); +add_prefs_store_module(_Backend, _Type, _Opts, Deps) -> + Deps. + +-spec parse_backend_opts(mam_backend(), mam_type(), module_opts(), module_map()) -> module_map(). +parse_backend_opts(cassandra, Type, Opts, Deps) -> + Opts1 = maps:with([db_message_format], Opts), + add_dep(cassandra_arch_module(Type), maps:merge(arch_defaults(), Opts1), Deps); +parse_backend_opts(riak, Type, Opts, Deps) -> + Opts1 = maps:with([db_message_format, riak], Opts), + add_dep(mod_mam_riak_timed_arch_yz, maps:merge(arch_defaults(), Opts1#{Type => true}), Deps); +parse_backend_opts(rdbms, Type, Opts, Deps) -> + lists:foldl(fun(OptionGroup, DepsIn) -> add_rdbms_deps(OptionGroup, Type, Opts, DepsIn) end, + Deps, [basic, user_cache, async_writer]); +parse_backend_opts(elasticsearch, Type, _Opts, Deps0) -> + add_dep(elasticsearch_arch_module(Type), Deps0). + +-spec add_rdbms_deps(basic | user_cache | async_writer, + mam_type(), module_opts(), module_map()) -> module_map(). +add_rdbms_deps(basic, Type, Opts, Deps) -> + Opts1 = maps:with([db_message_format, db_jid_format], Opts), + Deps1 = add_dep(rdbms_arch_module(Type), maps:merge(rdbms_arch_defaults(Type), Opts1), Deps), + add_dep(mod_mam_rdbms_user, user_db_types(Type), Deps1); +add_rdbms_deps(user_cache, Type, #{cache_users := true, cache := CacheOpts}, Deps) -> + Deps1 = case gen_mod:get_opt(module, CacheOpts, internal) of + internal -> Deps; + mod_cache_users -> add_dep(mod_cache_users, Deps) + end, + add_dep(mod_mam_cache_user, CacheOpts#{Type => true}, Deps1); +add_rdbms_deps(async_writer, Type, #{async_writer := AsyncOpts = #{enabled := true}}, Deps) -> + Deps1 = add_dep(rdbms_arch_module(Type), #{no_writer => true}, Deps), + add_dep(rdbms_async_arch_module(Type), AsyncOpts, Deps1); +add_rdbms_deps(_, _Type, _Opts, Deps) -> + Deps. + +% muc backend requires both pm and muc user DB to populate sender_id column +-spec user_db_types(mam_type()) -> module_opts(). +user_db_types(pm) -> #{pm => true}; +user_db_types(muc) -> #{pm => true, muc => true}. + +cassandra_arch_module(pm) -> mod_mam_cassandra_arch; +cassandra_arch_module(muc) -> mod_mam_muc_cassandra_arch. + +arch_defaults() -> #{db_message_format => mam_message_xml}. + +rdbms_arch_defaults(pm) -> + maps:merge(rdbms_arch_defaults(), #{db_jid_format => mam_jid_mini}); +rdbms_arch_defaults(muc) -> + maps:merge(rdbms_arch_defaults(), #{db_jid_format => mam_jid_rfc}). + +rdbms_arch_defaults() -> + #{db_message_format => mam_message_compressed_eterm, + no_writer => false}. + +rdbms_arch_module(pm) -> mod_mam_rdbms_arch; +rdbms_arch_module(muc) -> mod_mam_muc_rdbms_arch. + +rdbms_async_arch_module(pm) -> mod_mam_rdbms_arch_async; +rdbms_async_arch_module(muc) -> mod_mam_muc_rdbms_arch_async. + +elasticsearch_arch_module(pm) -> mod_mam_elasticsearch_arch; +elasticsearch_arch_module(muc) -> mod_mam_muc_elasticsearch_arch. + +prefs_module(rdbms, rdbms) -> mod_mam_rdbms_prefs; +prefs_module(cassandra, cassandra) -> mod_mam_cassandra_prefs; +prefs_module(_, mnesia) -> mod_mam_mnesia_prefs; +prefs_module(Backend, PrefsStore) -> + error(#{what => invalid_mam_user_prefs_store, + backend => Backend, + user_prefs_store => PrefsStore}). + +-spec add_dep(module(), module_map()) -> module_map(). +add_dep(Dep, Deps) -> + add_dep(Dep, #{}, Deps). + +-spec add_dep(module(), module_opts(), module_map()) -> module_map(). +add_dep(Dep, Opts, Deps) -> + PrevOpts = maps:get(Dep, Deps, #{}), + NewOpts = maps:merge(PrevOpts, Opts), + maps:put(Dep, NewOpts, Deps). + +config_metrics(Host) -> + mongoose_module_metrics:opts_for_module(Host, ?MODULE, [backend]). diff --git a/src/mam/mod_mam_cassandra_arch.erl b/src/mam/mod_mam_cassandra_arch.erl index 69eb26c2991..f228a820b6c 100644 --- a/src/mam/mod_mam_cassandra_arch.erl +++ b/src/mam/mod_mam_cassandra_arch.erl @@ -86,7 +86,7 @@ stop(HostType) -> ejabberd_hooks:delete(hooks(HostType)). %% ---------------------------------------------------------------------- -%% Add hooks for mod_mam +%% Add hooks for mod_mam_pm -spec hooks(host_type()) -> [ejabberd_hooks:hook()]. hooks(HostType) -> diff --git a/src/mam/mod_mam_meta.erl b/src/mam/mod_mam_meta.erl deleted file mode 100644 index aafa7178bf5..00000000000 --- a/src/mam/mod_mam_meta.erl +++ /dev/null @@ -1,296 +0,0 @@ -%%============================================================================== -%% Copyright 2016 Erlang Solutions Ltd. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%============================================================================== - --module(mod_mam_meta). --behaviour(gen_mod). --behaviour(mongoose_module_metrics). - --type module_opts() :: gen_mod:module_opts(). --type module_map() :: gen_mod_deps:module_map(). - --export([start/2, stop/1, config_spec/0, supported_features/0, deps/2, config_metrics/1]). - --export([remove_unused_backend_opts/1]). - --include("mongoose_config_spec.hrl"). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - --spec supported_features() -> [atom()]. -supported_features() -> - [dynamic_domains]. - --spec start(mongooseim:host_type(), module_opts()) -> any(). -start(_HostType, _Opts) -> - ok. - --spec stop(mongooseim:host_type()) -> any(). -stop(_HostType) -> - ok. - --spec config_spec() -> mongoose_config_spec:config_section(). -config_spec() -> - Items = maps:merge(common_config_items(), root_config_items()), - #section{ - items = Items#{<<"pm">> => pm_config_spec(), - <<"muc">> => muc_config_spec(), - <<"riak">> => riak_config_spec()}, - defaults = #{<<"backend">> => rdbms, - <<"no_stanzaid_element">> => false, - <<"is_archivable_message">> => mod_mam_utils, - <<"send_message">> => mod_mam_utils, - <<"archive_chat_markers">> => false, - <<"message_retraction">> => true, - <<"full_text_search">> => true, - <<"cache_users">> => true, - <<"default_result_limit">> => 50, - <<"max_result_limit">> => 50}, - process = fun ?MODULE:remove_unused_backend_opts/1 - }. - -remove_unused_backend_opts(Opts = #{backend := riak}) -> Opts; -remove_unused_backend_opts(Opts) -> maps:remove(riak, Opts). - -pm_config_spec() -> - #section{items = maps:merge(common_config_items(), pm_config_items()), - defaults = #{<<"archive_groupchats">> => false, - <<"same_mam_id_for_peers">> => false}}. - -muc_config_spec() -> - #section{items = maps:merge(common_config_items(), muc_config_items()), - defaults = #{<<"host">> => mod_muc:default_host()}}. - -root_config_items() -> - Cache = mongoose_user_cache:config_spec(), - AsyncWriter = async_config_spec(), - #{<<"cache">> => Cache#section{include = always}, - <<"async_writer">> => AsyncWriter#section{include = always}}. - -common_config_items() -> - #{%% General options - <<"backend">> => #option{type = atom, - validate = {enum, [rdbms, riak, cassandra, elasticsearch]}}, - <<"no_stanzaid_element">> => #option{type = boolean}, - <<"is_archivable_message">> => #option{type = atom, - validate = module}, - <<"send_message">> => #option{type = atom, - validate = module}, - <<"archive_chat_markers">> => #option{type = boolean}, - <<"message_retraction">> => #option{type = boolean}, - - %% Common backend options - <<"user_prefs_store">> => #option{type = atom, - validate = {enum, [rdbms, cassandra, mnesia]}}, - <<"full_text_search">> => #option{type = boolean}, - - %% RDBMS-specific options - <<"cache_users">> => #option{type = boolean}, - - %% Low-level options - <<"default_result_limit">> => #option{type = integer, - validate = non_negative}, - <<"max_result_limit">> => #option{type = integer, - validate = non_negative}, - <<"db_jid_format">> => #option{type = atom, - validate = module}, - <<"db_message_format">> => #option{type = atom, - validate = module}, - <<"extra_fin_element">> => #option{type = atom, - validate = module}, - <<"extra_lookup_params">> => #option{type = atom, - validate = module} - }. - -pm_config_items() -> - #{<<"async_writer">> => async_config_spec(), - <<"archive_groupchats">> => #option{type = boolean}, - <<"same_mam_id_for_peers">> => #option{type = boolean}}. - -muc_config_items() -> - #{<<"async_writer">> => async_config_spec(), - <<"host">> => #option{type = string, - validate = subdomain_template, - process = fun mongoose_subdomain_utils:make_subdomain_pattern/1}}. - -async_config_spec() -> - #section{ - items = #{<<"enabled">> => #option{type = boolean}, - <<"flush_interval">> => #option{type = integer, validate = non_negative}, - <<"batch_size">> => #option{type = integer, validate = non_negative}, - <<"pool_size">> => #option{type = integer, validate = non_negative}}, - defaults = #{<<"enabled">> => true, - <<"flush_interval">> => 2000, - <<"batch_size">> => 30, - <<"pool_size">> => 4 * erlang:system_info(schedulers_online)} - }. - -riak_config_spec() -> - #section{ - items = #{<<"search_index">> => #option{type = binary, - validate = non_empty}, - <<"bucket_type">> => #option{type = binary, - validate = non_empty}}, - defaults = #{<<"search_index">> => <<"mam">>, - <<"bucket_type">> => <<"mam_yz">>}, - include = always - }. - --spec deps(mongooseim:host_type(), module_opts()) -> gen_mod_deps:deps(). -deps(_HostType, Opts) -> - DepsWithPm = handle_nested_opts(pm, Opts, #{}), - DepsWithPmAndMuc = handle_nested_opts(muc, Opts, DepsWithPm), - - [{DepMod, DepOpts, hard} || {DepMod, DepOpts} <- maps:to_list(DepsWithPmAndMuc)]. - -%%-------------------------------------------------------------------- -%% Helpers -%%-------------------------------------------------------------------- - --type mam_type() :: pm | muc. --type mam_backend() :: rdbms | riak | cassandra | elasticsearch. - --spec handle_nested_opts(mam_type(), module_opts(), module_map()) -> module_map(). -handle_nested_opts(Key, RootOpts, Deps) -> - case maps:find(Key, RootOpts) of - error -> Deps; - {ok, Opts} -> - FullOpts = maps:merge(maps:without([pm, muc], RootOpts), Opts), - parse_opts(Key, FullOpts, Deps) - end. - --spec parse_opts(mam_type(), module_opts(), module_map()) -> module_map(). -parse_opts(Type, Opts, Deps) -> - %% Opts are merged root options with options inside pm or muc section - CoreMod = mam_type_to_core_mod(Type), - CoreModOpts = maps:with(valid_core_mod_opts(CoreMod), Opts), - WithCoreDeps = add_dep(CoreMod, CoreModOpts, Deps), - {Backend, BackendOpts} = maps:take(backend, Opts), - WithPrefs = add_prefs_store_module(Backend, Type, Opts, WithCoreDeps), - parse_backend_opts(Backend, Type, BackendOpts, WithPrefs). - --spec mam_type_to_core_mod(mam_type()) -> module(). -mam_type_to_core_mod(pm) -> mod_mam; -mam_type_to_core_mod(muc) -> mod_mam_muc. - -%% Get a list of options to pass into the two modules. -%% They don't have to be defined in pm or muc sections, the root section is enough. --spec valid_core_mod_opts(module()) -> [atom()]. -valid_core_mod_opts(mod_mam) -> - [archive_groupchats, same_mam_id_for_peers] ++ common_opts(); -valid_core_mod_opts(mod_mam_muc) -> - [host] ++ common_opts(). - -common_opts() -> - [async_writer, - is_archivable_message, - send_message, - archive_chat_markers, - extra_fin_element, - extra_lookup_params, - full_text_search, - message_retraction, - default_result_limit, - max_result_limit, - no_stanzaid_element]. - --spec add_prefs_store_module(mam_backend(), mam_type(), module_opts(), module_map()) -> module_map(). -add_prefs_store_module(Backend, Type, #{user_prefs_store := Store}, Deps) -> - PrefsModule = prefs_module(Backend, Store), - add_dep(PrefsModule, #{Type => true}, Deps); -add_prefs_store_module(_Backend, _Type, _Opts, Deps) -> - Deps. - --spec parse_backend_opts(mam_backend(), mam_type(), module_opts(), module_map()) -> module_map(). -parse_backend_opts(cassandra, Type, Opts, Deps) -> - Opts1 = maps:with([db_message_format], Opts), - add_dep(cassandra_arch_module(Type), maps:merge(arch_defaults(), Opts1), Deps); -parse_backend_opts(riak, Type, Opts, Deps) -> - Opts1 = maps:with([db_message_format, riak], Opts), - add_dep(mod_mam_riak_timed_arch_yz, maps:merge(arch_defaults(), Opts1#{Type => true}), Deps); -parse_backend_opts(rdbms, Type, Opts, Deps) -> - lists:foldl(fun(OptionGroup, DepsIn) -> add_rdbms_deps(OptionGroup, Type, Opts, DepsIn) end, - Deps, [basic, user_cache, async_writer]); -parse_backend_opts(elasticsearch, Type, _Opts, Deps0) -> - add_dep(elasticsearch_arch_module(Type), Deps0). - --spec add_rdbms_deps(basic | user_cache | async_writer, - mam_type(), module_opts(), module_map()) -> module_map(). -add_rdbms_deps(basic, Type, Opts, Deps) -> - Opts1 = maps:with([db_message_format, db_jid_format], Opts), - Deps1 = add_dep(rdbms_arch_module(Type), maps:merge(rdbms_arch_defaults(Type), Opts1), Deps), - add_dep(mod_mam_rdbms_user, user_db_types(Type), Deps1); -add_rdbms_deps(user_cache, Type, #{cache_users := true, cache := CacheOpts}, Deps) -> - Deps1 = case gen_mod:get_opt(module, CacheOpts, internal) of - internal -> Deps; - mod_cache_users -> add_dep(mod_cache_users, Deps) - end, - add_dep(mod_mam_cache_user, CacheOpts#{Type => true}, Deps1); -add_rdbms_deps(async_writer, Type, #{async_writer := AsyncOpts = #{enabled := true}}, Deps) -> - Deps1 = add_dep(rdbms_arch_module(Type), #{no_writer => true}, Deps), - add_dep(rdbms_async_arch_module(Type), AsyncOpts, Deps1); -add_rdbms_deps(_, _Type, _Opts, Deps) -> - Deps. - -% muc backend requires both pm and muc user DB to populate sender_id column --spec user_db_types(mam_type()) -> module_opts(). -user_db_types(pm) -> #{pm => true}; -user_db_types(muc) -> #{pm => true, muc => true}. - -cassandra_arch_module(pm) -> mod_mam_cassandra_arch; -cassandra_arch_module(muc) -> mod_mam_muc_cassandra_arch. - -arch_defaults() -> #{db_message_format => mam_message_xml}. - -rdbms_arch_defaults(pm) -> - maps:merge(rdbms_arch_defaults(), #{db_jid_format => mam_jid_mini}); -rdbms_arch_defaults(muc) -> - maps:merge(rdbms_arch_defaults(), #{db_jid_format => mam_jid_rfc}). - -rdbms_arch_defaults() -> - #{db_message_format => mam_message_compressed_eterm, - no_writer => false}. - -rdbms_arch_module(pm) -> mod_mam_rdbms_arch; -rdbms_arch_module(muc) -> mod_mam_muc_rdbms_arch. - -rdbms_async_arch_module(pm) -> mod_mam_rdbms_arch_async; -rdbms_async_arch_module(muc) -> mod_mam_muc_rdbms_arch_async. - -elasticsearch_arch_module(pm) -> mod_mam_elasticsearch_arch; -elasticsearch_arch_module(muc) -> mod_mam_muc_elasticsearch_arch. - -prefs_module(rdbms, rdbms) -> mod_mam_rdbms_prefs; -prefs_module(cassandra, cassandra) -> mod_mam_cassandra_prefs; -prefs_module(_, mnesia) -> mod_mam_mnesia_prefs; -prefs_module(Backend, PrefsStore) -> - error(#{what => invalid_mam_user_prefs_store, - backend => Backend, - user_prefs_store => PrefsStore}). - --spec add_dep(module(), module_map()) -> module_map(). -add_dep(Dep, Deps) -> - add_dep(Dep, #{}, Deps). - --spec add_dep(module(), module_opts(), module_map()) -> module_map(). -add_dep(Dep, Opts, Deps) -> - PrevOpts = maps:get(Dep, Deps, #{}), - NewOpts = maps:merge(PrevOpts, Opts), - maps:put(Dep, NewOpts, Deps). - -config_metrics(Host) -> - mongoose_module_metrics:opts_for_module(Host, ?MODULE, [backend]). diff --git a/src/mam/mod_mam_muc.erl b/src/mam/mod_mam_muc.erl index 5059ce3a461..cf1f1031f08 100644 --- a/src/mam/mod_mam_muc.erl +++ b/src/mam/mod_mam_muc.erl @@ -500,7 +500,7 @@ remove_archive(HostType, ArcID, ArcJID = #jid{}) -> mongoose_hooks:mam_muc_remove_archive(HostType, ArcID, ArcJID), ok. -%% See description in mod_mam. +%% See description in mod_mam_pm. -spec lookup_messages(HostType :: host_type(), Params :: map()) -> {ok, mod_mam:lookup_result()} | {error, 'policy-violation'} diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 9114fc63d32..e08248b9294 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -71,7 +71,7 @@ supported_features() -> host_type(), jid:jid()) -> ejabberd_gen_mam_archive:mam_muc_gdpr_data(). get_mam_muc_gdpr_data(Acc, HostType, #jid{luser = LUser, lserver = LServer} = _UserJID) -> - case mod_mam:archive_id(LServer, LUser) of + case mod_mam_pm:archive_id(LServer, LUser) of undefined -> Acc; SenderID -> @@ -82,7 +82,7 @@ get_mam_muc_gdpr_data(Acc, HostType, #jid{luser = LUser, lserver = LServer} = _U end. %% ---------------------------------------------------------------------- -%% Add hooks for mod_mam +%% Add hooks for mod_mam_pm -spec start_hooks(host_type()) -> ok. start_hooks(HostType) -> @@ -221,7 +221,7 @@ archive_size(Size, HostType, ArcID, ArcJID) when is_integer(Size) -> extend_params_with_sender_id(HostType, Params = #{remote_jid := SenderJID}) -> BareSenderJID = jid:to_bare(SenderJID), - SenderID = mod_mam:archive_id_int(HostType, BareSenderJID), + SenderID = mod_mam_pm:archive_id_int(HostType, BareSenderJID), Params#{sender_id => SenderID}. -spec archive_message(_Result, HostType :: mongooseim:host_type(), diff --git a/src/mam/mod_mam_params.erl b/src/mam/mod_mam_params.erl index 4924aaba303..70c740baf16 100644 --- a/src/mam/mod_mam_params.erl +++ b/src/mam/mod_mam_params.erl @@ -16,7 +16,7 @@ -module(mod_mam_params). --type mam_module() :: mod_mam | mod_mam_muc. +-type mam_module() :: mod_mam_pm | mod_mam_muc. -export([extra_params_module/2, max_result_limit/2, default_result_limit/2, has_full_text_search/2, is_archivable_message_module/2, send_message_mod/2, @@ -39,7 +39,7 @@ default_result_limit(Module, HostType) -> gen_mod:get_module_opt(HostType, Module, default_result_limit). --spec has_full_text_search(Module :: mod_mam | mod_mam_muc, mongooseim:host_type()) -> boolean(). +-spec has_full_text_search(Module :: mod_mam_pm | mod_mam_muc, mongooseim:host_type()) -> boolean(). has_full_text_search(Module, HostType) -> gen_mod:get_module_opt(HostType, Module, full_text_search). diff --git a/src/mam/mod_mam_pm.erl b/src/mam/mod_mam_pm.erl new file mode 100644 index 00000000000..40f08bb9641 --- /dev/null +++ b/src/mam/mod_mam_pm.erl @@ -0,0 +1,731 @@ +%%%------------------------------------------------------------------- +%%% @author Uvarov Michael +%%% @copyright (C) 2013, Uvarov Michael +%%% @doc XEP-0313: Message Archive Management +%%% +%%% The module uses several backend modules: +%%% +%%%
    +%%%
  • Preference manager ({@link mod_mam_muc_rdbms_prefs});
    • +%%%
  • Writer ({@link mod_mam_muc_rdbms_arch} or {@link mod_mam_muc_rdbms_async_pool_writer});
    • +%%%
  • Archive manager ({@link mod_mam_muc_rdbms_arch});
    • +%%%
  • User's ID generator ({@link mod_mam_muc_user}).
    • +%%%
+%%% +%%% Preferences can be also stored in Mnesia ({@link mod_mam_mnesia_prefs}). +%%% This module handles simple archives. +%%% +%%% This module should be started for each host. +%%% Message archivation is not shaped here (use standard support for this). +%%% MAM's IQs are shaped inside {@link shaper_srv}. +%%% +%%% Message identifiers (or UIDs in the spec) are generated based on: +%%% +%%%
    +%%%
  • date (using `timestamp()');
    • +%%%
  • node number (using {@link ejabberd_node_id}).
    • +%%%
+%%% @end +%%%------------------------------------------------------------------- +-module(mod_mam_pm). +-behavior(gen_mod). +-behaviour(mongoose_module_metrics). +-xep([{xep, 313}, {version, "0.4.1"}]). +-xep([{xep, 313}, {version, "0.5"}]). +-xep([{xep, 313}, {version, "0.6"}]). +%% ---------------------------------------------------------------------- +%% Exports + +%% Client API +-export([delete_archive/2, + archive_size/2, + archive_size_with_host_type/3, + archive_id/2]). + +%% gen_mod handlers +-export([start/2, stop/1, supported_features/0]). + +%% ejabberd handlers +-export([disco_local_features/1, + process_mam_iq/5, + user_send_packet/4, + remove_user/3, + filter_packet/1, + determine_amp_strategy/5, + sm_filter_offline_message/4]). + +%% gdpr callbacks +-export([get_personal_data/3]). + +%%private +-export([archive_message_from_ct/1]). +-export([lookup_messages/2]). +-export([archive_id_int/2]). + +-ignore_xref([archive_message_from_ct/1, + archive_size/2, archive_size_with_host_type/3, delete_archive/2, + determine_amp_strategy/5, disco_local_features/1, filter_packet/1, + get_personal_data/3, remove_user/3, sm_filter_offline_message/4, + user_send_packet/4]). + +-type host_type() :: mongooseim:host_type(). + +%% ---------------------------------------------------------------------- +%% Imports + +%% UID +-import(mod_mam_utils, + [get_or_generate_mam_id/1, + decode_compact_uuid/1]). + +%% XML +-import(mod_mam_utils, + [maybe_add_arcid_elems/4, + wrap_message/6, + result_set/4, + result_prefs/4, + make_fin_element/7, + parse_prefs/1, + is_mam_result_message/1, + features/2]). + +%% Forms +-import(mod_mam_utils, + [message_form/3]). + +%% Other +-import(mod_mam_utils, + [mess_id_to_external_binary/1, + is_complete_result_page/4]). + +%% ejabberd +-import(mod_mam_utils, + [is_jid_in_user_roster/3]). + + +-include("mongoose.hrl"). +-include("jlib.hrl"). +-include("amp.hrl"). + +%% ---------------------------------------------------------------------- +%% API + +-spec get_personal_data(gdpr:personal_data(), mongooseim:host_type(), jid:jid()) -> + gdpr:personal_data(). +get_personal_data(Acc, HostType, ArcJID) -> + Schema = ["id", "from", "message"], + Entries = mongoose_hooks:get_mam_pm_gdpr_data(HostType, ArcJID), + [{mam_pm, Schema, Entries} | Acc]. + +-spec delete_archive(jid:server(), jid:user()) -> 'ok'. +delete_archive(Server, User) + when is_binary(Server), is_binary(User) -> + ?LOG_DEBUG(#{what => mam_delete_archive, user => User, server => Server}), + ArcJID = jid:make_bare(User, Server), + HostType = jid_to_host_type(ArcJID), + ArcID = archive_id_int(HostType, ArcJID), + remove_archive_hook(HostType, ArcID, ArcJID), + ok. + +-spec archive_size(jid:server(), jid:user()) -> integer(). +archive_size(Server, User) + when is_binary(Server), is_binary(User) -> + ArcJID = jid:make_bare(User, Server), + HostType = jid_to_host_type(ArcJID), + ArcID = archive_id_int(HostType, ArcJID), + archive_size(HostType, ArcID, ArcJID). + +-spec archive_size_with_host_type(host_type(), jid:server(), jid:user()) -> integer(). +archive_size_with_host_type(HostType, Server, User) -> + ArcJID = jid:make_bare(User, Server), + ArcID = archive_id_int(HostType, ArcJID), + archive_size(HostType, ArcID, ArcJID). + +-spec archive_id(jid:server(), jid:user()) -> integer() | undefined. +archive_id(Server, User) + when is_binary(Server), is_binary(User) -> + ArcJID = jid:make_bare(User, Server), + HostType = jid_to_host_type(ArcJID), + archive_id_int(HostType, ArcJID). + +%% gen_mod callbacks +%% Starting and stopping functions for users' archives + +-spec start(host_type(), gen_mod:module_opts()) -> any(). +start(HostType, Opts) -> + ?LOG_INFO(#{what => mam_starting, host_type => HostType}), + ensure_metrics(HostType), + ejabberd_hooks:add(hooks(HostType)), + add_iq_handlers(HostType, Opts), + ok. + +-spec stop(host_type()) -> any(). +stop(HostType) -> + ?LOG_INFO(#{what => mam_stopping, host_type => HostType}), + ejabberd_hooks:delete(hooks(HostType)), + remove_iq_handlers(HostType), + ok. + +-spec supported_features() -> [atom()]. +supported_features() -> + [dynamic_domains]. + +%% ---------------------------------------------------------------------- +%% hooks and handlers + +%% `To' is an account or server entity hosting the archive. +%% Servers that archive messages on behalf of local users SHOULD expose archives +%% to the user on their bare JID (i.e. `From.luser'), +%% while a MUC service might allow MAM queries to be sent to the room's bare JID +%% (i.e `To.luser'). +-spec process_mam_iq(Acc :: mongoose_acc:t(), + From :: jid:jid(), To :: jid:jid(), IQ :: jlib:iq(), + _Extra) -> {mongoose_acc:t(), jlib:iq() | ignore}. +process_mam_iq(Acc, From, To, IQ, _Extra) -> + HostType = mongoose_acc:host_type(Acc), + mod_mam_utils:maybe_log_deprecation(IQ), + Action = mam_iq:action(IQ), + case is_action_allowed(HostType, Action, From, To) of + true -> + case mod_mam_utils:wait_shaper(HostType, To#jid.lserver, Action, From) of + ok -> + handle_error_iq(HostType, Acc, To, Action, + handle_mam_iq(Action, From, To, IQ, Acc)); + {error, max_delay_reached} -> + ?LOG_WARNING(#{what => mam_max_delay_reached, + text => <<"Return max_delay_reached error IQ from MAM">>, + action => Action, acc => Acc}), + mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + {Acc, return_max_delay_reached_error_iq(IQ)} + end; + false -> + mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + {Acc, return_action_not_allowed_error_iq(IQ)} + end. + +-spec disco_local_features(mongoose_disco:feature_acc()) -> mongoose_disco:feature_acc(). +disco_local_features(Acc = #{host_type := HostType, node := <<>>}) -> + mongoose_disco:add_features(features(?MODULE, HostType), Acc); +disco_local_features(Acc) -> + Acc. + +%% @doc Handle an outgoing message. +%% +%% Note: for outgoing messages, the server MUST use the value of the 'to' +%% attribute as the target JID. +-spec user_send_packet(Acc :: mongoose_acc:t(), From :: jid:jid(), + To :: jid:jid(), + Packet :: exml:element()) -> mongoose_acc:t(). +user_send_packet(Acc, From, To, Packet) -> + ?LOG_DEBUG(#{what => mam_user_send_packet, acc => Acc}), + {_, Acc2} = handle_package(outgoing, true, From, To, From, Packet, Acc), + Acc2. + +%% @doc Handle an incoming message. +%% +%% Note: For incoming messages, the server MUST use the value of the +%% 'from' attribute as the target JID. +%% +%% Return drop to drop the packet, or the original input to let it through. +%% From and To are jid records. +-type fpacket() :: {From :: jid:jid(), + To :: jid:jid(), + Acc :: mongoose_acc:t(), + Packet :: exml:element()}. +-spec filter_packet(Value :: fpacket() | drop) -> fpacket() | drop. +filter_packet(drop) -> + drop; +filter_packet({From, To, Acc1, Packet}) -> + ?LOG_DEBUG(#{what => mam_user_receive_packet, acc => Acc1}), + HostType = mongoose_acc:host_type(Acc1), + Type = mongoose_lib:get_message_type(Acc1), + {AmpEvent, PacketAfterArchive, Acc3} = + case mongoose_lib:does_local_user_exist(HostType, To, Type) of + false -> + {mam_failed, Packet, Acc1}; + true -> + case process_incoming_packet(From, To, Packet, Acc1) of + {undefined, Acc2} -> + {mam_failed, Packet, Acc2}; + {MessID, Acc2} -> + Packet2 = maybe_add_arcid_elems( + To, MessID, Packet, + mod_mam_params:add_stanzaid_element(?MODULE, HostType)), + {archived, Packet2, Acc2} + end + end, + Acc4 = mongoose_acc:update_stanza(#{ element => PacketAfterArchive, + from_jid => From, + to_jid => To }, Acc3), + Acc5 = mod_amp:check_packet(Acc4, AmpEvent), + {From, To, Acc5, mongoose_acc:element(Acc5)}. + +process_incoming_packet(From, To, Packet, Acc) -> + handle_package(incoming, true, To, From, From, Packet, Acc). + +%% hook handler +-spec remove_user(mongoose_acc:t(), jid:user(), jid:server()) -> mongoose_acc:t(). +remove_user(Acc, User, Server) -> + delete_archive(Server, User), + Acc. + +sm_filter_offline_message(_Drop=false, _From, _To, Packet) -> + %% If ... + is_mam_result_message(Packet); + %% ... than drop the message +sm_filter_offline_message(Other, _From, _To, _Packet) -> + Other. + +%% ---------------------------------------------------------------------- +%% Internal functions + +-spec jid_to_host_type(jid:jid()) -> host_type(). +jid_to_host_type(#jid{lserver=LServer}) -> + lserver_to_host_type(LServer). + +lserver_to_host_type(LServer) -> + case mongoose_domain_api:get_domain_host_type(LServer) of + {ok, HostType} -> + HostType; + {error, not_found} -> + error({get_domain_host_type_failed, LServer}) + end. + +-spec acc_to_host_type(mongoose_acc:t()) -> host_type(). +acc_to_host_type(Acc) -> + case mongoose_acc:host_type(Acc) of + undefined -> + lserver_to_host_type(mongoose_acc:lserver(Acc)); + HostType -> + HostType + end. + +-spec is_action_allowed(HostType :: host_type(), + Action :: mam_iq:action(), From :: jid:jid(), + To :: jid:jid()) -> boolean(). +is_action_allowed(HostType, Action, From, To) -> + case acl:match_rule(HostType, To#jid.lserver, Action, From, default) of + allow -> true; + deny -> false; + default -> is_action_allowed_by_default(Action, From, To) + end. + +-spec is_action_allowed_by_default(Action :: mam_iq:action(), From :: jid:jid(), + To :: jid:jid()) -> boolean(). +is_action_allowed_by_default(_Action, From, To) -> + compare_bare_jids(From, To). + +-spec compare_bare_jids(jid:simple_jid() | jid:jid(), + jid:simple_jid() | jid:jid()) -> boolean(). +compare_bare_jids(JID1, JID2) -> + jid:to_bare(JID1) =:= jid:to_bare(JID2). + +-spec handle_mam_iq(mam_iq:action(), From :: jid:jid(), To :: jid:jid(), + IQ :: jlib:iq(), Acc :: mongoose_acc:t()) -> + jlib:iq() | {error, term(), jlib:iq()}. +handle_mam_iq(Action, From, To, IQ, Acc) -> + case Action of + mam_get_prefs -> + handle_get_prefs(To, IQ, Acc); + mam_set_prefs -> + handle_set_prefs(To, IQ, Acc); + mam_set_message_form -> + handle_set_message_form(From, To, IQ, Acc); + mam_get_message_form -> + handle_get_message_form(From, To, IQ, Acc) + end. + +-spec handle_set_prefs(jid:jid(), jlib:iq(), mongoose_acc:t()) -> + jlib:iq() | {error, term(), jlib:iq()}. +handle_set_prefs(ArcJID=#jid{}, IQ=#iq{sub_el = PrefsEl}, Acc) -> + {DefaultMode, AlwaysJIDs, NeverJIDs} = parse_prefs(PrefsEl), + ?LOG_DEBUG(#{what => mam_set_prefs, default_mode => DefaultMode, + always_jids => AlwaysJIDs, never_jids => NeverJIDs, iq => IQ}), + HostType = acc_to_host_type(Acc), + ArcID = archive_id_int(HostType, ArcJID), + Res = set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs), + handle_set_prefs_result(Res, DefaultMode, AlwaysJIDs, NeverJIDs, IQ). + +handle_set_prefs_result(ok, DefaultMode, AlwaysJIDs, NeverJIDs, IQ) -> + Namespace = IQ#iq.xmlns, + ResultPrefsEl = result_prefs(DefaultMode, AlwaysJIDs, NeverJIDs, Namespace), + IQ#iq{type = result, sub_el = [ResultPrefsEl]}; +handle_set_prefs_result({error, Reason}, + _DefaultMode, _AlwaysJIDs, _NeverJIDs, IQ) -> + return_error_iq(IQ, Reason). + +-spec handle_get_prefs(jid:jid(), IQ :: jlib:iq(), Acc :: mongoose_acc:t()) -> + jlib:iq() | {error, term(), jlib:iq()}. +handle_get_prefs(ArcJID=#jid{}, IQ=#iq{}, Acc) -> + HostType = acc_to_host_type(Acc), + ArcID = archive_id_int(HostType, ArcJID), + Res = get_prefs(HostType, ArcID, ArcJID, always), + handle_get_prefs_result(Res, IQ). + +handle_get_prefs_result({DefaultMode, AlwaysJIDs, NeverJIDs}, IQ) -> + ?LOG_DEBUG(#{what => mam_get_prefs_result, default_mode => DefaultMode, + always_jids => AlwaysJIDs, never_jids => NeverJIDs, iq => IQ}), + Namespace = IQ#iq.xmlns, + ResultPrefsEl = result_prefs(DefaultMode, AlwaysJIDs, NeverJIDs, Namespace), + IQ#iq{type = result, sub_el = [ResultPrefsEl]}; +handle_get_prefs_result({error, Reason}, IQ) -> + return_error_iq(IQ, Reason). + +-spec handle_set_message_form(From :: jid:jid(), ArcJID :: jid:jid(), + IQ :: jlib:iq(), Acc :: mongoose_acc:t()) -> + jlib:iq() | ignore | {error, term(), jlib:iq()}. +handle_set_message_form(#jid{} = From, #jid{} = ArcJID, #iq{} = IQ, Acc) -> + HostType = acc_to_host_type(Acc), + ArcID = archive_id_int(HostType, ArcJID), + try iq_to_lookup_params(HostType, IQ) of + Params0 -> + do_handle_set_message_form(Params0, From, ArcID, ArcJID, IQ, HostType) + catch _C:R:S -> + report_issue({R, S}, mam_lookup_failed, ArcJID, IQ), + return_error_iq(IQ, R) + end. + + +-spec do_handle_set_message_form(Params :: mam_iq:lookup_params(), + From :: jid:jid(), + ArcId :: mod_mam:archive_id(), + ArcJID :: jid:jid(), + IQ :: jlib:iq(), + HostType :: mongooseim:host_type()) -> + jlib:iq() | ignore | {error, term(), jlib:iq()}. +do_handle_set_message_form(Params0, From, ArcID, ArcJID, + #iq{xmlns=MamNs, sub_el = QueryEl} = IQ, + HostType) -> + QueryID = exml_query:attr(QueryEl, <<"queryid">>, <<>>), + Params = mam_iq:lookup_params_with_archive_details(Params0, ArcID, ArcJID, From), + case lookup_messages(HostType, Params) of + {error, Reason} -> + report_issue(Reason, mam_lookup_failed, ArcJID, IQ), + return_error_iq(IQ, Reason); + {ok, {TotalCount, Offset, MessageRows}} -> + %% Forward messages + {FirstMessID, LastMessID} = forward_messages(HostType, From, ArcJID, MamNs, + QueryID, MessageRows, true), + %% Make fin iq + IsComplete = is_complete_result_page(TotalCount, Offset, MessageRows, Params), + IsStable = true, + ResultSetEl = result_set(FirstMessID, LastMessID, Offset, TotalCount), + ExtFinMod = mod_mam_params:extra_fin_element_module(?MODULE, HostType), + FinElem = make_fin_element(HostType, Params, IQ#iq.xmlns, IsComplete, IsStable, ResultSetEl, ExtFinMod), + IQ#iq{type = result, sub_el = [FinElem]} + end. + +iq_to_lookup_params(HostType, IQ) -> + Max = mod_mam_params:max_result_limit(?MODULE, HostType), + Def = mod_mam_params:default_result_limit(?MODULE, HostType), + Ext = mod_mam_params:extra_params_module(?MODULE, HostType), + mam_iq:form_to_lookup_params(IQ, Max, Def, Ext). + +forward_messages(HostType, From, ArcJID, MamNs, QueryID, MessageRows, SetClientNs) -> + %% Forward messages + {FirstMessID, LastMessID} = + case MessageRows of + [] -> {undefined, undefined}; + [_|_] -> {message_row_to_ext_id(hd(MessageRows)), + message_row_to_ext_id(lists:last(MessageRows))} + end, + SendModule = mod_mam_params:send_message_mod(?MODULE, HostType), + [send_message(SendModule, Row, ArcJID, From, + message_row_to_xml(MamNs, Row, QueryID, SetClientNs)) + || Row <- MessageRows], + {FirstMessID, LastMessID}. + +send_message(SendModule, Row, ArcJID, From, Packet) -> + mam_send_message:call_send_message(SendModule, Row, ArcJID, From, Packet). + +-spec handle_get_message_form(jid:jid(), jid:jid(), jlib:iq(), mongoose_acc:t()) -> + jlib:iq(). +handle_get_message_form(_From=#jid{}, _ArcJID=#jid{}, IQ=#iq{}, Acc) -> + HostType = acc_to_host_type(Acc), + return_message_form_iq(HostType, IQ). + +determine_amp_strategy(Strategy = #amp_strategy{deliver = Deliver}, + FromJID, ToJID, Packet, initial_check) -> + HostType = jid_to_host_type(ToJID), + ShouldBeStored = is_archivable_message(HostType, incoming, Packet) + andalso is_interesting(ToJID, FromJID) + andalso ejabberd_auth:does_user_exist(ToJID), + case ShouldBeStored of + true -> Strategy#amp_strategy{deliver = amp_deliver_strategy(Deliver)}; + false -> Strategy + end; +determine_amp_strategy(Strategy, _, _, _, _) -> + Strategy. + +amp_deliver_strategy([none]) -> [stored, none]; +amp_deliver_strategy([direct, none]) -> [direct, stored, none]. + +-spec handle_package(Dir :: incoming | outgoing, ReturnMessID :: boolean(), + LocJID :: jid:jid(), RemJID :: jid:jid(), SrcJID :: jid:jid(), + Packet :: exml:element(), Acc :: mongoose_acc:t()) -> + {MaybeMessID :: binary() | undefined, Acc :: mongoose_acc:t()}. +handle_package(Dir, ReturnMessID, + LocJID = #jid{}, RemJID = #jid{}, SrcJID = #jid{}, Packet, Acc) -> + HostType = acc_to_host_type(Acc), + case is_archivable_message(HostType, Dir, Packet) + andalso should_archive_if_groupchat(HostType, exml_query:attr(Packet, <<"type">>)) of + true -> + ArcID = archive_id_int(HostType, LocJID), + OriginID = mod_mam_utils:get_origin_id(Packet), + case is_interesting(HostType, LocJID, RemJID, ArcID) of + true -> + MessID = get_or_generate_mam_id(Acc), + Params = #{message_id => MessID, + archive_id => ArcID, + local_jid => LocJID, + remote_jid => RemJID, + source_jid => SrcJID, + origin_id => OriginID, + direction => Dir, + packet => Packet}, + Result = archive_message(HostType, Params), + ExtMessId = return_external_message_id_if_ok(ReturnMessID, Result, MessID), + {ExtMessId, return_acc_with_mam_id_if_configured(ExtMessId, HostType, Acc)}; + false -> + {undefined, Acc} + end; + false -> + {undefined, Acc} + end. + +should_archive_if_groupchat(HostType, <<"groupchat">>) -> + gen_mod:get_module_opt(HostType, ?MODULE, archive_groupchats); +should_archive_if_groupchat(_, _) -> + true. + +-spec return_external_message_id_if_ok(ReturnMessID :: boolean(), + ArchivingResult :: ok | any(), + MessID :: integer()) -> binary() | undefined. +return_external_message_id_if_ok(true, ok, MessID) -> mess_id_to_external_binary(MessID); +return_external_message_id_if_ok(_, _, _MessID) -> undefined. + +return_acc_with_mam_id_if_configured(undefined, _, Acc) -> + Acc; +return_acc_with_mam_id_if_configured(ExtMessId, HostType, Acc) -> + case gen_mod:get_module_opt(HostType, ?MODULE, same_mam_id_for_peers) of + false -> mongoose_acc:set(mam, mam_id, ExtMessId, Acc); + true -> mongoose_acc:set_permanent(mam, mam_id, ExtMessId, Acc) + end. + +is_interesting(LocJID, RemJID) -> + HostType = jid_to_host_type(LocJID), + ArcID = archive_id_int(HostType, LocJID), + is_interesting(HostType, LocJID, RemJID, ArcID). + +is_interesting(HostType, LocJID, RemJID, ArcID) -> + case get_behaviour(HostType, ArcID, LocJID, RemJID) of + always -> true; + never -> false; + roster -> is_jid_in_user_roster(HostType, LocJID, RemJID) + end. + +%% ---------------------------------------------------------------------- +%% Backend wrappers + +-spec archive_id_int(host_type(), jid:jid()) -> + non_neg_integer() | undefined. +archive_id_int(HostType, ArcJID=#jid{}) -> + mongoose_hooks:mam_archive_id(HostType, ArcJID). + +-spec archive_size(host_type(), mod_mam:archive_id(), jid:jid()) -> integer(). +archive_size(HostType, ArcID, ArcJID=#jid{}) -> + mongoose_hooks:mam_archive_size(HostType, ArcID, ArcJID). + +-spec get_behaviour(host_type(), mod_mam:archive_id(), LocJID :: jid:jid(), + RemJID :: jid:jid()) -> atom(). +get_behaviour(HostType, ArcID, LocJID=#jid{}, RemJID=#jid{}) -> + mongoose_hooks:mam_get_behaviour(HostType, ArcID, LocJID, RemJID). + +-spec set_prefs(host_type(), mod_mam:archive_id(), ArcJID :: jid:jid(), + DefaultMode :: atom(), AlwaysJIDs :: [jid:literal_jid()], + NeverJIDs :: [jid:literal_jid()]) -> any(). +set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> + mongoose_hooks:mam_set_prefs(HostType, ArcID, ArcJID, DefaultMode, + AlwaysJIDs, NeverJIDs). + +%% @doc Load settings from the database. +-spec get_prefs(HostType :: host_type(), ArcID :: mod_mam:archive_id(), + ArcJID :: jid:jid(), GlobalDefaultMode :: mod_mam:archive_behaviour() + ) -> mod_mam:preference() | {error, Reason :: term()}. +get_prefs(HostType, ArcID, ArcJID, GlobalDefaultMode) -> + mongoose_hooks:mam_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID). + +-spec remove_archive_hook(host_type(), mod_mam:archive_id(), jid:jid()) -> 'ok'. +remove_archive_hook(HostType, ArcID, ArcJID=#jid{}) -> + mongoose_hooks:mam_remove_archive(HostType, ArcID, ArcJID), + ok. + +-spec lookup_messages(HostType :: host_type(), Params :: map()) -> + {ok, mod_mam:lookup_result()} + | {error, 'policy-violation'} + | {error, Reason :: term()}. +lookup_messages(HostType, Params) -> + Result = lookup_messages_without_policy_violation_check(HostType, Params), + %% If a query returns a number of stanzas greater than this limit and the + %% client did not specify a limit using RSM then the server should return + %% a policy-violation error to the client. + mod_mam_utils:check_result_for_policy_violation(Params, Result). + +lookup_messages_without_policy_violation_check( + HostType, #{search_text := SearchText} = Params) -> + case SearchText /= undefined andalso + not mod_mam_params:has_full_text_search(?MODULE, HostType) of + true -> %% Use of disabled full text search + {error, 'not-supported'}; + false -> + StartT = erlang:monotonic_time(microsecond), + R = mongoose_hooks:mam_lookup_messages(HostType, Params), + Diff = erlang:monotonic_time(microsecond) - StartT, + mongoose_metrics:update(HostType, [backends, ?MODULE, lookup], Diff), + R + end. + +archive_message_from_ct(Params = #{local_jid := JID}) -> + HostType = jid_to_host_type(JID), + archive_message(HostType, Params). + +-spec archive_message(host_type(), mod_mam:archive_message_params()) -> + ok | {error, timeout}. +archive_message(HostType, Params) -> + StartT = erlang:monotonic_time(microsecond), + R = mongoose_hooks:mam_archive_message(HostType, Params), + Diff = erlang:monotonic_time(microsecond) - StartT, + mongoose_metrics:update(HostType, [backends, ?MODULE, archive], Diff), + R. + +%% ---------------------------------------------------------------------- +%% Helpers + +-spec message_row_to_xml(binary(), mod_mam:message_row(), QueryId :: binary(), boolean()) -> + exml:element(). +message_row_to_xml(MamNs, #{id := MessID, jid := SrcJID, packet := Packet}, + QueryID, SetClientNs) -> + {Microseconds, _NodeMessID} = decode_compact_uuid(MessID), + TS = calendar:system_time_to_rfc3339(Microseconds, [{offset, "Z"}, {unit, microsecond}]), + BExtMessID = mess_id_to_external_binary(MessID), + Packet1 = mod_mam_utils:maybe_set_client_xmlns(SetClientNs, Packet), + wrap_message(MamNs, Packet1, QueryID, BExtMessID, TS, SrcJID). + +-spec message_row_to_ext_id(mod_mam:message_row()) -> binary(). +message_row_to_ext_id(#{id := MessID}) -> + mess_id_to_external_binary(MessID). + +handle_error_iq(HostType, Acc, _To, _Action, {error, _Reason, IQ}) -> + mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + {Acc, IQ}; +handle_error_iq(_Host, Acc, _To, _Action, IQ) -> + {Acc, IQ}. + +-spec return_action_not_allowed_error_iq(jlib:iq()) -> jlib:iq(). +return_action_not_allowed_error_iq(IQ) -> + ErrorEl = jlib:stanza_errort(<<"">>, <<"cancel">>, <<"not-allowed">>, + <<"en">>, <<"The action is not allowed.">>), + IQ#iq{type = error, sub_el = [ErrorEl]}. + +-spec return_max_delay_reached_error_iq(jlib:iq()) -> jlib:iq(). +return_max_delay_reached_error_iq(IQ) -> + %% Message not found. + ErrorEl = mongoose_xmpp_errors:resource_constraint( + <<"en">>, <<"The action is cancelled because of flooding.">>), + IQ#iq{type = error, sub_el = [ErrorEl]}. + +-spec return_error_iq(jlib:iq(), Reason :: term()) -> {error, term(), jlib:iq()}. +return_error_iq(IQ, {Reason, {stacktrace, _Stacktrace}}) -> + return_error_iq(IQ, Reason); +return_error_iq(IQ, timeout) -> + E = mongoose_xmpp_errors:service_unavailable(<<"en">>, <<"Timeout">>), + {error, timeout, IQ#iq{type = error, sub_el = [E]}}; +return_error_iq(IQ, invalid_stanza_id) -> + Text = mongoose_xmpp_errors:not_acceptable(<<"en">>, <<"Invalid stanza id provided">>), + {error, invalid_stanza_id, IQ#iq{type = error, sub_el = [Text]}}; +return_error_iq(IQ, item_not_found) -> + {error, item_not_found, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:item_not_found()]}}; +return_error_iq(IQ, not_implemented) -> + {error, not_implemented, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:feature_not_implemented()]}}; +return_error_iq(IQ, Reason) -> + {error, Reason, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:internal_server_error()]}}. + +return_message_form_iq(HostType, IQ) -> + IQ#iq{type = result, sub_el = [message_form(?MODULE, HostType, IQ#iq.xmlns)]}. + +report_issue({Reason, {stacktrace, Stacktrace}}, Issue, ArcJID, IQ) -> + report_issue(Reason, Stacktrace, Issue, ArcJID, IQ); +report_issue(Reason, Issue, ArcJID, IQ) -> + report_issue(Reason, [], Issue, ArcJID, IQ). + +report_issue(invalid_stanza_id, _Stacktrace, _Issue, _ArcJID, _IQ) -> + expected; +report_issue(item_not_found, _Stacktrace, _Issue, _ArcJID, _IQ) -> + expected; +report_issue(not_implemented, _Stacktrace, _Issue, _ArcJID, _IQ) -> + expected; +report_issue(timeout, _Stacktrace, _Issue, _ArcJID, _IQ) -> + expected; +report_issue(Reason, Stacktrace, Issue, #jid{lserver=LServer, luser=LUser}, IQ) -> + ?LOG_ERROR(#{what => mam_error, + issue => Issue, server => LServer, user => LUser, + reason => Reason, iq => IQ, stacktrace => Stacktrace}). + +-spec is_archivable_message(HostType :: host_type(), + Dir :: incoming | outgoing, + Packet :: exml:element()) -> boolean(). +is_archivable_message(HostType, Dir, Packet) -> + M = mod_mam_params:is_archivable_message_module(?MODULE, HostType), + ArchiveChatMarkers = mod_mam_params:archive_chat_markers(?MODULE, HostType), + erlang:apply(M, is_archivable_message, [?MODULE, Dir, Packet, ArchiveChatMarkers]). + +-spec hooks(jid:lserver()) -> [ejabberd_hooks:hook()]. +hooks(HostType) -> + [{disco_local_features, HostType, ?MODULE, disco_local_features, 99}, + {user_send_packet, HostType, ?MODULE, user_send_packet, 60}, + {rest_user_send_packet, HostType, ?MODULE, user_send_packet, 60}, + {filter_local_packet, HostType, ?MODULE, filter_packet, 60}, + {remove_user, HostType, ?MODULE, remove_user, 50}, + {anonymous_purge_hook, HostType, ?MODULE, remove_user, 50}, + {amp_determine_strategy, HostType, ?MODULE, determine_amp_strategy, 20}, + {sm_filter_offline_message, HostType, ?MODULE, sm_filter_offline_message, 50}, + {get_personal_data, HostType, ?MODULE, get_personal_data, 50} + | mongoose_metrics_mam_hooks:get_mam_hooks(HostType)]. + +add_iq_handlers(HostType, Opts) -> + Component = ejabberd_sm, + %% `parallel' is the only one recommended here. + ExecutionType = gen_mod:get_opt(iqdisc, Opts, parallel), + IQHandlerFn = fun ?MODULE:process_mam_iq/5, + Extra = #{}, + [gen_iq_handler:add_iq_handler_for_domain(HostType, Namespace, + Component, IQHandlerFn, + Extra, ExecutionType) + || Namespace <- [?NS_MAM_04, ?NS_MAM_06]], + ok. + +remove_iq_handlers(HostType) -> + Component = ejabberd_sm, + [gen_iq_handler:remove_iq_handler_for_domain(HostType, Namespace, Component) + || Namespace <- [?NS_MAM_04, ?NS_MAM_06]], + ok. + +ensure_metrics(HostType) -> + mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, lookup], histogram), + mongoose_metrics:ensure_metric(HostType, [HostType, modMamLookups, simple], spiral), + mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, archive], histogram), + lists:foreach(fun(Name) -> + mongoose_metrics:ensure_metric(HostType, Name, spiral) + end, + spirals()). + +spirals() -> + [modMamPrefsSets, + modMamPrefsGets, + modMamArchiveRemoved, + modMamLookups, + modMamForwarded, + modMamArchived, + modMamFlushed, + modMamDropped, + modMamDroppedIQ]. diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 310431a7486..f72d05c8ecd 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -85,7 +85,7 @@ supported_features() -> ejabberd_gen_mam_archive:mam_pm_gdpr_data(). get_mam_pm_gdpr_data(Acc, HostType, #jid{luser = LUser, lserver = LServer} = ArcJID) -> - case mod_mam:archive_id(LServer, LUser) of + case mod_mam_pm:archive_id(LServer, LUser) of undefined -> Acc; ArcID -> @@ -99,7 +99,7 @@ uniform_to_gdpr(#{id := MessID, jid := RemoteJID, packet := Packet}) -> {integer_to_binary(MessID), jid:to_binary(RemoteJID), exml:to_binary(Packet)}. %% ---------------------------------------------------------------------- -%% Add hooks for mod_mam +%% Add hooks for mod_mam_pm -spec start_hooks(host_type()) -> ok. start_hooks(HostType) -> @@ -195,8 +195,8 @@ env_vars(HostType, ArcJID) -> column_to_id_fn => fun column_to_id/1, lookup_fn => fun lookup_query/5, decode_row_fn => fun row_to_uniform_format/2, - has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, HostType), - has_full_text_search => mod_mam_utils:has_full_text_search(mod_mam, HostType), + has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam_pm, HostType), + has_full_text_search => mod_mam_utils:has_full_text_search(mod_mam_pm, HostType), db_jid_codec => db_jid_codec(HostType, ?MODULE), db_message_codec => db_message_codec(HostType, ?MODULE)}. diff --git a/src/mam/mod_mam_rdbms_arch_async.erl b/src/mam/mod_mam_rdbms_arch_async.erl index d164b6d3980..dd765cfcec4 100644 --- a/src/mam/mod_mam_rdbms_arch_async.erl +++ b/src/mam/mod_mam_rdbms_arch_async.erl @@ -45,7 +45,7 @@ supported_features() -> [dynamic_domains]. %%% internal callbacks --spec make_pool_opts(mod_mam_meta:mam_type(), gen_mod:module_opts()) -> +-spec make_pool_opts(mod_mam:mam_type(), gen_mod:module_opts()) -> {mongoose_async_pools:pool_opts(), mongoose_async_pools:pool_extra()}. make_pool_opts(Type, Opts) -> Extra = add_batch_name(Type, Opts), diff --git a/src/mam/mod_mam_riak_timed_arch_yz.erl b/src/mam/mod_mam_riak_timed_arch_yz.erl index 0838c683457..d4dae30b25e 100644 --- a/src/mam/mod_mam_riak_timed_arch_yz.erl +++ b/src/mam/mod_mam_riak_timed_arch_yz.erl @@ -221,7 +221,7 @@ archive_message(Host, MessID, LocJID, RemJID, SrcJID, OwnerJID, Packet, Type) -> create_obj(Host, MsgId, SourceJID, BareOwnerJID, Packet, Type) -> ModMAM = case Type of - pm -> mod_mam; + pm -> mod_mam_pm; muc -> mod_mam_muc end, BodyChars = mod_mam_utils:packet_to_search_body(ModMAM, Host, Packet), diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index d73f4287771..0340a1a96c0 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -734,7 +734,7 @@ find_field([], _Name) -> field_to_value(FieldEl) -> exml_query:path(FieldEl, [{element, <<"value">>}, cdata], <<>>). --spec message_form(Mod :: mod_mam | mod_mam_muc, +-spec message_form(Mod :: mod_mam_pm | mod_mam_muc, HostType :: mongooseim:host_type(), binary()) -> exml:element(). message_form(Module, HostType, MamNs) -> @@ -801,7 +801,7 @@ normalize_search_text(Text, WordSeparator) -> Re2 = re:replace(Re1, "\s+", unicode:characters_to_list(WordSeparator), ReOpts), unicode:characters_to_binary(Re2). --spec packet_to_search_body(Module :: mod_mam | mod_mam_muc, +-spec packet_to_search_body(Module :: mod_mam_pm | mod_mam_muc, HostType :: mongooseim:host_type(), Packet :: exml:element()) -> binary(). packet_to_search_body(Module, HostType, Packet) -> @@ -816,14 +816,14 @@ packet_to_search_body(true, Packet) -> packet_to_search_body(false, _Packet) -> <<>>. --spec has_full_text_search(Module :: mod_mam | mod_mam_muc, +-spec has_full_text_search(Module :: mod_mam_pm | mod_mam_muc, HostType :: mongooseim:host_type()) -> boolean(). has_full_text_search(Module, HostType) -> gen_mod:get_module_opt(HostType, Module, full_text_search). %% Message retraction --spec has_message_retraction(Module :: mod_mam | mod_mam_muc, +-spec has_message_retraction(Module :: mod_mam_pm | mod_mam_muc, HostType :: mongooseim:host_type()) -> boolean(). has_message_retraction(Module, HostType) -> gen_mod:get_module_opt(HostType, Module, message_retraction). diff --git a/src/metrics/mongoose_metrics_mam_hooks.erl b/src/metrics/mongoose_metrics_mam_hooks.erl index 0a389f4da6d..870910db12f 100644 --- a/src/metrics/mongoose_metrics_mam_hooks.erl +++ b/src/metrics/mongoose_metrics_mam_hooks.erl @@ -40,7 +40,7 @@ %% Implementation %%------------------- -%% @doc Here will be declared which hooks should be registered when mod_mam is enabled. +%% @doc Here will be declared which hooks should be registered when mod_mam_pm is enabled. -spec get_mam_hooks(_) -> [ejabberd_hooks:hook(), ...]. get_mam_hooks(Host) -> [ diff --git a/src/mod_commands.erl b/src/mod_commands.erl index 856ab44b25c..fae96ef53df 100644 --- a/src/mod_commands.erl +++ b/src/mod_commands.erl @@ -495,7 +495,7 @@ lookup_recent_messages(ArcJID, With, Before, Limit) when is_binary(With) -> lookup_recent_messages(ArcJID, WithJID, Before, Limit) -> #jid{luser = LUser, lserver = LServer} = ArcJID, {ok, HostType} = mongoose_domain_api:get_domain_host_type(LServer), - Params = #{archive_id => mod_mam:archive_id(LServer, LUser), + Params = #{archive_id => mod_mam_pm:archive_id(LServer, LUser), owner_jid => ArcJID, borders => undefined, rsm => #rsm_in{direction = before, id = undefined}, % last msgs @@ -508,7 +508,7 @@ lookup_recent_messages(ArcJID, WithJID, Before, Limit) -> limit_passed => false, max_result_limit => 1, is_simple => true}, - R = mod_mam:lookup_messages(HostType, Params), + R = mod_mam_pm:lookup_messages(HostType, Params), {ok, {_, _, L}} = R, L. diff --git a/src/mongoose_client_api/mongoose_client_api_messages.erl b/src/mongoose_client_api/mongoose_client_api_messages.erl index c8d351d2987..d7ae1bdb306 100644 --- a/src/mongoose_client_api/mongoose_client_api_messages.erl +++ b/src/mongoose_client_api/mongoose_client_api_messages.erl @@ -57,13 +57,13 @@ maybe_to_json_with_jid(error, _, Req, State) -> maybe_to_json_with_jid(WithJID, #jid{} = JID, Req, State = #{creds := Creds}) -> HostType = mongoose_credentials:host_type(Creds), Now = os:system_time(microsecond), - ArchiveID = mod_mam:archive_id_int(HostType, JID), + ArchiveID = mod_mam_pm:archive_id_int(HostType, JID), QS = cowboy_req:parse_qs(Req), PageSize = maybe_integer(proplists:get_value(<<"limit">>, QS, <<"50">>)), Before = maybe_integer(proplists:get_value(<<"before">>, QS)), End = maybe_before_to_us(Before, Now), RSM = #rsm_in{direction = before, id = undefined}, - R = mod_mam:lookup_messages(HostType, + R = mod_mam_pm:lookup_messages(HostType, #{archive_id => ArchiveID, owner_jid => JID, rsm => RSM, diff --git a/test/common/config_parser_helper.erl b/test/common/config_parser_helper.erl index 7b8bdb8dc12..8f6a64fc0dc 100644 --- a/test/common/config_parser_helper.erl +++ b/test/common/config_parser_helper.erl @@ -411,12 +411,12 @@ all_modules() -> mod_mam_muc => mod_config(mod_mam_muc, #{archive_chat_markers => true, - async_writer => config([modules, mod_mam_meta, async_writer], + async_writer => config([modules, mod_mam, async_writer], #{enabled => false}), host => {fqdn, <<"muc.example.com">>}, no_stanzaid_element => true}), mod_caps => default_mod_config(mod_caps), - mod_mam_cache_user => (default_config([modules, mod_mam_meta, cache]))#{muc => true, pm => true}, + mod_mam_cache_user => (default_config([modules, mod_mam, cache]))#{muc => true, pm => true}, mod_offline => mod_config(mod_offline, #{backend => riak, riak => #{bucket_type => <<"offline">>}}), @@ -435,7 +435,7 @@ all_modules() -> mod_event_pusher_rabbit => custom_mod_event_pusher_rabbit(), mod_event_pusher_sns => custom_mod_event_pusher_sns(), mod_adhoc => #{iqdisc => one_queue, report_commands_node => true}, - mod_mam_rdbms_arch_async => default_config([modules, mod_mam_meta, async_writer]), + mod_mam_rdbms_arch_async => default_config([modules, mod_mam, async_writer]), mod_keystore => mod_config(mod_keystore, #{keys => #{access_secret => ram, access_psk => {file, "priv/access_psk"}, @@ -476,11 +476,11 @@ all_modules() -> refresh => #{unit => days, value => 13}}, iqdisc => one_queue}, mod_carboncopy => #{iqdisc => no_queue}, - mod_mam => - mod_config(mod_mam, + mod_mam_pm => + mod_config(mod_mam_pm, #{archive_chat_markers => true, archive_groupchats => false, - async_writer => default_config([modules, mod_mam_meta, async_writer]), + async_writer => default_config([modules, mod_mam, async_writer]), full_text_search => false, same_mam_id_for_peers => false, no_stanzaid_element => true}), @@ -573,11 +573,11 @@ all_modules() -> groupchat => [muclight], remove_on_kicked => true, reset_markers => [<<"displayed">>]}, - mod_mam_meta => - mod_config(mod_mam_meta, + mod_mam => + mod_config(mod_mam, #{archive_chat_markers => true, muc => - #{async_writer => config([modules, mod_mam_meta, async_writer], + #{async_writer => config([modules, mod_mam, async_writer], #{enabled => false}), db_message_format => mam_message_xml, host => {fqdn, <<"muc.example.com">>}, @@ -901,13 +901,13 @@ default_mod_config(mod_keystore) -> #{ram_key_size => 2048, keys => #{}}; default_mod_config(mod_last) -> #{iqdisc => one_queue, backend => mnesia}; +default_mod_config(mod_mam_pm) -> + maps:merge(common_mam_config(), default_config([modules, mod_mam, pm])); default_mod_config(mod_mam) -> - maps:merge(common_mam_config(), default_config([modules, mod_mam_meta, pm])); -default_mod_config(mod_mam_meta) -> (common_mam_config())#{backend => rdbms, cache_users => true, - cache => default_config([modules, mod_mam_meta, cache])}; + cache => default_config([modules, mod_mam, cache])}; default_mod_config(mod_mam_muc) -> - maps:merge(common_mam_config(), default_config([modules, mod_mam_meta, muc])); + maps:merge(common_mam_config(), default_config([modules, mod_mam, muc])); default_mod_config(mod_mam_rdbms_arch) -> #{no_writer => false, db_message_format => mam_message_compressed_eterm, @@ -1173,17 +1173,17 @@ default_config([modules, mod_privacy, riak]) -> #{defaults_bucket_type => <<"privacy_defaults">>, names_bucket_type => <<"privacy_lists_names">>, bucket_type => <<"privacy_lists">>}; -default_config([modules, mod_mam_meta, pm]) -> +default_config([modules, mod_mam, pm]) -> #{archive_groupchats => false, same_mam_id_for_peers => false}; -default_config([modules, mod_mam_meta, muc]) -> +default_config([modules, mod_mam, muc]) -> #{host => {prefix, <<"conference.">>}}; -default_config([modules, mod_mam_meta, cache]) -> +default_config([modules, mod_mam, cache]) -> #{module => internal, strategy => fifo, time_to_live => 480, number_of_segments => 3}; -default_config([modules, mod_mam_meta, async_writer]) -> +default_config([modules, mod_mam, async_writer]) -> #{batch_size => 30, enabled => true, flush_interval => 2000, pool_size => 4 * erlang:system_info(schedulers_online)}; -default_config([modules, mod_mam_meta, riak]) -> +default_config([modules, mod_mam, riak]) -> #{bucket_type => <<"mam_yz">>, search_index => <<"mam">>}; default_config([modules, mod_muc_light, cache_affs]) -> #{module => internal, strategy => fifo, @@ -1292,7 +1292,7 @@ common_mam_config() -> full_text_search => true, default_result_limit => 50, max_result_limit => 50, - async_writer => default_config([modules, mod_mam_meta, async_writer])}. + async_writer => default_config([modules, mod_mam, async_writer])}. mod_event_pusher_http_handler() -> #{pool_name => http_pool, diff --git a/test/config_parser_SUITE.erl b/test/config_parser_SUITE.erl index dfe223249d0..07e732bbcf7 100644 --- a/test/config_parser_SUITE.erl +++ b/test/config_parser_SUITE.erl @@ -197,10 +197,10 @@ groups() -> mod_keystore, mod_keystore_keys, mod_last, - mod_mam_meta, - mod_mam_meta_riak, - mod_mam_meta_pm, - mod_mam_meta_muc, + mod_mam, + mod_mam_riak, + mod_mam_pm, + mod_mam_muc, mod_muc, mod_muc_default_room, mod_muc_default_room_affiliations, @@ -1970,19 +1970,19 @@ mod_last(_Config) -> ?errh(T(#{<<"backend">> => <<"frontend">>})), ?errh(T(#{<<"riak">> => #{<<"bucket_type">> => 1}})). -mod_mam_meta(_Config) -> - check_module_defaults(mod_mam_meta), - T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam_meta">> => Opts}} end, - P = [modules, mod_mam_meta], +mod_mam(_Config) -> + check_module_defaults(mod_mam), + T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam">> => Opts}} end, + P = [modules, mod_mam], test_cache_config(P ++ [cache], fun(Opts) -> T(#{<<"cache">> => Opts}) end), - test_mod_mam_meta(P, T). + test_mod_mam(P, T). -mod_mam_meta_riak(_Config) -> +mod_mam_riak(_Config) -> T = fun(Opts) -> - #{<<"modules">> => #{<<"mod_mam_meta">> => Opts#{<<"backend">> => <<"riak">>}}} + #{<<"modules">> => #{<<"mod_mam">> => Opts#{<<"backend">> => <<"riak">>}}} end, - P = [modules, mod_mam_meta, riak], - ?cfgh(P, default_config([modules, mod_mam_meta, riak]), T(#{})), + P = [modules, mod_mam, riak], + ?cfgh(P, default_config([modules, mod_mam, riak]), T(#{})), ?cfgh(P ++ [bucket_type], <<"mam_bucket">>, T(#{<<"riak">> => #{<<"bucket_type">> => <<"mam_bucket">>}})), ?cfgh(P ++ [search_index], <<"mam_index">>, @@ -1990,10 +1990,10 @@ mod_mam_meta_riak(_Config) -> ?errh(T(#{<<"riak">> => #{<<"bucket_type">> => <<>>}})), ?errh(T(#{<<"riak">> => #{<<"search_index">> => <<>>}})). -mod_mam_meta_pm(_Config) -> - T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam_meta">> => #{<<"pm">> => Opts}}} end, - P = [modules, mod_mam_meta, pm], - test_mod_mam_meta(P, T), +mod_mam_pm(_Config) -> + T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam">> => #{<<"pm">> => Opts}}} end, + P = [modules, mod_mam, pm], + test_mod_mam(P, T), ?cfgh(P, default_config(P), T(#{})), ?cfgh(P ++ [archive_groupchats], true, T(#{<<"archive_groupchats">> => true})), ?cfgh(P ++ [same_mam_id_for_peers], true, T(#{<<"same_mam_id_for_peers">> => true})), @@ -2001,10 +2001,10 @@ mod_mam_meta_pm(_Config) -> ?errh(T(#{<<"archive_groupchats">> => <<"not really">>})), ?errh(T(#{<<"same_mam_id_for_peers">> => <<"not really">>})). -mod_mam_meta_muc(_Config) -> - T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam_meta">> => #{<<"muc">> => Opts}}} end, - P = [modules, mod_mam_meta, muc], - test_mod_mam_meta(P, T), +mod_mam_muc(_Config) -> + T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam">> => #{<<"muc">> => Opts}}} end, + P = [modules, mod_mam, muc], + test_mod_mam(P, T), ?cfgh(P, default_config(P), T(#{})), ?cfgh(P ++ [host], {prefix, <<"muc.">>}, T(#{<<"host">> => <<"muc.@HOST@">>})), ?cfgh(P ++ [host], {fqdn, <<"muc.test">>}, T(#{<<"host">> => <<"muc.test">>})), @@ -2014,7 +2014,7 @@ mod_mam_meta_muc(_Config) -> ?errh(T(#{<<"archive_groupchats">> => true})), % pm-only ?errh(T(#{<<"same_mam_id_for_peers">> => true})). % pm-only -test_mod_mam_meta(P, T) -> +test_mod_mam(P, T) -> test_async_writer(P, T), ?cfgh(P ++ [backend], rdbms, T(#{<<"backend">> => <<"rdbms">>})), diff --git a/test/config_parser_SUITE_data/modules.toml b/test/config_parser_SUITE_data/modules.toml index 874c8e28564..f11a1a4d6a5 100644 --- a/test/config_parser_SUITE_data/modules.toml +++ b/test/config_parser_SUITE_data/modules.toml @@ -153,7 +153,7 @@ backend = "mnesia" iqdisc = {"type" = "queues", "workers" = 10} -[modules.mod_mam_meta] +[modules.mod_mam] backend = "rdbms" no_stanzaid_element = true is_archivable_message = "mod_mam_utils" diff --git a/test/mam_misc_SUITE.erl b/test/mam_misc_SUITE.erl index b0b02612dde..0cecfafc8e0 100644 --- a/test/mam_misc_SUITE.erl +++ b/test/mam_misc_SUITE.erl @@ -112,7 +112,7 @@ must_be_accepted(_) -> %% Generators gen_modules() -> - oneof([mod_mam, mod_inbox]). + oneof([mod_mam_pm, mod_inbox]). gen_directions() -> oneof([outgoing, incoming]). diff --git a/test/mod_mam_meta_SUITE.erl b/test/mod_mam_SUITE.erl similarity index 70% rename from test/mod_mam_meta_SUITE.erl rename to test/mod_mam_SUITE.erl index eead670d3b6..36136ac8772 100644 --- a/test/mod_mam_meta_SUITE.erl +++ b/test/mod_mam_SUITE.erl @@ -1,4 +1,4 @@ --module(mod_mam_meta_SUITE). +-module(mod_mam_SUITE). -compile([export_all, nowarn_export_all]). -include_lib("eunit/include/eunit.hrl"). @@ -27,50 +27,50 @@ end_per_testcase(_CaseName, Config) -> Config. overrides_general_options(_Config) -> Deps = deps(#{backend => rdbms, - pm => config([modules, mod_mam_meta, pm], #{backend => cassandra}), - muc => default_config([modules, mod_mam_meta, muc]) + pm => config([modules, mod_mam, pm], #{backend => cassandra}), + muc => default_config([modules, mod_mam, muc]) }), ?assert(lists:keymember(mod_mam_cassandra_arch, 1, Deps)), ?assert(lists:keymember(mod_mam_muc_rdbms_arch, 1, Deps)), ?assertNot(lists:keymember(mod_mam_rdbms_arch, 1, Deps)). sets_rdbms_as_default_backend(_Config) -> - Deps = deps(#{pm => default_config([modules, mod_mam_meta, pm])}), + Deps = deps(#{pm => default_config([modules, mod_mam, pm])}), ?assert(lists:keymember(mod_mam_rdbms_arch, 1, Deps)). handles_only_pm(_Config) -> - Deps = deps(#{pm => default_config([modules, mod_mam_meta, pm])}), - ?assert(lists:keymember(mod_mam, 1, Deps)), + Deps = deps(#{pm => default_config([modules, mod_mam, pm])}), + ?assert(lists:keymember(mod_mam_pm, 1, Deps)), ?assertNot(lists:keymember(mod_mam_muc, 1, Deps)). handles_only_muc(_Config) -> - Deps = deps(#{muc => default_config([modules, mod_mam_meta, muc])}), - ?assertNot(lists:keymember(mod_mam, 1, Deps)), + Deps = deps(#{muc => default_config([modules, mod_mam, muc])}), + ?assertNot(lists:keymember(mod_mam_pm, 1, Deps)), ?assert(lists:keymember(mod_mam_muc, 1, Deps)). disables_sync_writer_on_async_writer(_Config) -> - PM = default_config([modules, mod_mam_meta, pm]), + PM = default_config([modules, mod_mam, pm]), Deps = deps(#{pm => PM}), check_equal_opts(mod_mam_rdbms_arch, mod_config(mod_mam_rdbms_arch, #{no_writer => true}), Deps). disables_sync_muc_writer_on_async_writer(_Config) -> - MUC = default_config([modules, mod_mam_meta, muc]), + MUC = default_config([modules, mod_mam, muc]), Deps = deps(#{muc => MUC}), check_equal_opts(mod_mam_muc_rdbms_arch, mod_config(mod_mam_muc_rdbms_arch, #{no_writer => true}), Deps). produces_valid_configurations(_Config) -> - AsyncOpts = default_config([modules, mod_mam_meta, async_writer]), + AsyncOpts = default_config([modules, mod_mam, async_writer]), PMCoreOpts = #{archive_groupchats => true, async_writer => AsyncOpts#{enabled => false}}, - PM = config([modules, mod_mam_meta, pm], PMCoreOpts#{user_prefs_store => rdbms}), + PM = config([modules, mod_mam, pm], PMCoreOpts#{user_prefs_store => rdbms}), MUCCoreOpts = #{host => <<"host">>}, MUCArchOpts = #{db_message_format => mam_message_xml}, - MUC = config([modules, mod_mam_meta, muc], + MUC = config([modules, mod_mam, muc], maps:merge(MUCCoreOpts, MUCArchOpts#{user_prefs_store => mnesia})), Deps = deps(#{pm => PM, muc => MUC}), - Cache = default_config([modules, mod_mam_meta, cache]), + Cache = default_config([modules, mod_mam, cache]), - check_equal_opts(mod_mam, mod_config(mod_mam, PMCoreOpts), Deps), + check_equal_opts(mod_mam_pm, mod_config(mod_mam_pm, PMCoreOpts), Deps), check_equal_opts(mod_mam_muc, mod_config(mod_mam_muc, MUCCoreOpts), Deps), check_equal_opts(mod_mam_rdbms_arch, default_mod_config(mod_mam_rdbms_arch), Deps), check_equal_opts(mod_mam_muc_rdbms_arch, mod_config(mod_mam_muc_rdbms_arch, @@ -82,25 +82,25 @@ produces_valid_configurations(_Config) -> check_equal_opts(mod_mam_muc_rdbms_arch_async, AsyncOpts, Deps). handles_riak_config(_Config) -> - PM = config([modules, mod_mam_meta, pm], #{user_prefs_store => mnesia}), - MUC = default_config([modules, mod_mam_meta, muc]), + PM = config([modules, mod_mam, pm], #{user_prefs_store => mnesia}), + MUC = default_config([modules, mod_mam, muc]), Deps = deps(#{backend => riak, db_message_format => some_format, - pm => config([modules, mod_mam_meta, pm], PM), - muc => config([modules, mod_mam_meta, muc], MUC)}), - ?assert(lists:keymember(mod_mam, 1, Deps)), + pm => config([modules, mod_mam, pm], PM), + muc => config([modules, mod_mam, muc], MUC)}), + ?assert(lists:keymember(mod_mam_pm, 1, Deps)), ?assert(lists:keymember(mod_mam_muc, 1, Deps)), check_equal_opts(mod_mam_riak_timed_arch_yz, #{pm => true, muc => true, db_message_format => some_format}, Deps), check_equal_opts(mod_mam_mnesia_prefs, #{pm => true}, Deps). handles_cassandra_config(_Config) -> - PM = config([modules, mod_mam_meta, pm], #{user_prefs_store => cassandra, + PM = config([modules, mod_mam, pm], #{user_prefs_store => cassandra, db_message_format => some_format}), - MUC = config([modules, mod_mam_meta, muc], #{user_prefs_store => mnesia}), + MUC = config([modules, mod_mam, muc], #{user_prefs_store => mnesia}), Deps = deps(#{backend => cassandra, - pm => config([modules, mod_mam_meta, pm], PM), - muc => config([modules, mod_mam_meta, muc], MUC)}), + pm => config([modules, mod_mam, pm], PM), + muc => config([modules, mod_mam, muc], MUC)}), check_equal_opts(mod_mam_mnesia_prefs, #{muc => true}, Deps), check_equal_opts(mod_mam_cassandra_prefs, #{pm => true}, Deps), @@ -109,10 +109,10 @@ handles_cassandra_config(_Config) -> example_muc_only_no_pref_good_performance(_Config) -> MUCOpts = #{host => {prefix, "muc."}}, - MUC = config([modules, mod_mam_meta, muc], MUCOpts), + MUC = config([modules, mod_mam, muc], MUCOpts), Deps = deps(#{muc => MUC}), - AsyncOpts = default_config([modules, mod_mam_meta, async_writer]), - Cache = default_config([modules, mod_mam_meta, cache]), + AsyncOpts = default_config([modules, mod_mam, async_writer]), + Cache = default_config([modules, mod_mam, cache]), check_equal_deps( [{mod_mam_rdbms_user, #{muc => true, pm => true}}, @@ -123,10 +123,10 @@ example_muc_only_no_pref_good_performance(_Config) -> ], Deps). example_pm_only_good_performance(_Config) -> - PM = default_config([modules, mod_mam_meta, pm]), + PM = default_config([modules, mod_mam, pm]), Deps = deps(#{pm => PM, user_prefs_store => mnesia}), - AsyncOpts = default_config([modules, mod_mam_meta, async_writer]), - Cache = default_config([modules, mod_mam_meta, cache]), + AsyncOpts = default_config([modules, mod_mam, async_writer]), + Cache = default_config([modules, mod_mam, cache]), check_equal_deps( [{mod_mam_rdbms_user, #{pm => true}}, @@ -134,7 +134,7 @@ example_pm_only_good_performance(_Config) -> {mod_mam_mnesia_prefs, #{pm => true}}, {mod_mam_rdbms_arch, mod_config(mod_mam_rdbms_arch, #{no_writer => true})}, {mod_mam_rdbms_arch_async, AsyncOpts}, - {mod_mam, default_mod_config(mod_mam)} + {mod_mam_pm, default_mod_config(mod_mam_pm)} ], Deps). %% Helpers @@ -147,4 +147,4 @@ check_equal_opts(Mod, Opts, Deps) -> ?assertEqual(Opts, ActualOpts). deps(Opts) -> - mod_mam_meta:deps(<<"host">>, mod_config(mod_mam_meta, Opts)). + mod_mam:deps(<<"host">>, mod_config(mod_mam, Opts)).