Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MAM config in maps #3559

Merged
merged 10 commits into from
Mar 2, 2022
Prev Previous commit
Next Next commit
Update config tests for MAM modules
chrzaszcz committed Mar 1, 2022
commit 84603d3219117e3c937291f2eac7a9268c8e0649
104 changes: 76 additions & 28 deletions test/common/config_parser_helper.erl
Original file line number Diff line number Diff line change
@@ -389,16 +389,17 @@ options("s2s_only") ->
{{s2s, <<"localhost">>}, custom_s2s()}].

all_modules() ->
#{mod_mam_rdbms_user => [{muc, true}, {pm, true}],
#{mod_mam_rdbms_user => #{muc => true, pm => true},
mod_event_pusher_hook_translator => [],
mod_mam_muc =>
[{archive_chat_markers, true},
{async_writer, [{enabled, false}]},
{full_text_search, true},
{host, {fqdn, <<"muc.example.com">>}},
{is_archivable_message, mod_mam_utils}],
mod_config(mod_mam_muc,
#{archive_chat_markers => true,
async_writer => config([modules, mod_mam_meta, async_writer],
#{enabled => false}),
host => {fqdn, <<"muc.example.com">>},
no_stanzaid_element => true}),
mod_caps => [{cache_life_time, 86}, {cache_size, 1000}],
mod_mam_cache_user => [{muc, true}, {pm, true}],
mod_mam_cache_user => #{cache => [], muc => true, pm => true},
mod_offline =>
[{access_max_user_messages, max_user_offline_messages},
{backend, riak},
@@ -451,7 +452,7 @@ all_modules() ->
{virtual_pubsub_hosts, [{fqdn, <<"host1">>}, {fqdn, <<"host2">>}]},
{wpool, [{workers, 200}]}],
mod_adhoc => #{iqdisc => one_queue, report_commands_node => true},
mod_mam_rdbms_arch_async => [{pm, []}],
mod_mam_rdbms_arch_async => #{pm => default_config([modules, mod_mam_meta, async_writer])},
mod_keystore =>
[{keys,
[{access_secret, ram},
@@ -484,10 +485,13 @@ all_modules() ->
iqdisc => one_queue},
mod_carboncopy => [{iqdisc, no_queue}],
mod_mam =>
[{archive_chat_markers, true},
{full_text_search, false},
{is_archivable_message, mod_mam_utils},
{no_stanzaid_element, true}],
mod_config(mod_mam,
#{archive_chat_markers => true,
archive_groupchats => false,
async_writer => default_config([modules, mod_mam_meta, async_writer]),
full_text_search => false,
same_mam_id_for_peers => false,
no_stanzaid_element => true}),
mod_disco =>
mod_config(mod_disco,
#{extra_domains => [<<"some_domain">>, <<"another_domain">>],
@@ -507,14 +511,14 @@ all_modules() ->
{ldap_rfilter, "(objectClass=inetOrgPerson)"},
{ldap_user_cache_validity, 1},
{ldap_userdesc, "cn"}],
mod_mam_mnesia_prefs => [{muc, true}],
mod_mam_mnesia_prefs => #{muc => true},
mod_jingle_sip =>
[{listen_port, 5600},
{local_host, "localhost"},
{proxy_host, "localhost"},
{proxy_port, 5600},
{sdp_origin, "127.0.0.1"}],
mod_mam_rdbms_prefs => [{pm, true}],
mod_mam_rdbms_prefs => #{pm => true},
mod_extdisco =>
#{iqdisc => one_queue,
service => [#{host => <<"stun1">>, password => <<"password">>,
@@ -584,23 +588,27 @@ all_modules() ->
remove_on_kicked => true,
reset_markers => [<<"displayed">>]},
mod_mam_meta =>
[{archive_chat_markers, true},
{backend, rdbms},
{full_text_search, true},
{is_archivable_message, mod_mam_utils},
{muc,
[{async_writer, [{enabled, false}]},
{host, {fqdn, <<"muc.example.com">>}},
{rdbms_message_format, simple},
{user_prefs_store, mnesia}]},
{no_stanzaid_element, true},
{pm, [{full_text_search, false}, {user_prefs_store, rdbms}]}],
mod_config(mod_mam_meta,
#{archive_chat_markers => true,
muc =>
#{async_writer => config([modules, mod_mam_meta, async_writer],
#{enabled => false}),
db_message_format => mam_message_xml,
host => {fqdn, <<"muc.example.com">>},
user_prefs_store => mnesia},
no_stanzaid_element => true,
pm =>
#{archive_groupchats => false,
full_text_search => false,
same_mam_id_for_peers => false,
user_prefs_store => rdbms}}),
mod_register =>
[{access, all},
{password_strength, 32},
{registration_watchers, [<<"JID1">>, <<"JID2">>]},
{welcome_message, {"Subject", "Body"}}],
mod_mam_rdbms_arch => [{no_writer, true}, {pm, true}],
mod_mam_rdbms_arch =>
mod_config(mod_mam_rdbms_arch, #{no_writer => true}),
mod_event_pusher_rabbit =>
[{chat_msg_exchange,
[{name, <<"chat_msg">>},
@@ -637,7 +645,7 @@ all_modules() ->
matches => 1,
search => true}),
mod_mam_muc_rdbms_arch =>
[{muc, true}, {db_jid_format, mam_jid_rfc}, {db_message_format, mam_message_xml}],
mod_config(mod_mam_muc_rdbms_arch, #{db_message_format => mam_message_xml}),
mod_stream_management =>
[{ack_freq, 1},
{buffer_max, 30},
@@ -889,4 +897,44 @@ default_mod_config(mod_vcard) ->
ldap_search_operator => 'and',
ldap_binary_search_fields => []};
default_mod_config(mod_version) ->
#{iqdisc => no_queue, os_info => false}.
#{iqdisc => no_queue, os_info => false};
default_mod_config(mod_mam_meta) ->
(common_mam_config())#{backend => rdbms, cache_users => true, cache => []};
default_mod_config(mod_mam) ->
maps:merge(common_mam_config(), default_config([modules, mod_mam_meta, pm]));
default_mod_config(mod_mam_muc) ->
maps:merge(common_mam_config(), default_config([modules, mod_mam_meta, muc]));
default_mod_config(mod_mam_rdbms_arch) ->
#{no_writer => false,
db_message_format => mam_message_compressed_eterm,
db_jid_format => mam_jid_mini};
default_mod_config(mod_mam_muc_rdbms_arch) ->
#{no_writer => false,
db_message_format => mam_message_compressed_eterm,
db_jid_format => mam_jid_rfc}.

default_config([modules, M]) ->
default_mod_config(M);
default_config([modules, mod_mam_meta, pm]) ->
#{archive_groupchats => false, same_mam_id_for_peers => false};
default_config([modules, mod_mam_meta, muc]) ->
#{host => {prefix, <<"conference.">>}};
default_config([modules, mod_mam_meta, async_writer]) ->
#{batch_size => 30, enabled => true, flush_interval => 2000,
pool_size => 4 * erlang:system_info(schedulers_online)};
default_config([modules, mod_mam_meta, riak]) ->
#{bucket_type => <<"mam_yz">>, search_index => <<"mam">>}.

common_mam_config() ->
#{no_stanzaid_element => false,
is_archivable_message => mod_mam_utils,
send_message => mod_mam_utils,
archive_chat_markers => false,
message_retraction => true,
full_text_search => true,
default_result_limit => 50,
max_result_limit => 50,
async_writer => default_config([modules, mod_mam_meta, async_writer])}.

config(Path, Opts) ->
maps:merge(default_config(Path), Opts).
123 changes: 65 additions & 58 deletions test/config_parser_SUITE.erl
Original file line number Diff line number Diff line change
@@ -34,6 +34,9 @@
end).

-import(mongoose_config_parser_toml, [extract_errors/1]).
-import(config_parser_helper, [merge_with_default_pool_config/1, default_s2s/0,
mod_config/2, default_mod_config/1,
config/2, default_config/1]).

-type key_prefix() :: top_level_key_prefix() | key_path_prefix().
-type top_level_key_prefix() :: atom().
@@ -218,6 +221,7 @@ groups() ->
mod_keystore_keys,
mod_last,
mod_mam_meta,
mod_mam_meta_riak,
mod_mam_meta_pm,
mod_mam_meta_muc,
mod_muc,
@@ -1467,7 +1471,7 @@ access_merge_host_and_global(_Config) ->
%% tests: s2s

s2s_host_config(_Config) ->
DefaultS2S = config_parser_helper:default_s2s(),
DefaultS2S = default_s2s(),
EmptyHostConfig = host_config(#{<<"s2s">> => #{}}),
?cfg(host_key(s2s), DefaultS2S,
EmptyHostConfig#{<<"s2s">> => #{<<"dns">> => #{<<"timeout">> => 5}}}),
@@ -2128,87 +2132,92 @@ mod_last(_Config) ->
?errh(T(#{<<"riak">> => #{<<"bucket_type">> => 1}})).

mod_mam_meta(_Config) ->
check_module_defaults(mod_mam_meta),
T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam_meta">> => Opts}} end,
M = fun(Cfg) -> modopts(mod_mam_meta, Cfg) end,
test_mod_mam_meta(T, M),
?cfgh(M([{bucket_type, <<"mam_bucket">>}]),
P = [modules, mod_mam_meta],
test_segmented_cache_config(<<"cache">>, cache, T,
fun([{cache, Opts}]) -> [{P ++ [cache], Opts}] end),
test_mod_mam_meta(T, P).

mod_mam_meta_riak(_Config) ->
T = fun(Opts) ->
#{<<"modules">> => #{<<"mod_mam_meta">> => Opts#{<<"backend">> => <<"riak">>}}}
end,
P = [modules, mod_mam_meta, riak],
?cfgh(P, default_config([modules, mod_mam_meta, riak]), T(#{})),
?cfgh(P ++ [bucket_type], <<"mam_bucket">>,
T(#{<<"riak">> => #{<<"bucket_type">> => <<"mam_bucket">>}})),
?cfgh(M([{search_index, <<"mam_index">>}]),
?cfgh(P ++ [search_index], <<"mam_index">>,
T(#{<<"riak">> => #{<<"search_index">> => <<"mam_index">>}})),
?errh(T(#{<<"riak">> => #{<<"bucket_type">> => <<>>}})),
?errh(T(#{<<"riak">> => #{<<"search_index">> => <<>>}})).

mod_mam_meta_pm(_Config) ->
T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam_meta">> => #{<<"pm">> => Opts}}} end,
M = fun(Cfg) -> modopts(mod_mam_meta, [{pm, Cfg}]) end,
test_mod_mam_meta(T, M),
?cfgh(M([{archive_groupchats, true}]),
T(#{<<"archive_groupchats">> => true})),
?cfgh(M([{same_mam_id_for_peers, true}]),
T(#{<<"same_mam_id_for_peers">> => true})),
P = [modules, mod_mam_meta, pm],
test_mod_mam_meta(T, P),
?cfgh(P, default_config(P), T(#{})),
?cfgh(P ++ [archive_groupchats], true, T(#{<<"archive_groupchats">> => true})),
?cfgh(P ++ [same_mam_id_for_peers], true, T(#{<<"same_mam_id_for_peers">> => true})),
?errh(T(#{<<"host">> => <<"muc.@HOST@">>})), % muc-only
?errh(T(#{<<"archive_groupchats">> => <<"not really">>})),
?errh(T(#{<<"same_mam_id_for_peers">> => <<"not really">>})).

mod_mam_meta_muc(_Config) ->
T = fun(Opts) -> #{<<"modules">> => #{<<"mod_mam_meta">> => #{<<"muc">> => Opts}}} end,
M = fun(Cfg) -> modopts(mod_mam_meta, [{muc, Cfg}]) end,
test_mod_mam_meta(T, M),
?cfgh(M([{host, {prefix, <<"muc.">>}}]),
T(#{<<"host">> => <<"muc.@HOST@">>})),
?cfgh(M([{host, {fqdn, <<"muc.test">>}}]),
T(#{<<"host">> => <<"muc.test">>})),
P = [modules, mod_mam_meta, muc],
test_mod_mam_meta(T, P),
?cfgh(P, default_config(P), T(#{})),
?cfgh(P ++ [host], {prefix, <<"muc.">>}, T(#{<<"host">> => <<"muc.@HOST@">>})),
?cfgh(P ++ [host], {fqdn, <<"muc.test">>}, T(#{<<"host">> => <<"muc.test">>})),
?errh(T(#{<<"host">> => <<"is this a host? no.">>})),
?errh(T(#{<<"host">> => [<<"invalid.sub@HOST@">>]})),
?errh(T(#{<<"host">> => [<<"invalid.sub.@HOST@.as.well">>]})),
?errh(T(#{<<"archive_groupchats">> => true})),
?errh(T(#{<<"same_mam_id_for_peers">> => true})).
?errh(T(#{<<"archive_groupchats">> => true})), % pm-only
?errh(T(#{<<"same_mam_id_for_peers">> => true})). % pm-only

test_mod_mam_meta(T, M) ->
test_async_worker(T, M),
test_cache_config(T, M),
?cfgh(M([{backend, rdbms}]),
test_mod_mam_meta(T, P) ->
test_async_writer(T, P),
?cfgh(P ++ [backend], rdbms,
T(#{<<"backend">> => <<"rdbms">>})),
?cfgh(M([{no_stanzaid_element, true}]),
?cfgh(P ++ [no_stanzaid_element], true,
T(#{<<"no_stanzaid_element">> => true})),
?cfgh(M([{is_archivable_message, mod_mam_utils}]),
?cfgh(P ++ [is_archivable_message], mod_mam_utils,
T(#{<<"is_archivable_message">> => <<"mod_mam_utils">>})),
?cfgh(M([{archive_chat_markers, false}]),
T(#{<<"archive_chat_markers">> => false})),
?cfgh(M([{message_retraction, true}]),
T(#{<<"message_retraction">> => true})),
?cfgh(M([{rdbms_message_format, simple}]),
T(#{<<"rdbms_message_format">> => <<"simple">>})),
?cfgh(M([{user_prefs_store, rdbms}]),
?cfgh(P ++ [archive_chat_markers], true,
T(#{<<"archive_chat_markers">> => true})),
?cfgh(P ++ [message_retraction], false,
T(#{<<"message_retraction">> => false})),
?cfgh(P ++ [user_prefs_store], rdbms,
T(#{<<"user_prefs_store">> => <<"rdbms">>})),
?cfgh(M([{full_text_search, false}]),
?cfgh(P ++ [full_text_search], false,
T(#{<<"full_text_search">> => false})),
?cfgh(M([{default_result_limit, 100}]),
?cfgh(P ++ [cache_users], false,
T(#{<<"cache_users">> => false})),
?cfgh(P ++ [default_result_limit], 100,
T(#{<<"default_result_limit">> => 100})),
?cfgh(M([{max_result_limit, 1000}]),
?cfgh(P ++ [max_result_limit], 1000,
T(#{<<"max_result_limit">> => 1000})),
?cfgh(M([{db_jid_format, mam_jid_rfc}]),
?cfgh(P ++ [db_jid_format], mam_jid_rfc,
T(#{<<"db_jid_format">> => <<"mam_jid_rfc">>})),
?cfgh(M([{db_message_format, mam_message_xml}]),
?cfgh(P ++ [db_message_format], mam_message_xml,
T(#{<<"db_message_format">> => <<"mam_message_xml">>})),
?cfgh(M([{simple, false}]),
T(#{<<"simple">> => false})),
?cfgh(M([{extra_fin_element, mod_mam_utils}]),
?cfgh(P ++ [extra_fin_element], mod_mam_utils,
T(#{<<"extra_fin_element">> => <<"mod_mam_utils">>})),
?cfgh(M([{extra_lookup_params, mod_mam_utils}]),
?cfgh(P ++ [extra_lookup_params], mod_mam_utils,
T(#{<<"extra_lookup_params">> => <<"mod_mam_utils">>})),
?errh(T(#{<<"backend">> => <<"notepad">>})),
?errh(T(#{<<"no_stanzaid_element">> => <<"true">>})),
?errh(T(#{<<"is_archivable_message">> => <<"mod_mam_fake">>})),
?errh(T(#{<<"archive_chat_markers">> => <<"maybe">>})),
?errh(T(#{<<"message_retraction">> => 1})),
?errh(T(#{<<"rdbms_message_format">> => <<"complex">>})),
?errh(T(#{<<"user_prefs_store">> => <<"textfile">>})),
?errh(T(#{<<"full_text_search">> => <<"disabled">>})),
?errh(T(#{<<"cache_users">> => []})),
?errh(T(#{<<"default_result_limit">> => -1})),
?errh(T(#{<<"max_result_limit">> => -2})),
?errh(T(#{<<"db_jid_format">> => <<"not_a_module">>})),
?errh(T(#{<<"db_message_format">> => <<"not_a_module">>})),
?errh(T(#{<<"simple">> => <<"yes">>})),
?errh(T(#{<<"extra_fin_element">> => <<"bad_module">>})),
?errh(T(#{<<"extra_lookup_params">> => <<"bad_module">>})).

@@ -2237,19 +2246,17 @@ test_segmented_cache_config(NameK, NameV, T, M) ->
?errh(T(#{NameK => #{<<"number_of_segments">> => 0}})),
?errh(T(#{NameK => #{<<"number_of_segments">> => <<"infinity">>}})).

test_async_worker(T, M) ->
?cfgh(M([{async_writer, [{flush_interval, 1500}]}]),
T(#{<<"async_writer">> => #{<<"flush_interval">> => 1500}})),
?cfgh(M([{async_writer, [{batch_size, 1500}]}]),
T(#{<<"async_writer">> => #{<<"batch_size">> => 1500}})),
?cfgh(M([{async_writer, [{pool_size, 1500}]}]),
T(#{<<"async_writer">> => #{<<"pool_size">> => 1500}})),
?cfgh(M([{async_writer, [{enabled, false}]}]),
T(#{<<"async_writer">> => #{<<"enabled">> => false}})),
?errh(T(#{<<"async_writer">> => #{<<"flush_interval">> => -1}})),
?errh(T(#{<<"async_writer">> => #{<<"batch_size">> => -1}})),
?errh(T(#{<<"async_writer">> => #{<<"pool_size">> => -1}})),
?errh(T(#{<<"async_writer">> => #{<<"enabled">> => <<"wrong">>}})).
test_async_writer(ParentT, ParentP) ->
P = ParentP ++ [async_writer],
T = fun(Opts) -> ParentT(#{<<"async_writer">> => Opts}) end,
?cfgh(P ++ [flush_interval], 1500, T(#{<<"flush_interval">> => 1500})),
?cfgh(P ++ [batch_size], 1500, T(#{<<"batch_size">> => 1500})),
?cfgh(P ++ [pool_size], 1500, T(#{<<"pool_size">> => 1500})),
?cfgh(P ++ [enabled], false, T(#{<<"enabled">> => false})),
?errh(T(#{<<"flush_interval">> => -1})),
?errh(T(#{<<"batch_size">> => -1})),
?errh(T(#{<<"pool_size">> => -1})),
?errh(T(#{<<"enabled">> => <<"wrong">>})).

mod_muc(_Config) ->
T = fun(Opts) -> #{<<"modules">> => #{<<"mod_muc">> => Opts}} end,
@@ -3112,7 +3119,7 @@ check_iqdisc_map(Module, RequiredOpts) ->
?errh(iq_disc_generic(Module, RequiredOpts, iqdisc(bad_haha))).

check_module_defaults(Mod) ->
ExpectedCfg = config_parser_helper:default_mod_config(Mod),
ExpectedCfg = default_mod_config(Mod),
?cfgh([modules, Mod], ExpectedCfg, #{<<"modules">> => #{atom_to_binary(Mod) => #{}}}).

modopts(Mod, Opts) ->
@@ -3155,7 +3162,7 @@ auth_raw(Method, Opts) ->
%% helpers for 'pool' tests

pool_config(PoolIn) ->
Pool = config_parser_helper:merge_with_default_pool_config(PoolIn),
Pool = merge_with_default_pool_config(PoolIn),
[{outgoing_pools, [Pool]}].

pool_raw(Type, Tag, Opts) ->
4 changes: 2 additions & 2 deletions test/config_parser_SUITE_data/modules.toml
Original file line number Diff line number Diff line change
@@ -156,9 +156,9 @@
archive_chat_markers = true
full_text_search = true
pm.user_prefs_store = "rdbms"
pm.full_text_search = false
pm.full_text_search = false
muc.host = "muc.example.com"
muc.rdbms_message_format = "simple"
muc.db_message_format = "mam_message_xml"
muc.async_writer.enabled = false
muc.user_prefs_store = "mnesia"