From a475d8f641f0f9e3c4fdc0a669b962af7dee311f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Tue, 31 Aug 2021 16:21:09 +0200 Subject: [PATCH 1/8] Configure mod_vcard for tests with dynamic domains --- big_tests/test.config | 15 +++++---------- rel/files/mongooseim.toml | 2 +- rel/mim1.vars-toml.config | 6 +++++- rel/vars-toml.config.in | 3 +-- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/big_tests/test.config b/big_tests/test.config index b80a83ec38e..93fc58f1d4e 100644 --- a/big_tests/test.config +++ b/big_tests/test.config @@ -259,8 +259,7 @@ {mod_private, "[modules.mod_private] backend = \"rdbms\""}, {mod_offline, " backend = \"rdbms\"\n"}, - {mod_vcard, "[modules.mod_vcard] - backend = \"rdbms\" + {mod_vcard, " backend = \"rdbms\" host = \"vjud.@HOST@\""}, {mod_roster, " backend = \"rdbms\"\n"}]}, {odbc_mssql_mnesia, @@ -280,8 +279,7 @@ {mod_private, "[modules.mod_private] backend = \"rdbms\""}, {mod_offline, " backend = \"rdbms\"\n"}, - {mod_vcard, "[modules.mod_vcard] - backend = \"rdbms\" + {mod_vcard, " backend = \"rdbms\" host = \"vjud.@HOST@\""}, {mod_roster, " backend = \"rdbms\"\n"}]}, {mysql_redis, @@ -311,8 +309,7 @@ {mod_private, "[modules.mod_private] backend = \"rdbms\""}, {mod_offline, " backend = \"rdbms\"\n"}, - {mod_vcard, "[modules.mod_vcard] - backend = \"rdbms\" + {mod_vcard, " backend = \"rdbms\" host = \"vjud.@HOST@\""}, {mod_roster, " backend = \"rdbms\"\n"}]}, {ldap_mnesia, @@ -346,8 +343,7 @@ {password_format, "password.format = \"scram\""}, {auth_ldap, "ldap.base = \"ou=Users,dc=esl,dc=com\" ldap.filter = \"(objectClass=inetOrgPerson)\""}, - {mod_vcard, "[modules.mod_vcard] - backend = \"ldap\" + {mod_vcard, " backend = \"ldap\" host = \"vjud.@HOST@\" ldap_base = \"ou=Users,dc=esl,dc=com\" ldap_filter = \"(objectClass=inetOrgPerson)\""}]}, @@ -378,8 +374,7 @@ {mod_private, "[modules.mod_private] backend = \"riak\""}, {mod_offline, " backend = \"riak\"\n"}, - {mod_vcard, "[modules.mod_vcard] - backend = \"riak\" + {mod_vcard, " backend = \"riak\" host = \"vjud.@HOST@\""}, {mod_roster, " backend = \"riak\"\n"} ]}, diff --git a/rel/files/mongooseim.toml b/rel/files/mongooseim.toml index 4e8bc701e3b..ebb595c4983 100644 --- a/rel/files/mongooseim.toml +++ b/rel/files/mongooseim.toml @@ -266,8 +266,8 @@ [modules.mod_sic] {{#mod_vcard}} +[modules.mod_vcard] {{{mod_vcard}}} - {{/mod_vcard}} [modules.mod_bosh] diff --git a/rel/mim1.vars-toml.config b/rel/mim1.vars-toml.config index c90530d3a96..f9d0d64c5d7 100644 --- a/rel/mim1.vars-toml.config +++ b/rel/mim1.vars-toml.config @@ -52,7 +52,11 @@ {{#mod_blocking}} [host_config.modules.mod_blocking] {{{mod_blocking}}} - {{/mod_blocking}}"}. + {{/mod_blocking}} + {{#mod_vcard}} + [host_config.modules.mod_vcard] + {{{mod_vcard}}} + {{/mod_vcard}}"}. {password_format, "password.format = \"scram\" password.hash = [\"sha256\"]"}. {scram_iterations, 64}. diff --git a/rel/vars-toml.config.in b/rel/vars-toml.config.in index c8d84cf926b..21f857a3635 100644 --- a/rel/vars-toml.config.in +++ b/rel/vars-toml.config.in @@ -28,8 +28,7 @@ {mod_blocking, ""}. {mod_private, "[modules.mod_private]"}. {mod_roster, ""}. -{mod_vcard, "[modules.mod_vcard] - host = \"vjud.@HOST@\""}. +{mod_vcard, " host = \"vjud.@HOST@\""}. {sm_backend, "\"mnesia\""}. {auth_method, "\"internal\""}. {cyrsasl_external, "\"standard\""}. From 914ef28424b1cc4f38cb4ea80e16d65a161ac6c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Tue, 31 Aug 2021 16:26:56 +0200 Subject: [PATCH 2/8] Avoid config duplication for mod_vcard, use config.spec --- big_tests/tests/mongoose_helper.erl | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/big_tests/tests/mongoose_helper.erl b/big_tests/tests/mongoose_helper.erl index 22b38a90c09..dc9181391a0 100644 --- a/big_tests/tests/mongoose_helper.erl +++ b/big_tests/tests/mongoose_helper.erl @@ -519,19 +519,7 @@ print_debug_info_for_module(Module) -> [ModConfig, IqConfig]). get_vcard_config(Config) -> - Preset = proplists:get_value(preset, Config, undefined), - Backend = preset_to_vcard_backend(Preset), - [{backend, Backend}, {host, {prefix, <<"vjud.">>}}] - ++ vcard_backend_specific_options(Backend). - -vcard_backend_specific_options(ldap) -> - [{ldap_base, "ou=Users,dc=esl,dc=com"}, - {ldap_filter, "(objectClass=inetOrgPerson)"}]; -vcard_backend_specific_options(_) -> - []. - -preset_to_vcard_backend("ldap_mnesia") -> ldap; -preset_to_vcard_backend("riak_mnesia") -> riak; -preset_to_vcard_backend("internal_mnesia") -> mnesia; -preset_to_vcard_backend("elasticsearch_and_cassandra_mnesia") -> mnesia; -preset_to_vcard_backend(_) -> rdbms. + HostType = domain_helper:host_type(), + CurrentConfigs = rpc(mim(), gen_mod, loaded_modules_with_opts, [HostType]), + {mod_vcard, CurrentVcardConfig} = lists:keyfind(mod_vcard, 1, CurrentConfigs), + CurrentVcardConfig. From 63deb54ea5fe9a14d41327e61df3d91402ceb8ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Wed, 1 Sep 2021 09:29:46 +0200 Subject: [PATCH 3/8] Add an utility to get host types from domain_helper - For static domains, there are two distinct host type - For dynamic domains, there is only one host type --- big_tests/tests/domain_helper.erl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/big_tests/tests/domain_helper.erl b/big_tests/tests/domain_helper.erl index 36aafcb3008..88a946ac94d 100644 --- a/big_tests/tests/domain_helper.erl +++ b/big_tests/tests/domain_helper.erl @@ -5,6 +5,8 @@ insert_domain/3, delete_domain/2, make_metrics_prefix/1, + host_types/0, + host_types/1, host_type/0, host_type/1, secondary_host_type/0, @@ -12,6 +14,12 @@ -import(distributed_helper, [get_or_fail/1, rpc/4, mim/0]). +host_types() -> + host_types(mim). + +host_types(NodeKey) -> + lists:usort([host_type(NodeKey), secondary_host_type(NodeKey)]). + host_type() -> host_type(mim). From 50d8df99c12af6f55ea8097c183071060d77f552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Wed, 1 Sep 2021 09:35:07 +0200 Subject: [PATCH 4/8] Use the already saved module options in vcard config --- big_tests/tests/dynamic_modules.erl | 8 ++++++-- big_tests/tests/mongoose_helper.erl | 8 +------- big_tests/tests/vcard_SUITE.erl | 17 +++++++---------- big_tests/tests/vcard_simple_SUITE.erl | 11 +++++------ 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/big_tests/tests/dynamic_modules.erl b/big_tests/tests/dynamic_modules.erl index f890a23e65b..167502de368 100644 --- a/big_tests/tests/dynamic_modules.erl +++ b/big_tests/tests/dynamic_modules.erl @@ -2,8 +2,8 @@ -include_lib("common_test/include/ct.hrl"). --export([save_modules_for_host_types/2]). --export([save_modules/2, ensure_modules/2, ensure_stopped/2, +-export([save_modules_for_host_types/2, + save_modules/2, get_saved_config/3, ensure_modules/2, ensure_stopped/2, restore_modules/2, restore_modules/1]). -export([stop/2, stop/3, start/3, start/4, restart/3, stop_running/2, start_running/1]). @@ -19,6 +19,10 @@ save_modules_for_host_types([], Config) -> save_modules(HostType, Config) -> [{{saved_modules, HostType}, get_current_modules(HostType)} | Config]. +get_saved_config(HostType, Module, Config) -> + SavedModules = proplists:get_value({saved_modules, HostType}, Config), + proplists:get_value(Module, SavedModules). + ensure_modules(HostType, RequiredModules) -> CurrentModules = get_current_modules(HostType), {ToReplace, ReplaceWith} = to_replace(RequiredModules, CurrentModules, [], []), diff --git a/big_tests/tests/mongoose_helper.erl b/big_tests/tests/mongoose_helper.erl index dc9181391a0..63a3af7daa7 100644 --- a/big_tests/tests/mongoose_helper.erl +++ b/big_tests/tests/mongoose_helper.erl @@ -46,7 +46,6 @@ -export([should_minio_be_running/1]). -export([new_mongoose_acc/1]). -export([print_debug_info_for_module/1]). --export([get_vcard_config/1]). -import(distributed_helper, [mim/0, rpc/4]). @@ -513,13 +512,8 @@ should_minio_be_running(Config) -> %% It is useful to debug dynamic IQ handler registration print_debug_info_for_module(Module) -> - ModConfig = rpc(mim(), gen_mod, hosts_and_opts_with_module, [mod_vcard]), + ModConfig = rpc(mim(), gen_mod, hosts_and_opts_with_module, [Module]), IqConfig = rpc(mim(), ets, tab2list, [sm_iqtable]), ct:pal("hosts_and_opts=~p~n iq_handlers=~p~n", [ModConfig, IqConfig]). -get_vcard_config(Config) -> - HostType = domain_helper:host_type(), - CurrentConfigs = rpc(mim(), gen_mod, loaded_modules_with_opts, [HostType]), - {mod_vcard, CurrentVcardConfig} = lists:keyfind(mod_vcard, 1, CurrentConfigs), - CurrentVcardConfig. diff --git a/big_tests/tests/vcard_SUITE.erl b/big_tests/tests/vcard_SUITE.erl index 14cdb57de56..54bd5b4b9e6 100644 --- a/big_tests/tests/vcard_SUITE.erl +++ b/big_tests/tests/vcard_SUITE.erl @@ -46,6 +46,8 @@ rpc/4]). -import(ldap_helper, [get_ldap_base/1, call_ldap/3]). +-import(domain_helper, [host_type/0, + host_types/0]). %%-------------------------------------------------------------------- %% Suite configuration @@ -1154,21 +1156,16 @@ prepare_vcard_module(Config) -> %% Keep the old config, so we can undo our changes, once finished testing Config1 = dynamic_modules:save_modules_for_host_types(host_types(), Config), %% Get a list of options, we can use as a prototype to start new modules - [{mod_vcard_opts, mongoose_helper:get_vcard_config(Config)} | Config1]. + HostType = domain_helper:host_type(), + VCardOpts = dynamic_modules:get_saved_config(host_type(), mod_vcard, Config1), + [{mod_vcard_opts, VCardOpts} | Config1]. restore_vcard_module(Config) -> dynamic_modules:restore_modules(Config). -host_types() -> - HostType = ct:get_config({hosts, mim, host_type}), - SecHostType = ct:get_config({hosts, mim, secondary_host_type}), - lists:usort([HostType, SecHostType]). - stop_vcard_mod(_Config) -> - HostType = ct:get_config({hosts, mim, host_type}), - SecHostType = ct:get_config({hosts, mim, secondary_host_type}), - dynamic_modules:stop(HostType, mod_vcard), - dynamic_modules:stop(SecHostType, mod_vcard). + [dynamic_modules:stop(HostType, mod_vcard) || HostType <- host_types()], + ok. params_all(Config) -> add_backend_param([], ?config(mod_vcard_opts, Config)). diff --git a/big_tests/tests/vcard_simple_SUITE.erl b/big_tests/tests/vcard_simple_SUITE.erl index 97651bba769..e00e12af2a9 100644 --- a/big_tests/tests/vcard_simple_SUITE.erl +++ b/big_tests/tests/vcard_simple_SUITE.erl @@ -35,6 +35,8 @@ require_rpc_nodes/1, subhost_pattern/1, rpc/4]). +-import(domain_helper, [host_type/0, + host_types/0]). %%-------------------------------------------------------------------- %% Suite configuration @@ -466,12 +468,9 @@ prepare_vcard_module(Config) -> %% Keep the old config, so we can undo our changes, once finished testing Config1 = dynamic_modules:save_modules_for_host_types(host_types(), Config), %% Get a list of options, we can use as a prototype to start new modules - [{mod_vcard_opts, mongoose_helper:get_vcard_config(Config)} | Config1]. - -host_types() -> - HostType = ct:get_config({hosts, mim, host_type}), - SecHostType = ct:get_config({hosts, mim, secondary_host_type}), - lists:usort([HostType, SecHostType]). + HostType = domain_helper:host_type(), + VCardOpts = dynamic_modules:get_saved_config(host_type(), mod_vcard, Config1), + [{mod_vcard_opts, VCardOpts} | Config1]. restore_vcard_module(Config) -> dynamic_modules:restore_modules(Config). From a0a194d58a1c87dc079198285c25ff1c09bfb4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 2 Sep 2021 09:37:47 +0200 Subject: [PATCH 5/8] Configure mod_private for tests with dynamic domains --- big_tests/test.config | 12 ++++-------- rel/files/mongooseim.toml | 1 + rel/mim1.vars-toml.config | 5 ++++- rel/vars-toml.config.in | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/big_tests/test.config b/big_tests/test.config index 93fc58f1d4e..90f41599e1d 100644 --- a/big_tests/test.config +++ b/big_tests/test.config @@ -256,8 +256,7 @@ connection.tls.server_name_indication = false"}, {mod_last, " backend = \"rdbms\""}, {mod_privacy, " backend = \"rdbms\""}, - {mod_private, "[modules.mod_private] - backend = \"rdbms\""}, + {mod_private, " backend = \"rdbms\""}, {mod_offline, " backend = \"rdbms\"\n"}, {mod_vcard, " backend = \"rdbms\" host = \"vjud.@HOST@\""}, @@ -276,8 +275,7 @@ connection.settings = \"DSN=mongoose-mssql;UID=sa;PWD=mongooseim_secret+ESL123\""}, {mod_last, " backend = \"rdbms\""}, {mod_privacy, " backend = \"rdbms\""}, - {mod_private, "[modules.mod_private] - backend = \"rdbms\""}, + {mod_private, " backend = \"rdbms\""}, {mod_offline, " backend = \"rdbms\"\n"}, {mod_vcard, " backend = \"rdbms\" host = \"vjud.@HOST@\""}, @@ -306,8 +304,7 @@ connection.tls.versions = [\"tlsv1.2\"]"}, {mod_last, " backend = \"rdbms\""}, {mod_privacy, " backend = \"rdbms\""}, - {mod_private, "[modules.mod_private] - backend = \"rdbms\""}, + {mod_private, " backend = \"rdbms\""}, {mod_offline, " backend = \"rdbms\"\n"}, {mod_vcard, " backend = \"rdbms\" host = \"vjud.@HOST@\""}, @@ -371,8 +368,7 @@ connection.tls.cacertfile = \"priv/ssl/cacert.pem\""}, {mod_last, " backend = \"riak\""}, {mod_privacy, " backend = \"riak\""}, - {mod_private, "[modules.mod_private] - backend = \"riak\""}, + {mod_private, " backend = \"riak\""}, {mod_offline, " backend = \"riak\"\n"}, {mod_vcard, " backend = \"riak\" host = \"vjud.@HOST@\""}, diff --git a/rel/files/mongooseim.toml b/rel/files/mongooseim.toml index ebb595c4983..1e7c5fde84f 100644 --- a/rel/files/mongooseim.toml +++ b/rel/files/mongooseim.toml @@ -248,6 +248,7 @@ {{{mod_blocking}}} {{/mod_blocking}} {{#mod_private}} +[modules.mod_private] {{{mod_private}}} {{/mod_private}} diff --git a/rel/mim1.vars-toml.config b/rel/mim1.vars-toml.config index f9d0d64c5d7..99d2f048912 100644 --- a/rel/mim1.vars-toml.config +++ b/rel/mim1.vars-toml.config @@ -49,6 +49,10 @@ [host_config.modules.mod_privacy] {{{mod_privacy}}} {{/mod_privacy}} + {{#mod_private}} + [host_config.modules.mod_private] + {{{mod_private}}} + {{/mod_private}} {{#mod_blocking}} [host_config.modules.mod_blocking] {{{mod_blocking}}} @@ -100,7 +104,6 @@ password = \"secret\""}. {mod_amp, "[modules.mod_amp]"}. -{mod_private, "[modules.mod_private]"}. {mod_cache_users, " time_to_live = 2 number_of_segments = 5 "}. diff --git a/rel/vars-toml.config.in b/rel/vars-toml.config.in index 21f857a3635..c3550e4a1ba 100644 --- a/rel/vars-toml.config.in +++ b/rel/vars-toml.config.in @@ -26,7 +26,7 @@ {mod_offline, ""}. {mod_privacy, ""}. {mod_blocking, ""}. -{mod_private, "[modules.mod_private]"}. +{mod_private, ""}. {mod_roster, ""}. {mod_vcard, " host = \"vjud.@HOST@\""}. {sm_backend, "\"mnesia\""}. From 6156a1803798e021c2f987b3318022c5cc8f8c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 2 Sep 2021 09:38:22 +0200 Subject: [PATCH 6/8] Avoid config duplication for mod_private, use config.spec --- big_tests/tests/private_SUITE.erl | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/big_tests/tests/private_SUITE.erl b/big_tests/tests/private_SUITE.erl index ec042d02a3f..87ef84f3faf 100644 --- a/big_tests/tests/private_SUITE.erl +++ b/big_tests/tests/private_SUITE.erl @@ -45,8 +45,6 @@ suite() -> escalus:suite(). init_per_suite(Config) -> - HostType = domain_helper:host_type(mim), - ok = dynamic_modules:ensure_modules(HostType, private_modules(HostType)), escalus:init_per_suite(Config). end_per_suite(Config) -> @@ -64,19 +62,6 @@ init_per_testcase(CaseName, Config) -> end_per_testcase(CaseName, Config) -> escalus:end_per_testcase(CaseName, Config). -private_modules(HostType) -> - [{mod_private, [{backend, mod_private_backend(HostType)}]}]. - -mod_private_backend(HostType) -> - case mam_helper:is_riak_enabled(HostType) of - true -> riak; - false -> - case mongoose_helper:is_rdbms_enabled(HostType) of - true -> rdbms; - false -> mnesia - end - end. - %%-------------------------------------------------------------------- %% Private storage tests %%-------------------------------------------------------------------- From b2dc27689170458e225860debd0669890a8d3ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 2 Sep 2021 09:38:55 +0200 Subject: [PATCH 7/8] Remove unnecessary newline --- rel/mim1.vars-toml.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rel/mim1.vars-toml.config b/rel/mim1.vars-toml.config index 99d2f048912..e071d2c8226 100644 --- a/rel/mim1.vars-toml.config +++ b/rel/mim1.vars-toml.config @@ -105,8 +105,7 @@ {mod_amp, "[modules.mod_amp]"}. {mod_cache_users, " time_to_live = 2 - number_of_segments = 5 -"}. + number_of_segments = 5"}. {zlib, "10_000"}. {c2s_dhfile, "\"priv/ssl/fake_dh_server.pem\""}. {s2s_dhfile, "\"priv/ssl/fake_dh_server.pem\""}. From 562798e2c54c7e09989b60106d301f808b09d248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 2 Sep 2021 11:43:29 +0200 Subject: [PATCH 8/8] Template mod_amp similarly to other modules --- rel/files/mongooseim.toml | 2 +- rel/mim1.vars-toml.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/files/mongooseim.toml b/rel/files/mongooseim.toml index 1e7c5fde84f..87f4df1e833 100644 --- a/rel/files/mongooseim.toml +++ b/rel/files/mongooseim.toml @@ -213,8 +213,8 @@ [modules.mod_adhoc] {{#mod_amp}} +[modules.mod_amp] {{{mod_amp}}} - {{/mod_amp}} [modules.mod_disco] users_can_see_hidden_services = false diff --git a/rel/mim1.vars-toml.config b/rel/mim1.vars-toml.config index e071d2c8226..bfbca586c52 100644 --- a/rel/mim1.vars-toml.config +++ b/rel/mim1.vars-toml.config @@ -10,6 +10,7 @@ {mod_privacy, ""}. {mod_blocking, ""}. +{mod_amp, ""}. {host_config, "[[host_config]] host = \"anonymous.localhost\" @@ -103,7 +104,6 @@ ip_address = \"127.0.0.1\" password = \"secret\""}. -{mod_amp, "[modules.mod_amp]"}. {mod_cache_users, " time_to_live = 2 number_of_segments = 5"}. {zlib, "10_000"}.