From 317772cc5dad1b4e488b1a93a99417a1448797cb Mon Sep 17 00:00:00 2001 From: Nelson Vides Date: Mon, 26 Feb 2024 16:20:44 +0100 Subject: [PATCH] Update pools scope=host to host_type --- .../tests/mod_event_pusher_rabbit_SUITE.erl | 4 ++-- big_tests/tests/push_http_SUITE.erl | 2 +- doc/configuration/outgoing-connections.md | 16 ++++++++++++---- doc/migrations/6.2.0_x.x.x.md | 4 ++++ src/config/mongoose_config_spec.erl | 17 +++++++++++++---- src/wpool/mongoose_wpool.erl | 7 ++++--- test/common/config_parser_helper.erl | 4 ++-- test/config_parser_SUITE.erl | 6 +++++- .../outgoing_pools.toml | 6 +++--- test/mongoose_wpool_SUITE.erl | 8 ++++---- 10 files changed, 50 insertions(+), 24 deletions(-) diff --git a/big_tests/tests/mod_event_pusher_rabbit_SUITE.erl b/big_tests/tests/mod_event_pusher_rabbit_SUITE.erl index 00f8fc5b94d..557687ff7fa 100644 --- a/big_tests/tests/mod_event_pusher_rabbit_SUITE.erl +++ b/big_tests/tests/mod_event_pusher_rabbit_SUITE.erl @@ -44,7 +44,7 @@ -define(CHAT_MSG_RECV_TOPIC, <<"custom_chat_msg_recv_topic">>). -define(GROUP_CHAT_MSG_SENT_TOPIC, <<"custom_group_chat_msg_sent_topic">>). -define(GROUP_CHAT_MSG_RECV_TOPIC, <<"custom_group_chat_msg_recv_topic">>). --define(WPOOL_CFG, #{scope => host, +-define(WPOOL_CFG, #{scope => host_type, opts => #{workers => 20, strategy => best_worker, call_timeout => 5000}}). -define(IF_EXCHANGE_EXISTS_RETRIES, 30). -define(WAIT_FOR_EXCHANGE_INTERVAL, 100). % ms @@ -178,7 +178,7 @@ end_per_testcase(CaseName, Config) -> rabbit_pool_starts_with_default_config(_Config) -> %% GIVEN Domain = domain(), - DefaultWpoolConfig = #{type => rabbit, scope => host, tag => rabbit_event_pusher_default, + DefaultWpoolConfig = #{type => rabbit, scope => host_type, tag => rabbit_event_pusher_default, opts => #{workers => 10, strategy => best_worker, call_timeout => 5000}, conn_opts => #{amqp_port => 5672, confirms_enabled => false, max_worker_queue_len => 1000}}, RabbitWpool = {rabbit, Domain, rabbit_event_pusher_default}, diff --git a/big_tests/tests/push_http_SUITE.erl b/big_tests/tests/push_http_SUITE.erl index e4390e75ef1..b986c667a35 100644 --- a/big_tests/tests/push_http_SUITE.erl +++ b/big_tests/tests/push_http_SUITE.erl @@ -171,7 +171,7 @@ start_pool() -> PoolOpts = #{strategy => random_worker, call_timeout => 5000, workers => 10}, ConnOpts = #{host => http_notifications_host(), request_timeout => 5000}, Pool = config([outgoing_pools, http, http_pool], - #{scope => host, opts => PoolOpts, conn_opts => ConnOpts}), + #{scope => host_type, opts => PoolOpts, conn_opts => ConnOpts}), [{ok, _Pid}] = rpc(mongoose_wpool, start_configured_pools, [[Pool], [<<"localhost">>]]). stop_pool() -> diff --git a/doc/configuration/outgoing-connections.md b/doc/configuration/outgoing-connections.md index 87808ad81bd..e2b60c755d6 100644 --- a/doc/configuration/outgoing-connections.md +++ b/doc/configuration/outgoing-connections.md @@ -15,9 +15,14 @@ This allows you to create multiple dedicated pools of the same type. ## General pool options ### `outgoing_pools.*.*.scope` -* **Syntax:** string, one of:`"global"`, `"host"`, `"single_host"` +* **Syntax:** string, one of:`"global"`, `"host_type"`, `"single_host_type"` * **Default:** `"global"` -* **Example:** `scope = "host"` +* **Example:** `scope = "host_type"` + +### `outgoing_pools.*.*.host_type` +* **Syntax:** string +* **Default:** no default; required if `"single_host"` scope is specified +* **Example:** `host_type = "basic_host_type"` ### `outgoing_pools.*.*.host` * **Syntax:** string @@ -27,13 +32,16 @@ This allows you to create multiple dedicated pools of the same type. `scope` can be set to: * `global` - meaning that the pool will be started once no matter how many XMPP hosts are served by MongooseIM -* `host` - the pool will be started for each XMPP host or host type served by MongooseIM -* `single_host` - the pool will be started for the selected host or host type only (you must provide the name). +* `host_type` - the pool will be started for each XMPP host or host type served by MongooseIM +* `single_host_type` - the pool will be started for the selected host or host type only (you must provide the name). !!! Note A global pool with name `default` is used by services that are not configured by host_type, like `service_domain_db` or `service_mongoose_system_metrics`, or by modules that don't support dynamic domains, like `mod_pubsub`. If a global pool is not configured, these services will fail. + !!! Note + `host` and `single_host` are still supported and behave equivalent to `host_type` and `single_host_type` respectively; however, they are deprecated in favour of the former. + ## Worker pool options All pools are managed by the [inaka/worker_pool](https://github.com/inaka/worker_pool) library. diff --git a/doc/migrations/6.2.0_x.x.x.md b/doc/migrations/6.2.0_x.x.x.md index 5b1df61df69..0c652b6b8ca 100644 --- a/doc/migrations/6.2.0_x.x.x.md +++ b/doc/migrations/6.2.0_x.x.x.md @@ -1,3 +1,7 @@ ## Database migration There is a new column in the `mam_message` table in the database, which is used to support including or excluding groupchat results in a user archive. See the migrations for Postgres, MySQL and MSSQL in the [`priv/migrations`](https://github.com/esl/MongooseIM/tree/master/priv/migrations) directory. Please be aware that the filtering process will only be effective for new messages and will not apply to those messages that have already been stored in the database. + +## Outgoing pools + +Pools now take `host_type` and `single_host_type` instead of `host` and `single_host` in their scope, see [outgoing pools](../configuration/outgoing-connections.md) for more information. diff --git a/src/config/mongoose_config_spec.erl b/src/config/mongoose_config_spec.erl index 866fa351d32..24b116b6f9f 100644 --- a/src/config/mongoose_config_spec.erl +++ b/src/config/mongoose_config_spec.erl @@ -476,8 +476,12 @@ wpool(ExtraDefaults) -> outgoing_pool_extra(Type) -> #section{items = #{<<"scope">> => #option{type = atom, - validate = {enum, [global, host, single_host]}}, - <<"host">> => #option{type = binary, + validate = {enum, [global, + host_type, single_host_type, + host, single_host]}}, %% TODO deprecated + <<"host_type">> => #option{type = binary, + validate = non_empty}, + <<"host">> => #option{type = binary, %% TODO deprecated validate = non_empty}, <<"connection">> => outgoing_pool_connection(Type) }, @@ -1078,7 +1082,7 @@ check_auth_method(Method, Opts) -> end. process_pool([Tag, Type|_], AllOpts = #{scope := ScopeIn, connection := Connection}) -> - Scope = pool_scope(ScopeIn, maps:get(host, AllOpts, none)), + Scope = pool_scope(ScopeIn, maps:get(host_type, AllOpts, maps:get(host, AllOpts, none))), Opts = maps:without([scope, host, connection], AllOpts), #{type => b2a(Type), scope => Scope, @@ -1086,11 +1090,16 @@ process_pool([Tag, Type|_], AllOpts = #{scope := ScopeIn, connection := Connecti opts => Opts, conn_opts => Connection}. +pool_scope(single_host_type, none) -> + error(#{what => pool_single_host_not_specified, + text => <<"\"host_type\" option is required if \"single_host_type\" is used.">>}); pool_scope(single_host, none) -> error(#{what => pool_single_host_not_specified, text => <<"\"host\" option is required if \"single_host\" is used.">>}); +pool_scope(single_host_type, Host) -> Host; pool_scope(single_host, Host) -> Host; -pool_scope(host, none) -> host; +pool_scope(host, none) -> host_type; +pool_scope(host_type, none) -> host_type; pool_scope(global, none) -> global. process_ldap_connection(ConnOpts = #{port := _}) -> ConnOpts; diff --git a/src/wpool/mongoose_wpool.erl b/src/wpool/mongoose_wpool.erl index 81397e16168..82e00bc4c27 100644 --- a/src/wpool/mongoose_wpool.erl +++ b/src/wpool/mongoose_wpool.erl @@ -8,6 +8,7 @@ -module(mongoose_wpool). -author("bartlomiej.gorny@erlang-solutions.com"). -include("mongoose.hrl"). +-include("mongoose_config_spec.hrl"). -record(mongoose_wpool, { name :: pool_name(), @@ -47,7 +48,7 @@ | rabbit | ldap. %% Config scope --type scope() :: global | host | mongooseim:host_type(). +-type scope() :: global | host_type | mongooseim:host_type(). -type host_type_or_global() :: mongooseim:host_type_or_global(). -type tag() :: atom(). @@ -361,9 +362,9 @@ make_callback_module_name(PoolType) -> expand_pools(Pools, HostTypes) -> %% First we select only pools for a specific vhost HostSpecific = [{Type, HT, Tag} || #{type := Type, scope := HT, tag := Tag} <- Pools, is_binary(HT)], - %% Then we expand all pools with `host` as HostType parameter but using host specific configs + %% Then we expand all pools with `host_type` as HostType parameter but using host_type specific configs %% if they were provided - F = fun(M = #{type := PoolType, scope := host, tag := Tag}) -> + F = fun(M = #{type := PoolType, scope := host_type, tag := Tag}) -> [M#{scope => HostType} || HostType <- HostTypes, not lists:member({PoolType, HostType, Tag}, HostSpecific)]; (Other) -> [Other] diff --git a/test/common/config_parser_helper.erl b/test/common/config_parser_helper.erl index f63a2b72627..a93adcf343c 100644 --- a/test/common/config_parser_helper.erl +++ b/test/common/config_parser_helper.erl @@ -327,12 +327,12 @@ options("outgoing_pools") -> opts => #{workers => 50}, conn_opts => #{host => "https://localhost:8443", request_timeout => 2000}}, - #{type => ldap, scope => host, tag => default, + #{type => ldap, scope => host_type, tag => default, opts => #{workers => 5}, conn_opts => #{password => <<"ldap-admin-password">>, root_dn => <<"cn=admin,dc=example,dc=com">>, servers => ["ldap-server.example.com"]}}, - #{type => rabbit, scope => host, tag => event_pusher, + #{type => rabbit, scope => host_type, tag => event_pusher, opts => #{workers => 20}, conn_opts => #{confirms_enabled => true, max_worker_queue_len => 100}}, diff --git a/test/config_parser_SUITE.erl b/test/config_parser_SUITE.erl index c740a30d927..e9f14933a9c 100644 --- a/test/config_parser_SUITE.erl +++ b/test/config_parser_SUITE.erl @@ -884,10 +884,14 @@ pool_scope(_Config) -> P = [outgoing_pools, 1, scope], Required = #{<<"connection">> => #{<<"host">> => <<"http://localhost">>}}, T = fun(Opts) -> pool_raw(<<"http">>, <<"default">>, maps:merge(Required, Opts)) end, - ?cfg(P, host, T(#{<<"scope">> => <<"host">>})), + ?cfg(P, host_type, T(#{<<"scope">> => <<"host">>})), + ?cfg(P, host_type, T(#{<<"scope">> => <<"host_type">>})), ?cfg(P, <<"localhost">>, T(#{<<"scope">> => <<"single_host">>, <<"host">> => <<"localhost">>})), + ?cfg(P, <<"localhost">>, T(#{<<"scope">> => <<"single_host_type">>, <<"host_type">> => <<"localhost">>})), ?err(T(#{<<"host">> => <<"localhost">>})), % missing scope + ?err(T(#{<<"host_type">> => <<"localhost">>})), % missing scope ?err(T(#{<<"scope">> => <<"single_host">>})), % missing host + ?err(T(#{<<"scope">> => <<"single_host_type">>})), % missing host ?err(T(#{<<"scope">> => <<"whatever">>})). pool_rdbms(_Config) -> diff --git a/test/config_parser_SUITE_data/outgoing_pools.toml b/test/config_parser_SUITE_data/outgoing_pools.toml index f10d90c45da..fb51c8e7b55 100644 --- a/test/config_parser_SUITE_data/outgoing_pools.toml +++ b/test/config_parser_SUITE_data/outgoing_pools.toml @@ -7,7 +7,7 @@ default_server_domain = "localhost" [outgoing_pools.redis.global_distrib] - scope = "single_host" + scope = "single_host_type" host = "localhost" workers = 10 @@ -50,7 +50,7 @@ connection.host = "localhost" [outgoing_pools.rabbit.event_pusher] - scope = "host" + scope = "host_type" workers = 20 [outgoing_pools.rabbit.event_pusher.connection] @@ -62,7 +62,7 @@ max_worker_queue_len = 100 [outgoing_pools.ldap.default] - scope = "host" + scope = "host_type" workers = 5 [outgoing_pools.ldap.default.connection] diff --git a/test/mongoose_wpool_SUITE.erl b/test/mongoose_wpool_SUITE.erl index e6ccd65efd6..c4c4fa98350 100644 --- a/test/mongoose_wpool_SUITE.erl +++ b/test/mongoose_wpool_SUITE.erl @@ -128,7 +128,7 @@ two_distinct_redis_pools_are_started(_C) -> ?assertMatch(#{host := "localhost2", port := 1806}, proplists:to_map(Props2)). generic_pools_are_started_for_all_vhosts(_C) -> - Pools = [#{type => generic, scope => host, tag => default, opts => #{}, conn_opts => #{}}], + Pools = [#{type => generic, scope => host_type, tag => default, opts => #{}, conn_opts => #{}}], StartRes = mongoose_wpool:start_configured_pools(Pools), ?assertMatch([_, _, _], StartRes), ?assertMatch([{generic, <<"a.com">>, default}, @@ -137,7 +137,7 @@ generic_pools_are_started_for_all_vhosts(_C) -> ordsets:from_list(mongoose_wpool:get_pools())). host_specific_pools_are_preserved(_C) -> - Pools = [#{type => generic, scope => host, tag => default, opts => #{}, conn_opts => #{}}, + Pools = [#{type => generic, scope => host_type, tag => default, opts => #{}, conn_opts => #{}}, #{type => generic, scope => <<"b.com">>, tag => default, opts => #{workers => 12}, conn_opts => #{}}], Expanded = mongoose_wpool:expand_pools(Pools, [<<"a.com">>, <<"b.com">>, <<"c.eu">>]), @@ -150,10 +150,10 @@ host_specific_pools_are_preserved(_C) -> Expanded). pools_for_different_tag_are_expanded_with_host_specific_config_preserved(_C) -> - Pools = [#{type => generic, scope => host, tag => default, opts => #{}, conn_opts => #{}}, + Pools = [#{type => generic, scope => host_type, tag => default, opts => #{}, conn_opts => #{}}, #{type => generic, scope => <<"b.com">>, tag => default, opts => #{workers => 12}, conn_opts => #{}}, - #{type => generic, scope => host, tag => other_tag, opts => #{}, conn_opts => #{}}], + #{type => generic, scope => host_type, tag => other_tag, opts => #{}, conn_opts => #{}}], Expanded = mongoose_wpool:expand_pools(Pools, [<<"a.com">>, <<"b.com">>, <<"c.eu">>]), ?assertMatch([#{type := generic, host_type := <<"a.com">>, tag := default, opts := [], conn_opts := #{}},