From 2768c2895b2bdd0953d56bc186e2ac509a674c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Fri, 15 Mar 2024 18:03:38 +0100 Subject: [PATCH 1/9] Add helpers for testing instrumentation events in big tests Testing procedure: 1. In init_per_suite, call instrument_helper:start(DeclaredEvents). The helper installs instrument_event_table on mim(), which will collect events with matching EventName and Labels. 2. In the tests, call instrument_helper:assert/3. The function fails if there is no matching event. 3. In end_per_suite, call instrument_helper:stop/0, which deletes the collected events, and logs a summary of event status (tested/untested). Don't delete --- big_tests/tests/instrument_event_table.erl | 35 +++++++++ big_tests/tests/instrument_helper.erl | 85 ++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 big_tests/tests/instrument_event_table.erl create mode 100644 big_tests/tests/instrument_helper.erl diff --git a/big_tests/tests/instrument_event_table.erl b/big_tests/tests/instrument_event_table.erl new file mode 100644 index 00000000000..2bc332eeb75 --- /dev/null +++ b/big_tests/tests/instrument_event_table.erl @@ -0,0 +1,35 @@ +%% @doc This module is injected to the mim node by instrument_helper. +%% It collects events in an ETS table. + +-module(instrument_event_table). + +-export([set_up/1, tear_down/0, get_events/2]). + +set_up(DeclaredEvents) -> + ets_helper:new(?MODULE, [bag]), % repeating measurements are stored only once + meck:new(mongoose_instrument_log, [no_link, passthrough]), + meck:expect(mongoose_instrument_log, handle_event, + fun(Event, Labels, _Config, Measurements) -> + case lists:member({Event, Labels}, DeclaredEvents) of + true -> ets:insert(?MODULE, {{Event, Labels}, Measurements}); + false -> ok + end, + meck:passthrough() + end). + +tear_down() -> + meck:unload(mongoose_instrument_log), + Logged = all_keys(?MODULE), + ets_helper:delete(?MODULE), + {ok, Logged}. + +get_events(EventName, Labels) -> + ets:lookup(?MODULE, {EventName, Labels}). + +all_keys(Tab) -> + all_keys(Tab, ets:first(Tab), []). + +all_keys(_Tab, '$end_of_table', Acc) -> + lists:sort(Acc); +all_keys(Tab, Key, Acc) -> + all_keys(Tab, ets:next(Tab, Key), [Key|Acc]). diff --git a/big_tests/tests/instrument_helper.erl b/big_tests/tests/instrument_helper.erl new file mode 100644 index 00000000000..1bcdc1ce3e0 --- /dev/null +++ b/big_tests/tests/instrument_helper.erl @@ -0,0 +1,85 @@ +%% @doc A helper module for checking instrumentation events in tests. +%% For now it always uses mim(). We can extend it to other nodes when needed. + +-module(instrument_helper). + +-export([declared_events/1, start/1, stop/0, assert/3]). + +-import(distributed_helper, [rpc/4, mim/0]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(STATUS_TABLE, instrument_event_status_table). + +-type event_name() :: atom(). +-type labels() :: #{atom() => term()}. +-type measurements() :: #{atom() => term()}. + +%% API + +%% @doc Helper to get `DeclaredEvents' needed by `start/1' +-spec declared_events([module()] | module()) -> [{event_name(), labels()}]. +declared_events(Modules) when is_list(Modules) -> + lists:flatmap(fun declared_events/1, Modules); +declared_events(Module) -> + Specs = rpc(mim(), Module, instrumentation, [domain_helper:host_type()]), + [{Event, Labels} || {Event, Labels, _Config} <- Specs]. + +%% @doc Only `DeclaredEvents' will be logged, and can be tested with `assert/3' +-spec start([{event_name(), labels()} | module()]) -> ok. +start(DeclaredEvents) -> + mongoose_helper:inject_module(ets_helper), + mongoose_helper:inject_module(instrument_event_table), + ets_helper:new(?STATUS_TABLE), + [ets:insert(?STATUS_TABLE, {Event, untested}) || Event <- DeclaredEvents], + ok = rpc(mim(), instrument_event_table, set_up, [DeclaredEvents]). + +-spec stop() -> ok. +stop() -> + #{tested := Tested, untested := Untested} = classify_events(), + ets_helper:delete(?STATUS_TABLE), + {ok, Logged} = rpc(mim(), instrument_event_table, tear_down, []), + ct:log("Tested instrumentation events:~n ~p", [lists:sort(Tested)]), + verify_unlogged(Untested -- Logged), + verify_logged_but_untested(Logged -- Tested). + +%% @doc `CheckF' can return a boolean or fail with `function_clause', which means `false'. +%% This is for convenience - you only have to code one clause. +-spec assert(event_name(), labels(), fun((measurements()) -> boolean())) -> ok. +assert(EventName, Labels, CheckF) -> + Events = rpc(mim(), instrument_event_table, get_events, [EventName, Labels]), + case lists:filter(fun({_, Measurements}) -> + try CheckF(Measurements) catch error:function_clause -> false end + end, Events) of + [] -> + ct:log("All ~p events with labels ~p:~n~p", [EventName, Labels, Events]), + ct:fail("No instrumentation events matched"); + Filtered -> + ct:log("Matching events: ~p", [Filtered]), + event_tested(EventName, Labels) + end. + +%% Internal functions + +%% Don't fail if some events are unlogged, because we don't have full test coverage (yet) +verify_unlogged([]) -> ok; +verify_unlogged(Events) -> + ct:log("Instrumentation events that were not logged - functionality not covered by tests:~n~p", + [lists:sort(Events)]). + +%% Fail if any events were logged, but there were no assertions for them +verify_logged_but_untested([]) -> ok; +verify_logged_but_untested(Events) -> + ct:fail("Instrumentation events that were logged, but not tested:~n~p~n" + "You need to test them with ~p:assert/3", + [lists:sort(Events), ?MODULE]). + +event_tested(EventName, Labels) -> + ets:insert(?STATUS_TABLE, {{EventName, Labels}, tested}), + ok. + +classify_events() -> + ets:foldl(fun classify_event/2, #{tested => [], untested => []}, ?STATUS_TABLE). + +classify_event({Event, Status}, M) -> + M#{Status => [Event | maps:get(Status, M)]}. From 457a0a93a0b3e6da6233bb9fb0490f3bed3ae3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 15 Feb 2024 09:55:35 +0100 Subject: [PATCH 2/9] Instrument mod_mam Unify event names: mod_mam_pm_* and mod_mam_muc_* No functional changes other than metric renaming --- src/mam/mod_mam_elasticsearch_arch.erl | 2 +- src/mam/mod_mam_muc.erl | 94 ++++++++++--------- src/mam/mod_mam_muc_elasticsearch_arch.erl | 2 +- src/mam/mod_mam_muc_rdbms_arch_async.erl | 27 +++--- src/mam/mod_mam_pm.erl | 104 +++++++++++---------- src/mam/mod_mam_rdbms_arch_async.erl | 27 +++--- 6 files changed, 137 insertions(+), 119 deletions(-) diff --git a/src/mam/mod_mam_elasticsearch_arch.erl b/src/mam/mod_mam_elasticsearch_arch.erl index f90bccb9e4e..92479724653 100644 --- a/src/mam/mod_mam_elasticsearch_arch.erl +++ b/src/mam/mod_mam_elasticsearch_arch.erl @@ -97,7 +97,7 @@ archive_message(_Result, ?LOG_ERROR(#{what => archive_message_failed, user => Owner, server => Host, remote => Remote, message_id => MessageId, reason => Reason}), - mongoose_metrics:update(Host, modMamDropped, 1), + mongoose_instrument:execute(mod_mam_pm_dropped, #{host_type => Host}, #{count => 1}), {ok, Err} end. diff --git a/src/mam/mod_mam_muc.erl b/src/mam/mod_mam_muc.erl index 625543978c7..8e6e898deeb 100644 --- a/src/mam/mod_mam_muc.erl +++ b/src/mam/mod_mam_muc.erl @@ -28,6 +28,7 @@ %%% @end %%%------------------------------------------------------------------- -module(mod_mam_muc). +-behaviour(gen_mod). %% ---------------------------------------------------------------------- %% Exports @@ -37,7 +38,7 @@ archive_id/2]). %% gen_mod handlers --export([start/2, stop/1, supported_features/0]). +-export([start/2, stop/1, supported_features/0, hooks/1, instrumentation/1]). %% ejabberd room handlers -export([disco_muc_features/3, @@ -54,8 +55,7 @@ -export([lookup_messages/2]). -export([archive_id_int/2]). --ignore_xref([archive_id/2, archive_message_for_ct/1, archive_size/2, delete_archive/2, - start/2, stop/1, supported_features/0]). +-ignore_xref([archive_id/2, archive_message_for_ct/1, archive_size/2, delete_archive/2]). -include_lib("mongoose.hrl"). -include_lib("jlib.hrl"). @@ -117,15 +117,12 @@ archive_id(MucHost, RoomName) when is_binary(MucHost), is_binary(RoomName) -> -spec start(host_type(), gen_mod:module_opts()) -> any(). start(HostType, Opts) -> ?LOG_DEBUG(#{what => mam_muc_starting}), - ensure_metrics(HostType), - gen_hook:add_handlers(hooks(HostType)), add_iq_handlers(HostType, Opts), ok. -spec stop(host_type()) -> any(). stop(HostType) -> ?LOG_DEBUG(#{what => mam_muc_stopping}), - gen_hook:delete_handlers(hooks(HostType)), remove_iq_handlers(HostType), ok. @@ -229,7 +226,8 @@ room_process_mam_iq(Acc, From, To, IQ, #{host_type := HostType}) -> handle_error_iq(Acc, HostType, To, Action, handle_mam_iq(HostType, Action, From, To, IQ)); {error, max_delay_reached} -> - mongoose_metrics:update(HostType, modMucMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_muc_dropped_iq, + #{host_type => HostType}, #{count => 1}), {Acc, return_max_delay_reached_error_iq(IQ)} end; {error, Reason} -> @@ -488,21 +486,25 @@ get_behaviour(HostType, ArcID, LocJID = #jid{}, RemJID = #jid{}) -> AlwaysJIDs :: [jid:literal_jid()], NeverJIDs :: [jid:literal_jid()]) -> any(). set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> - mongoose_hooks:mam_muc_set_prefs(HostType, ArcID, ArcJID, DefaultMode, - AlwaysJIDs, NeverJIDs). + Result = mongoose_hooks:mam_muc_set_prefs(HostType, ArcID, ArcJID, DefaultMode, + AlwaysJIDs, NeverJIDs), + mongoose_instrument:execute(mod_mam_muc_set_prefs, #{host_type => HostType}, #{count => 1}), + Result. %% @doc Load settings from the database. -spec get_prefs(HostType :: host_type(), ArcID :: mod_mam:archive_id(), ArcJID :: jid:jid(), GlobalDefaultMode :: mod_mam:archive_behaviour()) -> mod_mam:preference() | {error, Reason :: term()}. get_prefs(HostType, ArcID, ArcJID, GlobalDefaultMode) -> - mongoose_hooks:mam_muc_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID). + Result = mongoose_hooks:mam_muc_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID), + mongoose_instrument:execute(mod_mam_muc_get_prefs, #{host_type => HostType}, #{count => 1}), + Result. -spec remove_archive(host_type(), mod_mam:archive_id() | undefined, jid:jid()) -> ok. remove_archive(HostType, ArcID, ArcJID = #jid{}) -> mongoose_hooks:mam_muc_remove_archive(HostType, ArcID, ArcJID), - ok. + mongoose_instrument:execute(mod_mam_muc_remove_archive, #{host_type => HostType}, #{count => 1}). %% See description in mod_mam_pm. -spec lookup_messages(HostType :: host_type(), Params :: map()) -> @@ -523,27 +525,33 @@ lookup_messages_without_policy_violation_check(HostType, true -> %% Use of disabled full text search {error, 'not-supported'}; false -> - StartT = erlang:monotonic_time(microsecond), - R = case maps:get(message_ids, Params, undefined) of - undefined -> - mongoose_hooks:mam_muc_lookup_messages(HostType, - Params#{message_id => undefined}); - IDs -> - mod_mam_utils:lookup_specific_messages(HostType, Params, IDs, - fun mongoose_hooks:mam_muc_lookup_messages/2) - end, - Diff = erlang:monotonic_time(microsecond) - StartT, - mongoose_metrics:update(HostType, [backends, ?MODULE, lookup], Diff), - R + mongoose_instrument:span(mod_mam_muc_lookup, #{host_type => HostType}, + fun perform_lookup/2, [HostType, Params], fun measure_lookup/2) + end. + +perform_lookup(HostType, Params) -> + case maps:get(message_ids, Params, undefined) of + undefined -> + mongoose_hooks:mam_muc_lookup_messages(HostType, Params#{message_id => undefined}); + IDs -> + mod_mam_utils:lookup_specific_messages(HostType, Params, IDs, + fun mongoose_hooks:mam_muc_lookup_messages/2) end. +measure_lookup(Time, {ok, {_TotalCount, _Offset, MessageRows}}) -> + #{count => 1, time => Time, size => length(MessageRows)}; +measure_lookup(_, _OtherResult) -> + #{}. + archive_message_for_ct(Params = #{local_jid := RoomJid}) -> HostType = mod_muc_light_utils:room_jid_to_host_type(RoomJid), archive_message(HostType, Params). -spec archive_message(host_type(), mod_mam:archive_message_params()) -> ok | {error, timeout}. archive_message(HostType, Params) -> - mongoose_hooks:mam_muc_archive_message(HostType, Params). + mongoose_instrument:span(mod_mam_muc_archive_message, #{host_type => HostType}, + fun mongoose_hooks:mam_muc_archive_message/2, [HostType, Params], + fun(Time, _Result) -> #{time => Time, count => 1} end). %% ---------------------------------------------------------------------- %% Helpers @@ -586,7 +594,7 @@ message_row_to_ext_id(#{id := MessID}) -> -spec handle_error_iq(mongoose_acc:t(), host_type(), jid:jid(), atom(), {error, term(), jlib:iq()} | jlib:iq() | ignore) -> {mongoose_acc:t(), jlib:iq() | ignore}. handle_error_iq(Acc, HostType, _To, _Action, {error, _Reason, IQ}) -> - mongoose_metrics:update(HostType, modMucMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_muc_dropped_iq, #{host_type => HostType}, #{count => 1}), {Acc, IQ}; handle_error_iq(Acc, _HostType, _To, _Action, IQ) -> {Acc, IQ}. @@ -660,8 +668,7 @@ hooks(HostType) -> [{disco_muc_features, HostType, fun ?MODULE:disco_muc_features/3, #{}, 99}, {filter_room_packet, HostType, fun ?MODULE:filter_room_packet/3, #{}, 60}, {forget_room, HostType, fun ?MODULE:forget_room/3, #{}, 90}, - {get_personal_data, HostType, fun ?MODULE:get_personal_data/3, #{}, 50} - | mongoose_metrics_mam_hooks:get_mam_muc_hooks(HostType)]. + {get_personal_data, HostType, fun ?MODULE:get_personal_data/3, #{}, 50}]. add_iq_handlers(HostType, Opts) -> IQDisc = gen_mod:get_opt(iqdisc, Opts, parallel), @@ -685,20 +692,19 @@ remove_iq_handlers(HostType) -> ?NS_MAM_06, mod_muc_iq), ok. -ensure_metrics(HostType) -> - mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, lookup], histogram), - lists:foreach(fun(Name) -> - mongoose_metrics:ensure_metric(HostType, Name, spiral) - end, - spirals()). - -spirals() -> - [modMucMamPrefsSets, - modMucMamPrefsGets, - modMucMamArchiveRemoved, - modMucMamLookups, - modMucMamForwarded, - modMucMamArchived, - modMucMamFlushed, - modMucMamDropped, - modMucMamDroppedIQ]. +-spec instrumentation(host_type()) -> [mongoose_instrument:spec()]. +instrumentation(HostType) -> + [{mod_mam_muc_archive_message, #{host_type => HostType}, + #{metrics => #{count => spiral, time => histogram}}}, + {mod_mam_muc_lookup, #{host_type => HostType}, + #{metrics => #{count => spiral, size => histogram, time => histogram}}}, + {mod_mam_muc_dropped_iq, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_muc_dropped, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_muc_remove_archive, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_muc_get_prefs, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_muc_set_prefs, #{host_type => HostType}, + #{metrics => #{count => spiral}}}]. diff --git a/src/mam/mod_mam_muc_elasticsearch_arch.erl b/src/mam/mod_mam_muc_elasticsearch_arch.erl index 09a2ff31f7e..14a70faec24 100644 --- a/src/mam/mod_mam_muc_elasticsearch_arch.erl +++ b/src/mam/mod_mam_muc_elasticsearch_arch.erl @@ -97,7 +97,7 @@ archive_message(_Result, Params, #{host_type := HostType}) -> #{what => archive_muc_message_failed, reason => Reason, server => HostType, room => Room, source => SourceBinJid, message_id => MessageId})), - mongoose_metrics:update(HostType, modMamDropped, 1), + mongoose_instrument:execute(mod_mam_dropped, #{host_type => HostType}, #{count => 1}), {ok, Err} end. diff --git a/src/mam/mod_mam_muc_rdbms_arch_async.erl b/src/mam/mod_mam_muc_rdbms_arch_async.erl index 5e448c61583..c5f17c18d98 100644 --- a/src/mam/mod_mam_muc_rdbms_arch_async.erl +++ b/src/mam/mod_mam_muc_rdbms_arch_async.erl @@ -4,11 +4,8 @@ -include("mongoose_logger.hrl"). --define(PER_MESSAGE_FLUSH_TIME, [mod_mam_muc_rdbms_async_pool_writer, per_message_flush_time]). --define(FLUSH_TIME, [mod_mam_muc_rdbms_async_pool_writer, flush_time]). - -behaviour(gen_mod). --export([start/2, stop/1, hooks/1, supported_features/0]). +-export([start/2, stop/1, hooks/1, instrumentation/1, supported_features/0]). -export([archive_muc_message/3, mam_muc_archive_sync/3]). -export([flush/2]). -ignore_xref([flush/2]). @@ -37,8 +34,6 @@ mam_muc_archive_sync(Result, _Params, #{host_type := HostType}) -> start(HostType, Opts) -> {PoolOpts, Extra} = mod_mam_rdbms_arch_async:make_pool_opts(muc, Opts), mod_mam_rdbms_arch_async:prepare_insert_queries(muc, Extra), - mongoose_metrics:ensure_metric(HostType, ?PER_MESSAGE_FLUSH_TIME, histogram), - mongoose_metrics:ensure_metric(HostType, ?FLUSH_TIME, histogram), mongoose_async_pools:start_pool(HostType, muc_mam, PoolOpts). -spec stop(mongooseim:host_type()) -> any(). @@ -52,16 +47,24 @@ hooks(HostType) -> {mam_muc_archive_message, HostType, fun ?MODULE:archive_muc_message/3, #{}, 50} ]. +-spec instrumentation(mongooseim:host_type()) -> [mongoose_instrument:spec()]. +instrumentation(HostType) -> + [{mod_mam_muc_flushed, #{host_type => HostType}, + #{metrics => #{time_per_message => histogram, time => histogram, count => spiral}}}]. + -spec supported_features() -> [atom()]. supported_features() -> [dynamic_domains]. %%% flush callbacks flush(Acc, Extra = #{host_type := HostType, queue_length := MessageCount}) -> - {FlushTime, Result} = timer:tc(fun do_flush_muc/2, [Acc, Extra]), - mongoose_metrics:update(HostType, ?PER_MESSAGE_FLUSH_TIME, round(FlushTime / MessageCount)), - mongoose_metrics:update(HostType, ?FLUSH_TIME, FlushTime), - Result. + mongoose_instrument:span(mod_mam_muc_flushed, #{host_type => HostType}, + fun do_flush_muc/2, [Acc, Extra], + fun(Time, _Result) -> + #{time => Time, + time_per_message => round(Time / MessageCount), + count => MessageCount} + end). %% mam workers callbacks do_flush_muc(Acc, #{host_type := HostType, queue_length := MessageCount, @@ -84,7 +87,7 @@ do_flush_muc(Acc, #{host_type := HostType, queue_length := MessageCount, process_batch_result({updated, _Count}, _, _, _) -> ok; process_batch_result({error, Reason}, Rows, HostType, MessageCount) -> - mongoose_metrics:update(HostType, modMucMamDropped, MessageCount), + mongoose_instrument:execute(mod_mam_muc_dropped, #{host_type => HostType}, #{count => MessageCount}), Keys = [ maps:with([message_id, archive_id], Row) || Row <- Rows ], ?LOG_ERROR(#{what => archive_muc_batch_messages_failed, text => <<"A batch of muc messages failed to archive, modMucMamDropped metric updated">>, @@ -97,7 +100,7 @@ process_list_results(Results, HostType) -> process_single_result({{updated, _Count}, _}, _HostType) -> ok; process_single_result({{error, Reason}, #{message_id := MsgId, archive_id := ArcId}}, HostType) -> - mongoose_metrics:update(HostType, modMucMamDropped, 1), + mongoose_instrument:execute(mod_mam_muc_dropped, #{host_type => HostType}, #{count => 1}), ?LOG_ERROR(#{what => archive_muc_single_message_failed, text => <<"Single muc message failed to archive, modMucMamDropped metric updated">>, message_id => MsgId, archive_id => ArcId, reason => Reason}). diff --git a/src/mam/mod_mam_pm.erl b/src/mam/mod_mam_pm.erl index ac77b776c0e..81d353e8660 100644 --- a/src/mam/mod_mam_pm.erl +++ b/src/mam/mod_mam_pm.erl @@ -40,7 +40,7 @@ archive_id/2]). %% gen_mod handlers --export([start/2, stop/1, supported_features/0]). +-export([start/2, stop/1, supported_features/0, hooks/1, instrumentation/1]). %% hook handlers -export([disco_local_features/3, @@ -109,15 +109,12 @@ archive_id(Server, User) -spec start(host_type(), gen_mod:module_opts()) -> any(). start(HostType, Opts) -> ?LOG_INFO(#{what => mam_starting, host_type => HostType}), - ensure_metrics(HostType), - gen_hook:add_handlers(hooks(HostType)), add_iq_handlers(HostType, Opts), ok. -spec stop(host_type()) -> any(). stop(HostType) -> ?LOG_INFO(#{what => mam_stopping, host_type => HostType}), - gen_hook:delete_handlers(hooks(HostType)), remove_iq_handlers(HostType), ok. @@ -150,11 +147,13 @@ process_mam_iq(Acc, From, To, IQ, _Extra) -> ?LOG_WARNING(#{what => mam_max_delay_reached, text => <<"Return max_delay_reached error IQ from MAM">>, action => Action, acc => Acc}), - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_pm_dropped_iq, + #{host_type => HostType}, #{count => 1}), {Acc, return_max_delay_reached_error_iq(IQ)} end; false -> - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_pm_dropped_iq, + #{host_type => HostType}, #{count => 1}), {Acc, return_action_not_allowed_error_iq(IQ)} end. @@ -565,20 +564,24 @@ get_behaviour(HostType, ArcID, LocJID=#jid{}, RemJID=#jid{}) -> DefaultMode :: atom(), AlwaysJIDs :: [jid:literal_jid()], NeverJIDs :: [jid:literal_jid()]) -> any(). set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> - mongoose_hooks:mam_set_prefs(HostType, ArcID, ArcJID, DefaultMode, - AlwaysJIDs, NeverJIDs). + Result = mongoose_hooks:mam_set_prefs(HostType, ArcID, ArcJID, DefaultMode, + AlwaysJIDs, NeverJIDs), + mongoose_instrument:execute(mod_mam_pm_set_prefs, #{host_type => HostType}, #{count => 1}), + Result. %% @doc Load settings from the database. -spec get_prefs(HostType :: host_type(), ArcID :: mod_mam:archive_id(), ArcJID :: jid:jid(), GlobalDefaultMode :: mod_mam:archive_behaviour() ) -> mod_mam:preference() | {error, Reason :: term()}. get_prefs(HostType, ArcID, ArcJID, GlobalDefaultMode) -> - mongoose_hooks:mam_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID). + Result = mongoose_hooks:mam_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID), + mongoose_instrument:execute(mod_mam_pm_get_prefs, #{host_type => HostType}, #{count => 1}), + Result. --spec remove_archive_hook(host_type(), mod_mam:archive_id(), jid:jid()) -> 'ok'. +-spec remove_archive_hook(host_type(), mod_mam:archive_id(), jid:jid()) -> ok. remove_archive_hook(HostType, ArcID, ArcJID=#jid{}) -> mongoose_hooks:mam_remove_archive(HostType, ArcID, ArcJID), - ok. + mongoose_instrument:execute(mod_mam_pm_remove_archive, #{host_type => HostType}, #{count => 1}). -spec lookup_messages(HostType :: host_type(), Params :: map()) -> {ok, mod_mam:lookup_result()} @@ -598,20 +601,29 @@ lookup_messages_without_policy_violation_check( true -> %% Use of disabled full text search {error, 'not-supported'}; false -> - StartT = erlang:monotonic_time(microsecond), - R = case maps:get(message_ids, Params, undefined) of - undefined -> - mongoose_hooks:mam_lookup_messages(HostType, - Params#{message_id => undefined}); - IDs -> - mod_mam_utils:lookup_specific_messages(HostType, Params, IDs, - fun mongoose_hooks:mam_lookup_messages/2) - end, - Diff = erlang:monotonic_time(microsecond) - StartT, - mongoose_metrics:update(HostType, [backends, ?MODULE, lookup], Diff), - R + mongoose_instrument:span(mod_mam_pm_lookup, #{host_type => HostType}, + fun perform_lookup/2, [HostType, Params], + fun(Time, Result) -> measure_lookup(Params, Time, Result) end) + end. + +perform_lookup(HostType, Params) -> + case maps:get(message_ids, Params, undefined) of + undefined -> + mongoose_hooks:mam_lookup_messages(HostType, Params#{message_id => undefined}); + IDs -> + mod_mam_utils:lookup_specific_messages(HostType, Params, IDs, + fun mongoose_hooks:mam_lookup_messages/2) end. +measure_lookup(Params, Time, {ok, {_TotalCount, _Offset, MessageRows}}) -> + M = case Params of + #{is_simple := true} -> #{simple => 1}; + #{} -> #{} + end, + M#{count => 1, time => Time, size => length(MessageRows)}; +measure_lookup(_, _, _OtherResult) -> + #{}. + archive_message_from_ct(Params = #{local_jid := JID}) -> HostType = jid_to_host_type(JID), archive_message(HostType, Params). @@ -619,11 +631,9 @@ archive_message_from_ct(Params = #{local_jid := JID}) -> -spec archive_message(host_type(), mod_mam:archive_message_params()) -> ok | {error, timeout}. archive_message(HostType, Params) -> - StartT = erlang:monotonic_time(microsecond), - R = mongoose_hooks:mam_archive_message(HostType, Params), - Diff = erlang:monotonic_time(microsecond) - StartT, - mongoose_metrics:update(HostType, [backends, ?MODULE, archive], Diff), - R. + mongoose_instrument:span(mod_mam_pm_archive_message, #{host_type => HostType}, + fun mongoose_hooks:mam_archive_message/2, [HostType, Params], + fun(Time, _Result) -> #{time => Time, count => 1} end). %% ---------------------------------------------------------------------- %% Helpers @@ -643,7 +653,7 @@ message_row_to_ext_id(#{id := MessID}) -> mod_mam_utils:mess_id_to_external_binary(MessID). handle_error_iq(HostType, Acc, _To, _Action, {error, _Reason, IQ}) -> - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_pm_dropped_iq, #{host_type => HostType}, #{count => 1}), {Acc, IQ}; handle_error_iq(_Host, Acc, _To, _Action, IQ) -> {Acc, IQ}. @@ -718,7 +728,6 @@ hooks(HostType) -> {amp_determine_strategy, HostType, fun ?MODULE:determine_amp_strategy/3, #{}, 20}, {sm_filter_offline_message, HostType, fun ?MODULE:sm_filter_offline_message/3, #{}, 50}, {get_personal_data, HostType, fun ?MODULE:get_personal_data/3, #{}, 50} - | mongoose_metrics_mam_hooks:get_mam_hooks(HostType) ]. add_iq_handlers(HostType, Opts) -> @@ -739,22 +748,19 @@ remove_iq_handlers(HostType) -> || Namespace <- [?NS_MAM_04, ?NS_MAM_06]], ok. -ensure_metrics(HostType) -> - mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, lookup], histogram), - mongoose_metrics:ensure_metric(HostType, [modMamLookups, simple], spiral), - mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, archive], histogram), - lists:foreach(fun(Name) -> - mongoose_metrics:ensure_metric(HostType, Name, spiral) - end, - spirals()). - -spirals() -> - [modMamPrefsSets, - modMamPrefsGets, - modMamArchiveRemoved, - modMamLookups, - modMamForwarded, - modMamArchived, - modMamFlushed, - modMamDropped, - modMamDroppedIQ]. +-spec instrumentation(host_type()) -> [mongoose_instrument:spec()]. +instrumentation(HostType) -> + [{mod_mam_pm_archive_message, #{host_type => HostType}, + #{metrics => #{count => spiral, time => histogram}}}, + {mod_mam_pm_lookup, #{host_type => HostType}, + #{metrics => #{count => spiral, simple => spiral, size => histogram, time => histogram}}}, + {mod_mam_pm_dropped_iq, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_pm_dropped, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_pm_remove_archive, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_pm_get_prefs, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_pm_set_prefs, #{host_type => HostType}, + #{metrics => #{count => spiral}}}]. diff --git a/src/mam/mod_mam_rdbms_arch_async.erl b/src/mam/mod_mam_rdbms_arch_async.erl index 25ad646e310..ef971127b25 100644 --- a/src/mam/mod_mam_rdbms_arch_async.erl +++ b/src/mam/mod_mam_rdbms_arch_async.erl @@ -4,12 +4,9 @@ -include("mongoose_logger.hrl"). --define(PER_MESSAGE_FLUSH_TIME, [mod_mam_rdbms_async_pool_writer, per_message_flush_time]). --define(FLUSH_TIME, [mod_mam_rdbms_async_pool_writer, flush_time]). - -behaviour(gen_mod). --export([start/2, stop/1, hooks/1, supported_features/0]). +-export([start/2, stop/1, hooks/1, instrumentation/1, supported_features/0]). -export([archive_pm_message/3, mam_archive_sync/3]). -export([flush/2]). @@ -35,8 +32,6 @@ mam_archive_sync(Result, _Params, #{host_type := HostType}) -> start(HostType, Opts) -> {PoolOpts, Extra} = make_pool_opts(pm, Opts), prepare_insert_queries(pm, Extra), - mongoose_metrics:ensure_metric(HostType, ?PER_MESSAGE_FLUSH_TIME, histogram), - mongoose_metrics:ensure_metric(HostType, ?FLUSH_TIME, histogram), mongoose_async_pools:start_pool(HostType, pm_mam, PoolOpts). -spec stop(mongooseim:host_type()) -> any(). @@ -50,6 +45,11 @@ hooks(HostType) -> {mam_archive_message, HostType, fun ?MODULE:archive_pm_message/3, #{}, 50} ]. +-spec instrumentation(mongooseim:host_type()) -> [mongoose_instrument:spec()]. +instrumentation(HostType) -> + [{mod_mam_pm_flushed, #{host_type => HostType}, + #{metrics => #{time_per_message => histogram, time => histogram, count => spiral}}}]. + -spec supported_features() -> [atom()]. supported_features() -> [dynamic_domains]. @@ -86,10 +86,13 @@ multi_name(Name, Times) -> %%% flush callbacks flush(Acc, Extra = #{host_type := HostType, queue_length := MessageCount}) -> - {FlushTime, Result} = timer:tc(fun do_flush_pm/2, [Acc, Extra]), - mongoose_metrics:update(HostType, ?PER_MESSAGE_FLUSH_TIME, round(FlushTime / MessageCount)), - mongoose_metrics:update(HostType, ?FLUSH_TIME, FlushTime), - Result. + mongoose_instrument:span(mod_mam_pm_flushed, #{host_type => HostType}, + fun do_flush_pm/2, [Acc, Extra], + fun(Time, _Result) -> + #{time => Time, + time_per_message => round(Time / MessageCount), + count => MessageCount} + end). %% mam workers callbacks do_flush_pm(Acc, #{host_type := HostType, queue_length := MessageCount, @@ -112,7 +115,7 @@ do_flush_pm(Acc, #{host_type := HostType, queue_length := MessageCount, process_batch_result({updated, _Count}, _, _, _) -> ok; process_batch_result({error, Reason}, Rows, HostType, MessageCount) -> - mongoose_metrics:update(HostType, modMamDropped, MessageCount), + mongoose_instrument:execute(mod_mam_pm_dropped, #{host_type => HostType}, #{count => MessageCount}), Keys = [ maps:with([message_id, archive_id], Row) || Row <- Rows ], ?LOG_ERROR(#{what => archive_message_failed, text => <<"archive_message batch query failed">>, @@ -125,7 +128,7 @@ process_list_results(Results, HostType) -> process_single_result({{updated, _Count}, _}, _HostType) -> ok; process_single_result({{error, Reason}, #{message_id := MsgId, archive_id := ArcId}}, HostType) -> - mongoose_metrics:update(HostType, modMamDropped, 1), + mongoose_instrument:execute(mod_mam_pm_dropped, #{host_type => HostType}, #{count => 1}), ?LOG_ERROR(#{what => archive_message_failed, text => <<"archive_message batch query failed">>, message_id => MsgId, archive_id => ArcId, reason => Reason}). From c754e1e80263fb5b54a3635851f38f3c0aaba237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 15 Feb 2024 09:57:48 +0100 Subject: [PATCH 3/9] Remove unused module All instrumentation is done with mongoose_instrument now. --- src/metrics/mongoose_metrics_mam_hooks.erl | 173 --------------------- 1 file changed, 173 deletions(-) delete mode 100644 src/metrics/mongoose_metrics_mam_hooks.erl diff --git a/src/metrics/mongoose_metrics_mam_hooks.erl b/src/metrics/mongoose_metrics_mam_hooks.erl deleted file mode 100644 index ece05900fdc..00000000000 --- a/src/metrics/mongoose_metrics_mam_hooks.erl +++ /dev/null @@ -1,173 +0,0 @@ -%%%------------------------------------------------------------------- -%%% @author Piotr Nosek -%%% @doc MAM hooks for general metrics -%%% -%%% @end -%%% Created : 13 Feb 2017 by Piotr Nosek -%%%------------------------------------------------------------------- --module(mongoose_metrics_mam_hooks). - --include("mongoose.hrl"). --include("jlib.hrl"). - --export([get_mam_hooks/1, - get_mam_muc_hooks/1]). - -%%------------------- -%% Internal exports -%%------------------- --export([mam_get_prefs/3, - mam_set_prefs/3, - mam_remove_archive/3, - mam_lookup_messages/3, - mam_archive_message/3, - mam_flush_messages/3, - mam_muc_get_prefs/3, - mam_muc_set_prefs/3, - mam_muc_remove_archive/3, - mam_muc_lookup_messages/3, - mam_muc_archive_message/3, - mam_muc_flush_messages/3]). - -%%------------------- -%% Implementation -%%------------------- - -%% @doc Here will be declared which hooks should be registered when mod_mam_pm is enabled. --spec get_mam_hooks(mongooseim:host_type()) -> gen_hook:hook_list(). -get_mam_hooks(HostType) -> - [ - {mam_set_prefs, HostType, fun ?MODULE:mam_set_prefs/3, #{}, 50}, - {mam_get_prefs, HostType, fun ?MODULE:mam_get_prefs/3, #{}, 50}, - {mam_archive_message, HostType, fun ?MODULE:mam_archive_message/3, #{}, 50}, - {mam_remove_archive, HostType, fun ?MODULE:mam_remove_archive/3, #{}, 50}, - {mam_lookup_messages, HostType, fun ?MODULE:mam_lookup_messages/3, #{}, 100}, - {mam_flush_messages, HostType, fun ?MODULE:mam_flush_messages/3, #{}, 50} - ]. - -%% @doc Here will be declared which hooks should be registered when mod_mam_muc is enabled. --spec get_mam_muc_hooks(mongooseim:host_type()) -> gen_hook:hook_list(). -get_mam_muc_hooks(HostType) -> - [ - {mam_muc_set_prefs, HostType, fun ?MODULE:mam_muc_set_prefs/3, #{}, 50}, - {mam_muc_get_prefs, HostType, fun ?MODULE:mam_muc_get_prefs/3, #{}, 50}, - {mam_muc_archive_message, HostType, fun ?MODULE:mam_muc_archive_message/3, #{}, 50}, - {mam_muc_remove_archive, HostType, fun ?MODULE:mam_muc_remove_archive/3, #{}, 50}, - {mam_muc_lookup_messages, HostType, fun ?MODULE:mam_muc_lookup_messages/3, #{}, 100}, - {mam_muc_flush_messages, HostType, fun ?MODULE:mam_muc_flush_messages/3, #{}, 50} - ]. - --spec mam_get_prefs(Acc, Params, Extra) -> {ok, Acc} when - Acc :: mod_mam:preference() | {error, Reason :: term()}, - Params :: map(), - Extra :: gen_hook:extra(). -mam_get_prefs(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamPrefsGets, 1), - {ok, Result}. - --spec mam_set_prefs(Acc, Params, Extra) -> {ok, Acc} when - Acc :: term(), - Params :: map(), - Extra :: gen_hook:extra(). -mam_set_prefs(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamPrefsSets, 1), - {ok, Result}. - --spec mam_remove_archive(Acc, Params, Extra) -> {ok, Acc} when - Acc :: term(), - Params :: #{archive_id := mod_mam:archive_id() | undefined, owner := jid:jid()}, - Extra :: gen_hook:extra(). -mam_remove_archive(Acc, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamArchiveRemoved, 1), - {ok, Acc}. - --spec mam_lookup_messages(Acc, Params, Extra) -> {ok, Acc} when - Acc :: {ok, mod_mam:lookup_result()} | {error, term()}, - Params :: mam_iq:lookup_params(), - Extra :: gen_hook:extra(). -mam_lookup_messages({ok, {_TotalCount, _Offset, MessageRows}} = Result, - #{is_simple := IsSimple}, - #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamForwarded, length(MessageRows)), - mongoose_metrics:update(HostType, modMamLookups, 1), - case IsSimple of - true -> - mongoose_metrics:update(HostType, [modMamLookups, simple], 1); - _ -> - ok - end, - {ok, Result}; -mam_lookup_messages(Result = {error, _}, _Params, _Extra) -> - {ok, Result}. - --spec mam_archive_message(Acc, Params, Extra) -> {ok, Acc} when - Acc :: ok | {error, timeout}, - Params :: mod_mam:archive_message_params(), - Extra :: gen_hook:extra(). -mam_archive_message(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamArchived, 1), - {ok, Result}. - --spec mam_flush_messages(Acc, Params, Extra) -> {ok, Acc} when - Acc :: ok, - Params :: #{count := integer()}, - Extra :: gen_hook:extra(). -mam_flush_messages(Acc, #{count := MessageCount}, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamFlushed, MessageCount), - {ok, Acc}. - -%% ---------------------------------------------------------------------------- -%% mod_mam_muc - --spec mam_muc_get_prefs(Acc, Params, Extra) -> {ok, Acc} when - Acc :: mod_mam:preference() | {error, Reason :: term()}, - Params :: map(), - Extra :: gen_hook:extra(). -mam_muc_get_prefs(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamPrefsGets, 1), - {ok, Result}. - --spec mam_muc_set_prefs(Acc, Params, Extra) -> {ok, Acc} when - Acc :: term(), - Params :: map(), - Extra :: gen_hook:extra(). -mam_muc_set_prefs(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamPrefsSets, 1), - {ok, Result}. - --spec mam_muc_remove_archive(Acc, Params, Extra) -> {ok, Acc} when - Acc :: term(), - Params :: #{archive_id := mod_mam:archive_id() | undefined, room := jid:jid()}, - Extra :: gen_hook:extra(). -mam_muc_remove_archive(Acc, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamArchiveRemoved, 1), - {ok, Acc}. - --spec mam_muc_lookup_messages(Acc, Params, Extra) -> {ok, Acc} when - Acc :: {ok, mod_mam:lookup_result()} | {error, term()}, - Params :: mam_iq:lookup_params(), - Extra :: gen_hook:extra(). -mam_muc_lookup_messages({ok, {_TotalCount, _Offset, MessageRows}} = Result, - _Params, - #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamForwarded, length(MessageRows)), - mongoose_metrics:update(HostType, modMucMamLookups, 1), - {ok, Result}; -mam_muc_lookup_messages(Result = {error, _}, _Params, _Extra) -> - {ok, Result}. - --spec mam_muc_archive_message(Acc, Params, Extra) -> {ok, Acc} when - Acc :: ok | {error, timeout}, - Params :: mod_mam:archive_message_params(), - Extra :: gen_hook:extra(). -mam_muc_archive_message(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamArchived, 1), - {ok, Result}. - --spec mam_muc_flush_messages(Acc, Params, Extra) -> {ok, Acc} when - Acc :: ok, - Params :: map(), - Extra :: gen_hook:extra(). -mam_muc_flush_messages(Acc, #{count := MessageCount}, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamFlushed, MessageCount), - {ok, Acc}. From f36032a0ce61613f0cb7893d363e6f8ef8329b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 15 Feb 2024 10:56:09 +0100 Subject: [PATCH 4/9] Update ignore_xref --- src/instrument/mongoose_instrument.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/instrument/mongoose_instrument.erl b/src/instrument/mongoose_instrument.erl index ea3df8799f3..ea5bc1c0f50 100644 --- a/src/instrument/mongoose_instrument.erl +++ b/src/instrument/mongoose_instrument.erl @@ -13,7 +13,7 @@ %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, code_change/3, handle_info/2, terminate/2]). --ignore_xref([start_link/0, set_up/3, tear_down/2, span/4, span/5, execute/3]). +-ignore_xref([start_link/0, set_up/3, tear_down/2, span/4]). -include("mongoose.hrl"). -include("mongoose_config_spec.hrl"). From 57626f58ff38ac38f6ac9beffd8a511f53c47598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 14 Mar 2024 11:07:07 +0100 Subject: [PATCH 5/9] Put more information in MAM instrumentation Purpose: - More information in logs - Ability tomatch specific events in big tests --- src/instrument/mongoose_instrument.erl | 2 +- src/mam/mod_mam_muc.erl | 25 +++++++++++++++---------- src/mam/mod_mam_pm.erl | 16 ++++++++++------ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/instrument/mongoose_instrument.erl b/src/instrument/mongoose_instrument.erl index ea5bc1c0f50..2e0a902c672 100644 --- a/src/instrument/mongoose_instrument.erl +++ b/src/instrument/mongoose_instrument.erl @@ -21,7 +21,7 @@ -type event_name() :: atom(). -type labels() :: #{host_type => mongooseim:host_type()}. % to be extended -type metrics() :: #{atom() => spiral | histogram}. % to be extended --type measurements() :: #{atom() => integer() | atom() | binary()}. +-type measurements() :: #{atom() => term()}. -type spec() :: {event_name(), labels(), config()}. -type config() :: #{metrics => metrics()}. % to be extended -type handler_key() :: atom(). % key in the `instrumentation' section of the config file diff --git a/src/mam/mod_mam_muc.erl b/src/mam/mod_mam_muc.erl index 8e6e898deeb..cb3c31d0db8 100644 --- a/src/mam/mod_mam_muc.erl +++ b/src/mam/mod_mam_muc.erl @@ -227,7 +227,7 @@ room_process_mam_iq(Acc, From, To, IQ, #{host_type := HostType}) -> handle_mam_iq(HostType, Action, From, To, IQ)); {error, max_delay_reached} -> mongoose_instrument:execute(mod_mam_muc_dropped_iq, - #{host_type => HostType}, #{count => 1}), + #{host_type => HostType}, #{acc => Acc, count => 1}), {Acc, return_max_delay_reached_error_iq(IQ)} end; {error, Reason} -> @@ -488,7 +488,8 @@ get_behaviour(HostType, ArcID, LocJID = #jid{}, RemJID = #jid{}) -> set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> Result = mongoose_hooks:mam_muc_set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs), - mongoose_instrument:execute(mod_mam_muc_set_prefs, #{host_type => HostType}, #{count => 1}), + mongoose_instrument:execute(mod_mam_muc_set_prefs, #{host_type => HostType}, + #{jid => ArcJID, count => 1}), Result. %% @doc Load settings from the database. @@ -497,14 +498,16 @@ set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> -> mod_mam:preference() | {error, Reason :: term()}. get_prefs(HostType, ArcID, ArcJID, GlobalDefaultMode) -> Result = mongoose_hooks:mam_muc_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID), - mongoose_instrument:execute(mod_mam_muc_get_prefs, #{host_type => HostType}, #{count => 1}), + mongoose_instrument:execute(mod_mam_muc_get_prefs, #{host_type => HostType}, + #{jid => ArcJID, count => 1}), Result. -spec remove_archive(host_type(), mod_mam:archive_id() | undefined, jid:jid()) -> ok. remove_archive(HostType, ArcID, ArcJID = #jid{}) -> mongoose_hooks:mam_muc_remove_archive(HostType, ArcID, ArcJID), - mongoose_instrument:execute(mod_mam_muc_remove_archive, #{host_type => HostType}, #{count => 1}). + mongoose_instrument:execute(mod_mam_muc_remove_archive, #{host_type => HostType}, + #{jid => ArcJID, count => 1}). %% See description in mod_mam_pm. -spec lookup_messages(HostType :: host_type(), Params :: map()) -> @@ -526,7 +529,8 @@ lookup_messages_without_policy_violation_check(HostType, {error, 'not-supported'}; false -> mongoose_instrument:span(mod_mam_muc_lookup, #{host_type => HostType}, - fun perform_lookup/2, [HostType, Params], fun measure_lookup/2) + fun perform_lookup/2, [HostType, Params], + fun(Time, Result) -> measure_lookup(Params, Time, Result) end) end. perform_lookup(HostType, Params) -> @@ -538,9 +542,9 @@ perform_lookup(HostType, Params) -> fun mongoose_hooks:mam_muc_lookup_messages/2) end. -measure_lookup(Time, {ok, {_TotalCount, _Offset, MessageRows}}) -> - #{count => 1, time => Time, size => length(MessageRows)}; -measure_lookup(_, _OtherResult) -> +measure_lookup(Params, Time, {ok, {_TotalCount, _Offset, MessageRows}}) -> + #{params => Params, count => 1, time => Time, size => length(MessageRows)}; +measure_lookup(_Params, _Time, _OtherResult) -> #{}. archive_message_for_ct(Params = #{local_jid := RoomJid}) -> @@ -551,7 +555,7 @@ archive_message_for_ct(Params = #{local_jid := RoomJid}) -> archive_message(HostType, Params) -> mongoose_instrument:span(mod_mam_muc_archive_message, #{host_type => HostType}, fun mongoose_hooks:mam_muc_archive_message/2, [HostType, Params], - fun(Time, _Result) -> #{time => Time, count => 1} end). + fun(Time, _Result) -> #{params => Params, time => Time, count => 1} end). %% ---------------------------------------------------------------------- %% Helpers @@ -594,7 +598,8 @@ message_row_to_ext_id(#{id := MessID}) -> -spec handle_error_iq(mongoose_acc:t(), host_type(), jid:jid(), atom(), {error, term(), jlib:iq()} | jlib:iq() | ignore) -> {mongoose_acc:t(), jlib:iq() | ignore}. handle_error_iq(Acc, HostType, _To, _Action, {error, _Reason, IQ}) -> - mongoose_instrument:execute(mod_mam_muc_dropped_iq, #{host_type => HostType}, #{count => 1}), + mongoose_instrument:execute(mod_mam_muc_dropped_iq, #{host_type => HostType}, + #{acc => Acc, count => 1}), {Acc, IQ}; handle_error_iq(Acc, _HostType, _To, _Action, IQ) -> {Acc, IQ}. diff --git a/src/mam/mod_mam_pm.erl b/src/mam/mod_mam_pm.erl index 81d353e8660..32c2bc2b765 100644 --- a/src/mam/mod_mam_pm.erl +++ b/src/mam/mod_mam_pm.erl @@ -566,7 +566,8 @@ get_behaviour(HostType, ArcID, LocJID=#jid{}, RemJID=#jid{}) -> set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> Result = mongoose_hooks:mam_set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs), - mongoose_instrument:execute(mod_mam_pm_set_prefs, #{host_type => HostType}, #{count => 1}), + mongoose_instrument:execute(mod_mam_pm_set_prefs, #{host_type => HostType}, + #{jid => ArcJID, count => 1}), Result. %% @doc Load settings from the database. @@ -575,13 +576,15 @@ set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> ) -> mod_mam:preference() | {error, Reason :: term()}. get_prefs(HostType, ArcID, ArcJID, GlobalDefaultMode) -> Result = mongoose_hooks:mam_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID), - mongoose_instrument:execute(mod_mam_pm_get_prefs, #{host_type => HostType}, #{count => 1}), + mongoose_instrument:execute(mod_mam_pm_get_prefs, #{host_type => HostType}, + #{jid => ArcJID, count => 1}), Result. -spec remove_archive_hook(host_type(), mod_mam:archive_id(), jid:jid()) -> ok. remove_archive_hook(HostType, ArcID, ArcJID=#jid{}) -> mongoose_hooks:mam_remove_archive(HostType, ArcID, ArcJID), - mongoose_instrument:execute(mod_mam_pm_remove_archive, #{host_type => HostType}, #{count => 1}). + mongoose_instrument:execute(mod_mam_pm_remove_archive, #{host_type => HostType}, + #{jid => ArcJID, count => 1}). -spec lookup_messages(HostType :: host_type(), Params :: map()) -> {ok, mod_mam:lookup_result()} @@ -620,7 +623,7 @@ measure_lookup(Params, Time, {ok, {_TotalCount, _Offset, MessageRows}}) -> #{is_simple := true} -> #{simple => 1}; #{} -> #{} end, - M#{count => 1, time => Time, size => length(MessageRows)}; + M#{params => Params, count => 1, time => Time, size => length(MessageRows)}; measure_lookup(_, _, _OtherResult) -> #{}. @@ -633,7 +636,7 @@ archive_message_from_ct(Params = #{local_jid := JID}) -> archive_message(HostType, Params) -> mongoose_instrument:span(mod_mam_pm_archive_message, #{host_type => HostType}, fun mongoose_hooks:mam_archive_message/2, [HostType, Params], - fun(Time, _Result) -> #{time => Time, count => 1} end). + fun(Time, _Result) -> #{params => Params, time => Time, count => 1} end). %% ---------------------------------------------------------------------- %% Helpers @@ -653,7 +656,8 @@ message_row_to_ext_id(#{id := MessID}) -> mod_mam_utils:mess_id_to_external_binary(MessID). handle_error_iq(HostType, Acc, _To, _Action, {error, _Reason, IQ}) -> - mongoose_instrument:execute(mod_mam_pm_dropped_iq, #{host_type => HostType}, #{count => 1}), + mongoose_instrument:execute(mod_mam_pm_dropped_iq, #{host_type => HostType}, + #{acc => Acc, count => 1}), {Acc, IQ}; handle_error_iq(_Host, Acc, _To, _Action, IQ) -> {Acc, IQ}. From 581e96cded55208ac80e6ec039ca013b5306f222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 15 Feb 2024 09:59:56 +0100 Subject: [PATCH 6/9] Don't remove room archive if mam_muc is disabled The checks in mongoose_instrument would cause an exception. --- big_tests/tests/muc_helper.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/big_tests/tests/muc_helper.erl b/big_tests/tests/muc_helper.erl index 783a1d4ad4d..9ea1305d3f1 100644 --- a/big_tests/tests/muc_helper.erl +++ b/big_tests/tests/muc_helper.erl @@ -317,7 +317,12 @@ story_with_room(Config, RoomOpts, [{Owner, _}|_] = UserSpecs, StoryFun) -> StoryFun2 = fun(Args) -> apply(StoryFun, [Config2 | Args]) end, escalus_story:story_with_client_list(Config2, UserSpecs, StoryFun2) after - mam_helper:destroy_room(Config2) + case dynamic_modules:get_current_modules(domain_helper:host_type()) of + #{mod_mam_muc := _} -> + mam_helper:destroy_room(Config2); + #{} -> + ok + end end. %%-------------------------------------------------------------------- From a5fa50d04dce2e9f11878d749fd00e6cd1ee7084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 15 Feb 2024 09:59:23 +0100 Subject: [PATCH 7/9] Update MAM tests after renaming metrics --- big_tests/tests/mam_SUITE.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/big_tests/tests/mam_SUITE.erl b/big_tests/tests/mam_SUITE.erl index f2ba8757508..b6040f53a0a 100644 --- a/big_tests/tests/mam_SUITE.erl +++ b/big_tests/tests/mam_SUITE.erl @@ -825,7 +825,9 @@ init_metrics(muc_archive_request, Config) -> %% Check that metric is incremented on MUC flushed case ?config(configuration, Config) of rdbms_async_pool -> - MongooseMetrics = [{['_', 'modMucMamFlushed'], changed}], + HostType = domain_helper:host_type(mim), + HostTypePrefix = domain_helper:make_metrics_prefix(HostType), + MongooseMetrics = [{[HostTypePrefix, mod_mam_muc_flushed, count], changed}], [{mongoose_metrics, MongooseMetrics} | Config]; _ -> Config @@ -3303,7 +3305,7 @@ metric_incremented_on_archive_request(ConfigIn) -> end, HostType = domain_helper:host_type(mim), HostTypePrefix = domain_helper:make_metrics_prefix(HostType), - MongooseMetrics = [{[HostTypePrefix, backends, mod_mam_pm, lookup], changed}], + MongooseMetrics = [{[HostTypePrefix, mod_mam_pm_lookup, count], changed}], Config = [{mongoose_metrics, MongooseMetrics} | ConfigIn], escalus_fresh:story(Config, [{alice, 1}], F). From 5d5fd5a2488d46b18e57c8d65fabe893e2eed75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Fri, 15 Mar 2024 18:37:03 +0100 Subject: [PATCH 8/9] Test instrumentation in mam_SUITE Only events, that are already triggered, are instrumented. Archive deletion was triggered only in end_per_testcase, so I decided to add it to two test cases, which already tested instrumentation. The plan is to extend the tests to all events in a separate task. --- big_tests/tests/mam_SUITE.erl | 60 +++++++++++++++++++++++++++++++--- big_tests/tests/mam_helper.erl | 47 ++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/big_tests/tests/mam_SUITE.erl b/big_tests/tests/mam_SUITE.erl index b6040f53a0a..4d660f438b9 100644 --- a/big_tests/tests/mam_SUITE.erl +++ b/big_tests/tests/mam_SUITE.erl @@ -106,7 +106,12 @@ muc_light_host/0, host_type/0, config_opts/1, - stanza_metadata_request/0 + stanza_metadata_request/0, + assert_archive_message_event/2, + assert_lookup_event/2, + assert_flushed_event_if_async/2, + assert_dropped_iq_event/2, + assert_event_with_jid/2 ]). -import(muc_light_helper, @@ -285,6 +290,7 @@ mam_cases() -> [mam_service_discovery, mam_service_discovery_to_client_bare_jid, mam_service_discovery_to_different_client_bare_jid_results_in_error, + archive_is_instrumented, easy_archive_request, easy_archive_request_for_the_receiver, message_sent_to_yourself, @@ -359,7 +365,8 @@ muc_cases() -> [muc_service_discovery | muc_cases_with_room()]. muc_cases_with_room() -> - [muc_archive_request, + [muc_archive_is_instrumented, + muc_archive_request, muc_multiple_devices, muc_protected_message, muc_deny_protected_room_access, @@ -489,12 +496,19 @@ suite() -> require_rpc_nodes([mim]) ++ escalus:suite(). init_per_suite(Config) -> + instrument_helper:start(instrument_helper:declared_events(instrumented_modules())), muc_helper:load_muc(), mam_helper:prepare_for_suite( increase_limits( delete_users([{escalus_user_db, {module, escalus_ejabberd}} | escalus:init_per_suite(Config)]))). +instrumented_modules() -> + case mongoose_helper:is_rdbms_enabled(host_type()) of + true -> [mod_mam_rdbms_arch_async, mod_mam_muc_rdbms_arch_async]; + false -> [] + end ++ [mod_mam_pm, mod_mam_muc]. + end_per_suite(Config) -> muc_helper:unload_muc(), %% Next function creates a lot of sessions... @@ -503,7 +517,8 @@ end_per_suite(Config) -> mongoose_helper:kick_everyone(), %% so we don't have sessions anymore and other tests will not fail mongoose_helper:restore_config(Config), - escalus:end_per_suite(Config). + escalus:end_per_suite(Config), + instrument_helper:stop(). user_names() -> [alice, bob, kate, carol]. @@ -1057,6 +1072,19 @@ same_stanza_id(Config) -> end, escalus_fresh:story(Config, [{alice, 1}, {bob, 1}], F). +archive_is_instrumented(Config) -> + F = fun(Alice, Bob) -> + escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"OH, HAI!">>)), + escalus:wait_for_stanza(Bob), + assert_archive_message_event(mod_mam_pm_archive_message, escalus_utils:get_jid(Alice)), + mam_helper:wait_for_archive_size(Alice, 1), + assert_flushed_event_if_async(mod_mam_pm_flushed, Config), + {S, U} = {escalus_utils:get_server(Alice), escalus_utils:get_username(Alice)}, + mam_helper:delete_archive(S, U), + assert_event_with_jid(mod_mam_pm_remove_archive, escalus_utils:get_short_jid(Alice)) + end, + escalus_fresh:story(Config, [{alice, 1}, {bob, 1}], F). + %% Querying the archive for messages easy_archive_request(Config) -> P = ?config(props, Config), @@ -1072,6 +1100,7 @@ easy_archive_request(Config) -> mam_helper:wait_for_archive_size(Alice, 1), escalus:send(Alice, stanza_archive_request(P, <<"q1">>)), Res = wait_archive_respond(Alice), + assert_lookup_event(mod_mam_pm_lookup, escalus_utils:get_jid(Alice)), assert_respond_size(1, Res), assert_respond_query_id(P, <<"q1">>, parse_result_iq(Res)), ok @@ -2086,6 +2115,26 @@ test_retract_muc_message(Config) -> end, escalus:story(Config, [{alice, 1}, {bob, 1}], F). +muc_archive_is_instrumented(Config) -> + F = fun(Alice, Bob) -> + Room = ?config(room, Config), + RoomAddr = room_address(Room), + Text = <<"Hi, Bob!">>, + escalus:send(Alice, stanza_muc_enter_room(Room, nick(Alice))), + escalus:send(Bob, stanza_muc_enter_room(Room, nick(Bob))), + escalus:send(Alice, escalus_stanza:groupchat_to(RoomAddr, Text)), + + %% Bob received presences, the room's subject and the message. + escalus:wait_for_stanzas(Bob, 4), + assert_archive_message_event(mod_mam_muc_archive_message, RoomAddr), + maybe_wait_for_archive(Config), + assert_flushed_event_if_async(mod_mam_muc_flushed, Config), + + mam_helper:delete_room_archive(muc_host(), ?config(room, Config)), + assert_event_with_jid(mod_mam_muc_remove_archive, RoomAddr) + end, + escalus:story(Config, [{alice, 1}, {bob, 1}], F). + muc_archive_request(Config) -> P = ?config(props, Config), F = fun(Alice, Bob) -> @@ -2141,6 +2190,7 @@ muc_archive_request(Config) -> ?assert_equal(escalus_utils:jid_to_lower(RoomAddr), By), ?assert_equal_extra(true, has_x_user_element(ArcMsg), [{forwarded_message, ArcMsg}]), + assert_lookup_event(mod_mam_muc_lookup, escalus_utils:get_jid(Bob)), ok end, escalus:story(Config, [{alice, 1}, {bob, 1}], F). @@ -2991,7 +3041,7 @@ server_returns_item_not_found_for_nonexistent_id(Config, RSM, StanzaID, Conditio Res = escalus:wait_for_stanza(Alice), escalus:assert(is_iq_error, [IQ], Res), escalus:assert(is_error, Condition, Res), - ok + assert_dropped_iq_event(Config, escalus_utils:get_jid(Alice)) end, parallel_story(Config, [{alice, 1}], F). @@ -3184,9 +3234,11 @@ prefs_set_request(Config) -> [<<"montague@montague.net">>], mam_ns_binary())), ReplySet = escalus:wait_for_stanza(Alice), + assert_event_with_jid(mod_mam_pm_set_prefs, escalus_utils:get_short_jid(Alice)), escalus:send(Alice, stanza_prefs_get_request(mam_ns_binary())), ReplyGet = escalus:wait_for_stanza(Alice), + assert_event_with_jid(mod_mam_pm_get_prefs, escalus_utils:get_short_jid(Alice)), ResultIQ1 = parse_prefs_result_iq(ReplySet), ResultIQ2 = parse_prefs_result_iq(ReplyGet), diff --git a/big_tests/tests/mam_helper.erl b/big_tests/tests/mam_helper.erl index 01668a093ee..fa7696439fa 100644 --- a/big_tests/tests/mam_helper.erl +++ b/big_tests/tests/mam_helper.erl @@ -1336,3 +1336,50 @@ rewrite_nodename($.) -> <<"-">>; rewrite_nodename(X) -> <>. assert_list_size(N, List) when N =:= length(List) -> List. + +%% Assertions for instrumentation events + +assert_archive_message_event(EventName, BinJid) -> + assert_event(EventName, + fun(#{count := 1, time := T, params := #{local_jid := LocalJid}}) -> + eq_bjid(LocalJid, BinJid) andalso pos_int(T) + end). + +assert_lookup_event(EventName, BinJid) -> + assert_event(EventName, + fun(#{count := 1, size := 1, time := T, params := #{caller_jid := CallerJid}}) -> + eq_bjid(CallerJid, BinJid) andalso pos_int(T) + end). + +%% The event might originate from a different test case running in parallel, +%% but there is no easy way around it other than adding all flushed messages to measurements. +assert_flushed_event_if_async(EventName, Config) -> + case ?config(configuration, Config) of + C when C =:= rdbms_async_pool; + C =:= rdbms_async_cache -> + assert_event(EventName, + fun(#{count := Count, time := T, time_per_message := T1}) -> + pos_int(Count) andalso pos_int(T) andalso pos_int(T1) andalso T >= T1 + end); + _ -> + ok + end. + +assert_dropped_iq_event(Config, BinJid) -> + EventName = case ?config(room, Config) of + undefined -> mod_mam_pm_dropped_iq; + _ -> mod_mam_muc_dropped_iq + end, + assert_event(EventName, fun(#{acc := #{stanza := #{from_jid := FromJid}}}) -> + eq_bjid(FromJid, BinJid) + end). + +assert_event_with_jid(EventName, BinJid) -> + assert_event(EventName, fun(#{count := 1, jid := Jid}) -> eq_bjid(Jid, BinJid) end). + +assert_event(EventName, F) -> + instrument_helper:assert(EventName, #{host_type => host_type()}, F). + +pos_int(T) -> is_integer(T) andalso T > 0. + +eq_bjid(Jid, BinJid) -> Jid =:= jid:from_binary(BinJid). From ce475da425a13107be62d6817cba2e88ae6fb0a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Fri, 15 Mar 2024 18:48:20 +0100 Subject: [PATCH 9/9] Remove metrics tests obsoleted by the new instrumentation checks Also: get rid of a debug printout --- big_tests/tests/mam_SUITE.erl | 64 ++--------------------------------- 1 file changed, 2 insertions(+), 62 deletions(-) diff --git a/big_tests/tests/mam_SUITE.erl b/big_tests/tests/mam_SUITE.erl index 4d660f438b9..a03a2d4a036 100644 --- a/big_tests/tests/mam_SUITE.erl +++ b/big_tests/tests/mam_SUITE.erl @@ -283,8 +283,7 @@ chat_markers_cases() -> dont_archive_chat_markers]. mam_metrics_cases() -> - [metric_incremented_on_archive_request, - metric_incremented_when_store_message]. + [metric_incremented_when_store_message]. mam_cases() -> [mam_service_discovery, @@ -341,9 +340,7 @@ muc_text_search_cases() -> archived_cases() -> [archived, - filter_forwarded, - metrics_incremented_for_async_pools - ]. + filter_forwarded]. stanzaid_cases() -> [message_with_stanzaid, @@ -724,9 +721,6 @@ init_steps() -> [fun init_users/2, fun init_archive/2, fun start_room/2, fun init_metrics/2, fun escalus:init_per_testcase/2]. -maybe_skip(metrics_incremented_for_async_pools, Config) -> - skip_if(?config(configuration, Config) =/= rdbms_async_pool, - "Not an async-pool test"); maybe_skip(C, Config) when C =:= retract_message; C =:= retract_wrong_message; C =:= ignore_bad_retraction; @@ -802,9 +796,6 @@ fresh_users(C) -> false -> [] end. -init_archive(C, Config) when C =:= metrics_incremented_for_async_pools; - C =:= metric_incremented_when_store_message -> - clean_archives(Config); init_archive(C, Config) when ?requires_pm_archive(C) -> bootstrap_archive(Config); init_archive(C, Config) when ?requires_muc_archive(C) -> @@ -836,17 +827,6 @@ init_metrics(metric_incremented_when_store_message, ConfigIn) -> _ -> ConfigIn end; -init_metrics(muc_archive_request, Config) -> - %% Check that metric is incremented on MUC flushed - case ?config(configuration, Config) of - rdbms_async_pool -> - HostType = domain_helper:host_type(mim), - HostTypePrefix = domain_helper:make_metrics_prefix(HostType), - MongooseMetrics = [{[HostTypePrefix, mod_mam_muc_flushed, count], changed}], - [{mongoose_metrics, MongooseMetrics} | Config]; - _ -> - Config - end; init_metrics(_CaseName, Config) -> Config. @@ -1866,7 +1846,6 @@ archived(Config) -> %% Bob receives a message. Msg = escalus:wait_for_stanza(Bob), - try StanzaId = exml_query:subelement(Msg, <<"stanza-id">>), %% JID of the archive (i.e. where the client would send queries to) By = exml_query:attr(StanzaId, <<"by">>), @@ -1882,10 +1861,6 @@ archived(Config) -> #forwarded_message{result_id=ArcId} = parse_forwarded_message(ArcMsg), ?assert_equal(Id, ArcId), ok - catch Class:Reason:StackTrace -> - ct:pal("Msg ~p", [Msg]), - erlang:raise(Class, Reason, StackTrace) - end end, %% Made fresh in init_per_testcase escalus:story(Config, [{alice, 1}, {bob, 1}], F). @@ -3346,41 +3321,6 @@ discover_features(Config, Client, Service) -> ?assert_equal(message_retraction_is_enabled(Config), escalus_pred:has_feature(retract_tombstone_ns(), Stanza)). -metric_incremented_on_archive_request(ConfigIn) -> - P = ?config(props, ConfigIn), - F = fun(Alice) -> - escalus:send(Alice, stanza_archive_request(P, <<"metric_q1">>)), - Res = wait_archive_respond(Alice), - assert_respond_size(0, Res), - assert_respond_query_id(P, <<"metric_q1">>, parse_result_iq(Res)), - ok - end, - HostType = domain_helper:host_type(mim), - HostTypePrefix = domain_helper:make_metrics_prefix(HostType), - MongooseMetrics = [{[HostTypePrefix, mod_mam_pm_lookup, count], changed}], - Config = [{mongoose_metrics, MongooseMetrics} | ConfigIn], - escalus_fresh:story(Config, [{alice, 1}], F). - -metrics_incremented_for_async_pools(Config) -> - OldValue = get_mongoose_async_metrics(), - archived(Config), - Validator = fun(NewValue) -> OldValue =/= NewValue end, - mongoose_helper:wait_until( - fun get_mongoose_async_metrics/0, - Validator, #{name => ?FUNCTION_NAME}). - -get_mongoose_async_metrics() -> - HostType = domain_helper:host_type(mim), - HostTypePrefix = domain_helper:make_metrics_prefix(HostType), - #{batch_flushes => get_mongoose_async_metrics(HostTypePrefix, batch_flushes), - timed_flushes => get_mongoose_async_metrics(HostTypePrefix, timed_flushes)}. - -get_mongoose_async_metrics(HostTypePrefix, MetricName) -> - Metric = [HostTypePrefix, mongoose_async_pools, pm_mam, MetricName], - {ok, Value} = rpc(mim(), mongoose_metrics, get_metric_value, [Metric]), - {value, Count} = lists:keyfind(value, 1, Value), - Count. - metric_incremented_when_store_message(Config) -> archived(Config).