Merge pull request #488 from esl/auth-metrics

Auth metrics

apps/ejabberd/src/ejabberd_app.erl

@@ -58,7 +58,6 @@ start(normal, _Args) ->
     maybe_start_alarms(),
     connect_nodes(),
     {ok, _} = Sup = ejabberd_sup:start_link(),
-    mongoose_metrics:init(),
     ejabberd_system_monitor:add_handler(),
     ejabberd_rdbms:start(),
     mongoose_riak:start(),
@@ -68,6 +67,7 @@ start(normal, _Args) ->
     %%ejabberd_debug:eprof_start(),
     %%ejabberd_debug:fprof_start(),
     start_modules(),
+    mongoose_metrics:init(),
     ejabberd_listener:start_listeners(),
     ejabberd_admin:start(),
     ?INFO_MSG("ejabberd ~s is started in the node ~p", [?VERSION, node()]),

apps/ejabberd/src/ejabberd_auth.erl

@@ -68,6 +68,7 @@
                     | ejabberd_auth_ldap
                     | ejabberd_auth_odbc.
 
+-define(METRIC(Host, Name), [backends, auth, Host, Name]).
 %%%----------------------------------------------------------------------
 %%% API
 %%%----------------------------------------------------------------------
@@ -77,6 +78,7 @@ start() ->
 
 -spec start(Host :: ejabberd:server()) -> 'ok'.
 start(Host) ->
+    ensure_metrics(Host),
     lists:foreach(
       fun(M) ->
               M:start(Host)
@@ -196,12 +198,17 @@ do_check_password_with_authmodule(LUser, LServer, Password, Digest, DigestGen) -
                           ) -> 'false' | {'true', authmodule()}.
 check_password_loop([], _Args) ->
     false;
-check_password_loop([AuthModule | AuthModules], Args) ->
+check_password_loop(AuthModules, [_, LServer | _] = Args) ->
+    timed_call(LServer, check_password, fun do_check_password_loop/2, [AuthModules, Args]).
+
+do_check_password_loop([], _) ->
+    false;
+do_check_password_loop([AuthModule | AuthModules], Args) ->
     case apply(AuthModule, check_password, Args) of
         true ->
             {true, AuthModule};
         false ->
-            check_password_loop(AuthModules, Args)
+            do_check_password_loop(AuthModules, Args)
     end.
 
 -spec check_digest(binary(), fun(), binary(), binary()) -> boolean().
@@ -266,23 +273,26 @@ do_try_register_if_does_not_exist(true, _, _, _) ->
 do_try_register_if_does_not_exist(_, LUser, LServer, Password) ->
     case lists:member(LServer, ?MYHOSTS) of
         true ->
-            Res = lists:foldl(
-                fun(_M, {atomic, ok} = Res) ->
-                    Res;
-                    (M, _) ->
-                        M:try_register(LUser, LServer, Password)
-                end, {error, not_allowed}, auth_modules(LServer)),
-            case Res of
-                ok ->
-                    ejabberd_hooks:run(register_user, LServer,
-                        [LUser, LServer]),
-                    ok;
-                _ -> Res
-            end;
+            timed_call(LServer, try_register,
+                       fun do_try_register_if_does_not_exist_timed/3, [LUser, LServer, Password]);
         false ->
             {error, not_allowed}
     end.
 
+do_try_register_if_does_not_exist_timed(LUser, LServer, Password) ->
+    Backends = auth_modules(LServer),
+    do_try_register_in_backend(Backends, LUser, LServer, Password).
+
+do_try_register_in_backend([], _, _, _) ->
+    {error, not_allowed};
+do_try_register_in_backend([M | Backends], LUser, LServer, Password) ->
+    case M:try_register(LUser, LServer, Password) of
+        ok ->
+            ejabberd_hooks:run(register_user, LServer,
+                [LUser, LServer]);
+        _ ->
+            do_try_register_in_backend(Backends, LUser, LServer, Password)
+    end.
 
 %% @doc Registered users list do not include anonymous users logged
 -spec dirty_get_registered_users() -> [ejabberd:simple_jid()].
@@ -424,6 +434,9 @@ is_user_exists(User, Server) ->
 do_does_user_exist(LUser, LServer) when LUser =:= error; LServer =:= error ->
     false;
 do_does_user_exist(LUser, LServer) ->
+    timed_call(LServer, does_user_exist, fun does_user_exist_timed/2, [LUser, LServer]).
+
+does_user_exist_timed(LUser, LServer) ->
     lists:any(
         fun(M) ->
             case M:does_user_exist(LUser, LServer) of
@@ -566,3 +579,13 @@ auth_modules(LServer) ->
                   is_atom(Method) -> [Method]
               end,
     [list_to_atom("ejabberd_auth_" ++ atom_to_list(M)) || M <- Methods].
+
+ensure_metrics(Host) ->
+    Metrics = [check_password, try_register, does_user_exist],
+    [mongoose_metrics:ensure_metric(?METRIC(Host, Metric), histogram)
+     || Metric <- Metrics].
+
+timed_call(LServer, Metric, Fun, Args) ->
+    {Time, Result} = timer:tc(Fun, Args),
+    mongoose_metrics:update(?METRIC(LServer, Metric), Time),
+    Result.

apps/ejabberd/src/mongoose_metrics.erl

@@ -20,6 +20,7 @@
 %% API
 -export([init/0,
          update/2,
+         ensure_metric/2,
          start_host_metrics_subscriptions/3,
          start_vm_metrics_subscriptions/2,
          start_global_metrics_subscriptions/2,
@@ -103,11 +104,11 @@ init_predefined_host_metrics(Host) ->
 
 -spec create_generic_hook_metric(ejabberd:lserver(), atom()) -> no_return().
 create_generic_hook_metric(Host, Hook) ->
-    do_create_generic_hook_metric({Host, filter_hook(Hook)}).
+    do_create_generic_hook_metric([Host, filter_hook(Hook)]).
 
 -spec increment_generic_hook_metric(ejabberd:lserver(), atom()) -> no_return().
 increment_generic_hook_metric(Host, Hook) ->
-    do_increment_generic_hook_metric({Host, filter_hook(Hook)}).
+    do_increment_generic_hook_metric([Host, filter_hook(Hook)]).
 
 do_create_generic_hook_metric({_, skip}) ->
     ok;
@@ -124,7 +125,7 @@ get_dist_data_stats() ->
     [{connections, length(DistStats)} | merge_stats(DistStats)].
 
 get_odbc_data_stats() ->
-    RegularODBCWorkers = [ejabberd_odbc_sup:get_pids(Host) || Host <- ?MYHOSTS],
+    RegularODBCWorkers = [catch ejabberd_odbc_sup:get_pids(Host) || Host <- ?MYHOSTS],
     get_odbc_stats(lists:flatten(RegularODBCWorkers)).
 
 get_odbc_mam_async_stats() ->
@@ -255,10 +256,10 @@ create_metrics(Host) ->
     lists:foreach(fun(Name) -> ensure_metric(Name, histogram) end,
                   get_histograms(Host)).
 
-ensure_metric({Host, Metric}, Type) ->
-    case exometer:info([Host, Metric], type) of
+ensure_metric(Name, Type) ->
+    case exometer:info(Name, type) of
         Type -> {ok, already_present};
-        undefined -> exometer:new([Host, Metric], Type)
+        undefined -> exometer:new(Name, Type)
     end.
 
 -spec metrics_hooks('add' | 'delete', ejabberd:server()) -> 'ok'.
@@ -399,7 +400,7 @@ get_vm_stats() ->
       [total, processes_used, atom_used, binary, ets, system]}].
 
 get_counters(Host, Counters) ->
-    [{Host, Counter} || Counter <- Counters].
+    [[Host, Counter] || Counter <- Counters].
 
 do_start_vm_metrics_subscriptions(Reporter, Interval) ->
     [exometer_report:subscribe(Reporter, Metric, DataPoints, Interval)
@@ -422,6 +423,8 @@ subscribe_metric(Reporter, {Name, _, _}, Interval) ->
 
 subscribe_to_all(Reporter, Interval) ->
     start_global_metrics_subscriptions(Reporter, Interval),
+    start_backend_metrics_subscriptions(Reporter, Interval),
+    start_data_metrics_subscriptions(Reporter, Interval),
     lists:foreach(
       fun(Host) ->
               start_host_metrics_subscriptions(Reporter, Host, Interval)

doc/operation-and-maintenance/Mongoose-metrics.md

@@ -1,10 +1,16 @@
 # MongooseIM metrics
 
-MongooseIM by default collects many metrics showing user behaviour and general system statistics. They are managed by [Feuerlabs/exometer](https://github.com/Feuerlabs/exometer). 
-Metrics are organized by XMPP hosts, meaning if MongooseIM servers host `a.com` and `b.com` metrics for specific host only can be obtained.
-There are also some global metrics common for every hosts.
+MongooseIM by default collects many metrics showing user behaviour and general system statistics.
+They are managed by [Feuerlabs/exometer](https://github.com/Feuerlabs/exometer).
+
+All metrics are divided into following groups:
 
-## Metrics description
+* host metrics - organized by XMPP hosts, meaning if MongooseIM servers host `a.com` and `b.com` metrics for specific host only can be obtained. 
+* global metrics - metrics common for all XMPP hosts 
+* backend metrics - these are mainly metrics specific for backend modules
+* data metrics - various metrics related to data sizes (e.g sent / received stanza size statistics)
+
+## Host metrics description
 
 | Metric name | Type | Group | Description |
 | ----------- | ---- | ----- | ----------- |
@@ -95,26 +101,41 @@ There are also some global metrics common for every hosts.
 | xmppStanzaSent | spiral | host, XMPP | Numb of stanzas sent to clients|
 
 
-## Metrics internals
+Metrics assgined to group `hook` are generic metrics updated when given hook is run. In case of these metric its names are the same as corresponding hook name.
+Most of `XMPP` metrics are also triggered by hook and their count is the same as corresponding hook runs. They are named differently to maintain backward compatibility.
+To see `hook <-> XMPP metric` translation please refer to [mongoose_metrics_hooks](https://github.com/esl/MongooseIM/blob/exometer/apps/ejabberd/src/mongoose_metrics_hooks.erl#L71) file.
+
 
-In exometer every metrics is treated as a path. Because of that exometer's name of above metrics looks like following:
+## Global metrics description
 
-```erlang
-[HostOrGlobal, MetricName]
-```
+| Metric name | Type | Description |
+| ----------- | ---- | ----------- |
+| nodeSessionCount | function | number of sessions on given Erlang node |
+| totalSessionCount | function | total number of sessions in the cluster |
+| uniqueSessionCount | function | number of unique sessions in the cluster |
 
-Where:
+### Definitions
+* **session** - it is user and resource,
+* **unique session** - only user, without resource. E.g if a user is connected to the server from 3 different resource it will be counted 3 times in the session counter but only once in the unique sessions counter.
 
-  * `MetricName` is one of above metrics,
-  * `HostOrGlobal` is either atom `global` or binary string with XMPP host served by the server (f.e `<<"localhost">>`)
+## Backends metrics description
 
-### Metric types
+Existence of this kind of metrics is driven by the status of corresponding module.
+For example if mod_roster is enabled relevant `backends` metrics for this module are created and updated.
+This kind of metrics are histograms and contain information about execution time of certain functions from backend module.
 
-* `spiral` - returned value is `{ok,[{count,100},{one,10}]}` where `count` means total number of events and `one` - number of events in last minute 
-* `counter` - returned value is `{ok,[{value,10},{ms_since_reset,245940984}]}` where `value` is the value of the metric and `ms_since_reset` - number of milliseconds since metric reset
-* `value` - returned value is `{ok, 120}`
+The only `backends` metrics which are always present are related to authentication module as this one is always enabled.
 
-### Metrics and hooks
+## Data metrics description
 
-Metrics assgined to group `hook` are generic metrics update when given hook is run. In case of these metric its names are the same as corresponding hook name.
-Most of `XMPP` metrics are also triggered by hook and their count is the same as corresponding hook runs. They are named differently to maintain backward compatibility, to see hook <-> XMPP metric translation please refer to [mongoose_metrics_hooks](https://github.com/esl/MongooseIM/blob/exometer/apps/ejabberd/src/mongoose_metrics_hooks.erl#L71) file. 
+| Metric name | Type | Description |
+| ----------- | ---- | ----------- |
+| [xmpp,received,xml_stanza_size] | histogram | Received (by the server) stanza size stats after possible decryption and decompression |
+| [xmpp,sent,xml_stanza_size] | histogra | Sent (to the clinet) stanza size stats before possible encryption and compression |
+| [xmpp,received,compressed_size] | histogram | Received compressed stanza size stats |
+| [xmpp,sent,compressed_size] | histogram | Sent compressed stanza size stats |
+| [xmpp,received,encrypted_size] | histogram | Received encrypted stanza size stats |
+| [xmpp,sent,encrypted_size] | histogram | Sent encrypted stanza size stats|
+| [odbc,regular] | function | Various network stats for regular ODBC workers |
+| [odbc,mam_async] | function | Various network stats for MAM async ODBC workers |
+| dist | function | Network stats for Erlang distributed communication |