WIP

esl · Jul 22, 2024 · ba9fb25 · ba9fb25
1 parent 3d2012d
commit ba9fb25
Show file tree

Hide file tree

Showing 10 changed files with 212 additions and 183 deletions.
diff --git a/src/config/mongoose_config_parser_toml.erl b/src/config/mongoose_config_parser_toml.erl
@@ -54,7 +54,7 @@
 
 -export_type([toml_key/0, toml_value/0, toml_section/0,
               option_value/0, config/0, config_error/0, config_part/0,
-              list_processor/0, processor/0]).
+              list_processor/0, processor/0, path/0]).
 
 -spec parse_file(FileName :: string()) -> mongoose_config_parser:state().
 parse_file(FileName) ->

diff --git a/src/ejabberd_app.erl b/src/ejabberd_app.erl
@@ -78,7 +78,6 @@ do_start() ->
     mongoose_modules:start(),
     service_mongoose_system_metrics:verify_if_configured(),
     mongoose_listener:start(),
-    mongoose_metrics:init_mongooseim_metrics(),
     mongoose_instrument:persist(),
     gen_hook:reload_hooks(),
     update_status_file(started),

diff --git a/src/instrument/mongoose_instrument.erl b/src/instrument/mongoose_instrument.erl
@@ -47,14 +47,15 @@
 -type handlers() :: {[handler_fun()], config()}.
 -type execution_time() :: integer().
 -type measure_fun(Result) :: fun((execution_time(), Result) -> measurements()).
+-type handler_module_opts() :: #{atom() => any()}.
 
 -callback config_spec() -> mongoose_config_spec:config_section().
--callback start() -> ok.
--callback stop() -> ok.
+-callback start(handler_module_opts()) -> ok.
+-callback stop(handler_module_opts()) -> ok.
 -callback set_up(event_name(), labels(), config()) -> boolean().
 -callback handle_event(event_name(), labels(), config(), measurements()) -> any().
 
--optional_callbacks([config_spec/0, start/0, stop/0]).
+-optional_callbacks([config_spec/0, start/1, stop/1]).
 
 -export_type([event_name/0, labels/0, label_key/0, label_value/0, config/0, measurements/0,
               spec/0, handlers/0, metrics/0, metric_name/0, metric_type/0, probe_config/0]).
@@ -169,7 +170,8 @@ remove_handler(Key) ->
 
 -spec init([]) -> {ok, state()}.
 init([]) ->
-    lists:foreach(fun start_handler/1, handler_modules()),
+    [start_handler(handler_module(Key), Opts)
+     || {Key, Opts = #{}} <- maps:to_list(mongoose_config:get_opt(instrumentation))],
     erlang:process_flag(trap_exit, true), % Make sure that terminate is called
     persistent_term:erase(?MODULE), % Prevent inconsistency when restarted after a kill
     {ok, #{events => #{}, probe_timers => #{}}}.
@@ -197,7 +199,7 @@ handle_call({add_handler, Key, ConfigOpts}, _From, State = #{events := Events})
         {error, not_found} ->
             mongoose_config:set_opt([instrumentation, Key], ConfigOpts),
             Module = handler_module(Key),
-            start_handler(Module),
+            start_handler(Module, ConfigOpts),
             NewEvents = update_handlers(Events, [], [Module]),
             update_if_persisted(Events, NewEvents),
             {reply, ok, State#{events := NewEvents}};
@@ -211,11 +213,12 @@ handle_call({remove_handler, Key}, _From, State = #{events := Events}) ->
         {error, not_found} ->
             {reply, {error, #{what => handler_not_configured, handler_key => Key}}, State};
         {ok, _} ->
+            ConfigOpts = mongoose_config:get_opt([instrumentation, Key]),
             mongoose_config:unset_opt([instrumentation, Key]),
             Module = handler_module(Key),
             NewEvents = update_handlers(Events, [Module], []),
             update_if_persisted(Events, NewEvents),
-            stop_handler(Module),
+            stop_handler(Module, ConfigOpts),
             {reply, ok, State#{events := NewEvents}}
     end;
 handle_call(persist, _From, State = #{events := Events}) ->
@@ -240,7 +243,9 @@ handle_info(Info, State) ->
 -spec terminate(any(), state()) -> ok.
 terminate(_Reason, _State) ->
     persistent_term:erase(?MODULE),
-    lists:foreach(fun stop_handler/1, handler_modules()).
+    [stop_handler(handler_module(Key), Opts)
+     || {Key, Opts = #{}} <- maps:to_list(mongoose_config:get_opt(instrumentation))],
+    ok.
 
 -spec code_change(any(), state(), any()) -> {ok, state()}.
 code_change(_OldVsn, State, _Extra) ->
@@ -409,17 +414,17 @@ config_spec(Key) ->
 all_handler_keys() ->
     [prometheus, exometer, log].
 
--spec start_handler(module()) -> ok.
-start_handler(Module) ->
-    case mongoose_lib:is_exported(Module, start, 0) of
-        true -> Module:start();
+-spec start_handler(module(), handler_module_opts()) -> ok.
+start_handler(Module, Opts) ->
+    case mongoose_lib:is_exported(Module, start, 1) of
+        true -> Module:start(Opts);
         false -> ok
     end.
 
--spec stop_handler(module()) -> ok.
-stop_handler(Module) ->
-    case mongoose_lib:is_exported(Module, stop, 0) of
-        true -> Module:stop();
+-spec stop_handler(module(), handler_module_opts()) -> ok.
+stop_handler(Module, Opts) ->
+    case mongoose_lib:is_exported(Module, stop, 1) of
+        true -> Module:stop(Opts);
         false -> ok
     end.
 

diff --git a/src/instrument/mongoose_instrument_exometer.erl b/src/instrument/mongoose_instrument_exometer.erl
@@ -4,28 +4,93 @@
 
 -define(PREFIXES, {?MODULE, prefixes}).
 
--export([config_spec/0, start/0, stop/0, set_up/3, handle_event/4]).
+-export([config_spec/0, start/1, stop/1, set_up/3, handle_event/4]).
+
+%% Config spec callbacks
+-export([process_graphite_reporter/2]).
 
 -include("mongoose.hrl").
 -include("mongoose_config_spec.hrl").
 
+-type opts() :: #{all_metrics_are_global := boolean(),
+                  report := #{reporter_name() => reporter_opts()}}.
+-type reporter_name() :: atom(). %% exometer_report:reporter_name() is not exported
+-type reporter_opts() :: #{module := module(),
+                           prefix => string(),
+                           env_prefix => string(),
+                           connect_timeout := integer(),
+                           host := string(),
+                           port := inet:port_number(),
+                           api_key := string()}.
+
 -spec config_spec() -> mongoose_config_spec:config_section().
 config_spec() ->
-    #section{items = #{<<"all_metrics_are_global">> => #option{type = boolean}},
-             defaults = #{<<"all_metrics_are_global">> => false}}.
-
--spec start() -> ok.
-start() ->
-    AllGlobal = mongoose_config:get_opt([instrumentation, exometer, all_metrics_are_global]),
+    #section{items = #{<<"all_metrics_are_global">> => #option{type = boolean},
+                       <<"report">> => report_config_spec()},
+             defaults = #{<<"all_metrics_are_global">> => false}
+            }.
+
+-spec report_config_spec() -> mongoose_config_spec:config_section().
+report_config_spec() ->
+    Reporters = [<<"graphite">>],
+    Common = common_reporter_config_spec(),
+    ReporterSpecs = [{Reporter, #list{items = reporter_config_spec(Common, Reporter), wrap = none}}
+                     || Reporter <- Reporters],
+    #section{items = maps:from_list(ReporterSpecs),
+             include = always}.
+
+-spec reporter_config_spec(mongoose_config_spec:config_section(), binary()) ->
+          mongoose_config_spec:config_section().
+reporter_config_spec(BaseSpec, Reporter) ->
+    mongoose_config_utils:merge_sections(BaseSpec, reporter_config_spec(Reporter)).
+
+-spec reporter_config_spec(binary()) -> mongoose_config_spec:config_section().
+reporter_config_spec(<<"graphite">>) ->
+    #section{items = #{<<"prefix">> => #option{type = string},
+                       <<"env_prefix">> => #option{type = string},
+                       <<"connect_timeout">> => #option{type = integer, validate = positive},
+                       <<"host">> => #option{type = string, validate = network_address},
+                       <<"port">> => #option{type = integer, validate = port},
+                       <<"api_key">> => #option{type = string}},
+             defaults = #{<<"connect_timeout">> => 5000, % milliseconds
+                          <<"port">> => 2003,
+                          <<"api_key">> => ""},
+             required = [<<"host">>],
+             process = fun ?MODULE:process_graphite_reporter/2}.
+
+-spec common_reporter_config_spec() -> mongoose_config_spec:config_section().
+common_reporter_config_spec() ->
+    #section{items = #{<<"interval">> => #option{type = integer, validate = positive}},
+             defaults = #{<<"interval">> => 60000} % milliseconds
+            }.
+
+-spec process_graphite_reporter(mongoose_config_parser_toml:path(), map()) ->
+          {reporter_name(), reporter_opts()}.
+process_graphite_reporter(_Path, #{host := Host, port := Port} = Opts) ->
+    Name = list_to_atom(lists:flatten(io_lib:format("graphite:~s:~p", [Host, Port]))),
+    {Name, Opts#{module => exometer_report_graphite}}.
+
+-spec start(opts()) -> ok.
+start(#{all_metrics_are_global := AllGlobal, report := Reporters}) ->
     Prefixes = [{HostType, make_host_type_prefix(HostType, AllGlobal)}
                 || HostType <- ?ALL_HOST_TYPES],
-    persistent_term:put(?PREFIXES, maps:from_list(Prefixes)).
+    persistent_term:put(?PREFIXES, maps:from_list(Prefixes)),
+    maps:foreach(fun add_reporter/2, Reporters).
 
--spec stop() -> ok.
-stop() ->
+-spec stop(opts()) -> ok.
+stop(#{report := Reporters}) ->
+    maps:foreach(fun remove_reporter/2, Reporters),
     persistent_term:erase(?PREFIXES),
     ok.
 
+-spec add_reporter(reporter_name(), reporter_opts()) -> ok | {error, any()}.
+add_reporter(Name, Opts) ->
+    exometer_report:add_reporter(Name, maps:to_list(maps:remove(interval, Opts))).
+
+-spec remove_reporter(reporter_name(), reporter_opts()) -> ok | {error, any()}.
+remove_reporter(Name, _Opts) ->
+    exometer_report:remove_reporter(Name).
+
 -spec set_up(mongoose_instrument:event_name(), mongoose_instrument:labels(),
              mongoose_instrument:config()) -> boolean().
 set_up(EventName, Labels, #{metrics := Metrics}) ->
@@ -51,7 +116,26 @@ set_up_metric(EventName, Labels, MetricName, MetricType) ->
     try exometer:new(Name, MetricType)
     catch
         error:exists -> ok = exometer:reset(Name)
-    end.
+    end,
+    subscribe_reporters(Name, MetricType).
+
+-spec subscribe_reporters(exometer:name(), exometer:type()) -> ok.
+subscribe_reporters(Name, Type) ->
+    maps:foreach(
+      fun(ReporterName, #{interval := Interval}) ->
+              case exometer_report:subscribe(ReporterName, Name, datapoints(Type), Interval) of
+                  ok ->
+                      ok;
+                  Other ->
+                      ?LOG_ERROR(#{what => metrics_subscribe_failed,
+                                   reporter => ReporterName, metric_name => Name,
+                                   reason => Other})
+              end
+      end, mongoose_config:get_opt([instrumentation, exometer, report])).
+
+datapoints(counter) -> [value];
+datapoints(histogram) -> [min, mean, max, median, 95, 99, 999];
+datapoints(_) -> default.
 
 -spec handle_metric_event(mongoose_instrument:event_name(), mongoose_instrument:labels(),
                           mongoose_instrument:metric_name(), mongoose_instrument:metric_type(),

diff --git a/src/instrument/mongoose_instrument_prometheus.erl b/src/instrument/mongoose_instrument_prometheus.erl
@@ -82,7 +82,9 @@ initialize_metric(_Name, _LabelValues, _) ->
 metric_spec(EventName, LabelKeys, MetricName) ->
     [{name, full_metric_name(EventName, MetricName)},
      {help, metric_help(EventName, MetricName)},
-     {labels, LabelKeys}].
+     {labels, LabelKeys},
+     {duration_unit, false} % prevent unwanted implicit conversions, e.g. seconds -> microseconds
+    ].
 
 -spec histogram_buckets() -> [integer()].
 histogram_buckets() ->

diff --git a/src/metrics/mongoose_metrics.erl b/src/metrics/mongoose_metrics.erl
@@ -19,7 +19,6 @@
 
 %% API
 -export([init/0,
-         init_mongooseim_metrics/0,
          update/3,
          ensure_metric/3,
          get_metric_value/1,
@@ -30,15 +29,12 @@
          get_global_metric_names/0,
          get_aggregated_values/1,
          remove_host_type_metrics/1,
-         remove_all_metrics/0,
-         get_report_interval/0
+         remove_all_metrics/0
         ]).
 
--ignore_xref([remove_host_type_metrics/1, get_report_interval/0,
-              sample_metric/1, get_metric_value/1]).
+-ignore_xref([remove_host_type_metrics/1, sample_metric/1, get_metric_value/1]).
 
 -define(PREFIXES, mongoose_metrics_prefixes).
--define(DEFAULT_REPORT_INTERVAL, 60000). %%60s
 
 -type metric_name() :: atom() | list(atom() | binary()).
 -type short_metric_type() :: spiral | histogram | counter | gauge.
@@ -52,18 +48,6 @@
 init() ->
     prepare_prefixes().
 
--spec init_mongooseim_metrics() -> ok.
-init_mongooseim_metrics() ->
-    init_subscriptions().
-
-init_subscriptions() ->
-    Reporters = exometer_report:list_reporters(),
-    lists:foreach(
-        fun({Name, _ReporterPid}) ->
-                Interval = get_report_interval(),
-                subscribe_to_all(Name, Interval)
-        end, Reporters).
-
 -spec update(HostType :: mongooseim:host_type_or_global(), Name :: term() | list(),
              Change :: term()) -> any().
 update(HostType, Name, Change) when is_list(Name) ->
@@ -151,21 +135,11 @@ pick_prefix_by_all_metrics_are_global(HostType) ->
         false -> get_host_type_prefix(HostType)
     end.
 
-pick_by_all_metrics_are_global(WhenGlobal, WhenNot) ->
-    case all_metrics_are_global() of
-        true -> WhenGlobal;
-        false -> WhenNot
-    end.
-
 -spec name_by_all_metrics_are_global(HostType :: mongooseim:host_type_or_global(),
                                      Name :: list()) -> FinalName :: list().
 name_by_all_metrics_are_global(HostType, Name) ->
     [pick_prefix_by_all_metrics_are_global(HostType) | Name].
 
-get_report_interval() ->
-    application:get_env(exometer_core, mongooseim_report_interval,
-                        ?DEFAULT_REPORT_INTERVAL).
-
 remove_metric({Name, _, _}) ->
     exometer_admin:delete_entry(Name).
 
@@ -188,32 +162,3 @@ do_create_metric(PrefixedMetric, ExometerType, ExometerOpts) ->
         ok -> ok;
         {'EXIT', Error} -> {error, Error}
     end.
-
-start_metrics_subscriptions(Reporter, MetricPrefix, Interval) ->
-    [subscribe_metric(Reporter, Metric, Interval)
-     || Metric <- exometer:find_entries(MetricPrefix)].
-
-subscribe_metric(Reporter, {Name, counter, _}, Interval) ->
-    subscribe_verbose(Reporter, Name, [value], Interval);
-subscribe_metric(Reporter, {Name, histogram, _}, Interval) ->
-    subscribe_verbose(Reporter, Name, [min, mean, max, median, 95, 99, 999], Interval);
-subscribe_metric(Reporter, {Name, _, _}, Interval) ->
-    subscribe_verbose(Reporter, Name, default, Interval).
-
-subscribe_verbose(Reporter, Name, Types, Interval) ->
-    case exometer_report:subscribe(Reporter, Name, Types, Interval) of
-        ok -> ok;
-        Other ->
-            ?LOG_ERROR(#{what => metrics_subscribe_failed,
-                         reporter => Reporter, metric_name => Name,
-                         reason => Other}),
-            Other
-    end.
-
-subscribe_to_all(Reporter, Interval) ->
-    HostTypePrefixes = pick_by_all_metrics_are_global([], ?ALL_HOST_TYPES),
-    lists:foreach(
-      fun(Prefix) ->
-              UnspacedPrefix = get_host_type_prefix(Prefix),
-              start_metrics_subscriptions(Reporter, [UnspacedPrefix], Interval)
-      end, [global | HostTypePrefixes]).