gdpr logs collection

big_tests/tests/gdpr_SUITE.erl

@@ -254,12 +254,20 @@ retrieve_inbox(Config) ->
         end).
 
 retrieve_logs(Config) ->
-    mongoose_helper:successful_rpc(error_logger, error_msg,
-                                   ["event=disturbance_in_the_force, jid=sith@localhost", []]),
-    Dir = request_and_unzip_personal_data(<<"sith">>, <<"localhost">>, Config),
-    Filename = filename:join(Dir, "logs.txt"),
-    {ok, Content} = file:read_file(Filename),
-    {match, _} = re:run(Content, "disturbance_in_the_force").
+    escalus:fresh_story(Config, [{alice, 1}],
+        fun(Alice) ->
+            User = escalus_client:username(Alice),
+            Domain = escalus_client:server(Alice),
+            JID = binary_to_list(User) ++ "@" ++ binary_to_list(Domain),
+            MIM2Node = distributed_helper:mim2(),
+            mongoose_helper:successful_rpc(net_kernel, connect_node, [MIM2Node]),
+            mongoose_helper:successful_rpc(MIM2Node, error_logger, error_msg,
+                                           ["event=disturbance_in_the_force, jid=" ++ JID, []]),
+            Dir = request_and_unzip_personal_data(User, Domain, Config),
+            Filename = filename:join(Dir, "logs-" ++ atom_to_list(MIM2Node) ++ ".txt"),
+            {ok, Content} = file:read_file(Filename),
+            {match, _} = re:run(Content, "disturbance_in_the_force")
+        end).
 
 %% ------------------------- Data retrieval - Negative case -------------------------
 

priv/parse_logs.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+## Args = [FileName, Username, Domain | FileList]
+
+echo "$@" >$1
+
+file=$1
+user=$2
+domain=$3
+
+shift 3
+
+pattern="${user}@${domain}"
+
+filelist=()
+
+for f in "$@"; do
+filelist+=("${f}"*)
+done
+
+grep -iF "$pattern" "${filelist[@]}" > "$file" || true

src/admin_extra/service_admin_extra_gdpr.erl

@@ -2,37 +2,53 @@
 
 -include("ejabberd_commands.hrl").
 
--export(
-    [commands/0,
-     retrieve_all/3]).
+-export([commands/0,
+         retrieve_all/3,
+         retrieve_logs/2]).
 
 -spec commands() -> [ejabberd_commands:cmd()].
 commands() -> [
-    #ejabberd_commands{name = retrieve_personal_data, tags = [gdpr],
-                       desc = "Retrieve user's presonal data.",
-                       longdesc = "Retrieves all personal data from MongooseIM for a given user. Example:\n"
+        #ejabberd_commands{name = retrieve_personal_data, tags = [gdpr],
+            desc = "Retrieve user's presonal data.",
+            longdesc = "Retrieves all personal data from MongooseIM for a given user. Example:\n"
                        " %TODO ", % TODO add example
-                       module = ?MODULE,
-                       function = retrieve_all,
-                       args = [{username, binary}, {domain, binary}, {path, binary}], % TODO add arguments if needed
-                       result = {content, binary}}  % TODO check if returned type is correct and convinient in use
-].
+            module = ?MODULE,
+            function = retrieve_all,
+            args = [{username, binary}, {domain, binary}, {path, binary}], % TODO add arguments if needed
+            result = {content, binary}}  % TODO check if returned type is correct and convinient in use
+
+    ].
 
 -spec retrieve_all(gdpr:username(), gdpr:domain(), Path :: binary()) -> RetrievedFilesInZipName :: binary().
 retrieve_all(Username, Domain, ResultFilePath) ->
     case user_exists(Username, Domain) of
-    true ->
-        DataFromTables = get_data_from_tables(Username, Domain),
-        CsvFiles = lists:map(
-            fun({Tablename, Schema, Entitis}) ->
-                BinTablename = atom_to_binary(Tablename, utf8),
-                to_csv_file(<<BinTablename/binary, <<".csv">>/binary>>, Schema, Entitis) end,
-            DataFromTables),
-        {ok, R} = zip:create(ResultFilePath, lists:map(fun binary_to_list/1, CsvFiles)),
-        R;
-    false -> {error, "User does not exist"}
+        true ->
+            DataFromTables = get_data_from_tables(Username, Domain),
+            TmpDir = make_tmp_dir(),
+            CsvFiles = lists:map(
+                fun({TableName, Schema, Entities}) ->
+                    BinTableName = atom_to_binary(TableName, utf8),
+                    FileName = <<BinTableName/binary, ".csv">>,
+                    to_csv_file(FileName, Schema, Entities, TmpDir),
+                    binary_to_list(FileName)
+                end,
+                DataFromTables),
+            LogFiles = get_all_logs(Username, Domain, TmpDir),
+            ZipFile = binary_to_list(ResultFilePath),
+            {ok, ZipFile} = zip:create(ZipFile, CsvFiles ++ LogFiles, [{cwd, TmpDir}]),
+            remove_tmp_dir(TmpDir),
+            ResultFilePath;
+        false -> {error, "User does not exist"}
     end.
 
+-spec retrieve_logs(gdpr:username(), gdpr:domain()) -> {ok, ZippedLogs :: binary()}.
+retrieve_logs(Username, Domain) ->
+    TmpDir = make_tmp_dir(),
+    LogFile = get_logs(Username, Domain, TmpDir),
+    {ok, {_, ZippedLogs}} = zip:create("archive.zip", [LogFile], [memory, {cwd, TmpDir}]),
+    remove_tmp_dir(TmpDir),
+    {ok, ZippedLogs}.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%                       Private funs
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -41,20 +57,68 @@ retrieve_all(Username, Domain, ResultFilePath) ->
     [{gdpr:binary_table_name(), gdpr:schema(), gdpr:entities()}].
 get_data_from_tables(Username, Domain) ->
     Modules = get_modules(),
-    lists:flatten([M:get_personal_data(Username, Domain)|| M <- Modules]).
+    lists:flatten([M:get_personal_data(Username, Domain) || M <- Modules]).
 
--spec to_csv_file(CsvFilename :: binary(), gdpr:schema(), gdpr:entities()) -> CsvFilename :: binary().
-to_csv_file(Filename, DataSchema, DataRows) ->
-    {ok, File} = file:open(Filename, [write]),
+-spec to_csv_file(CsvFilename :: binary(), gdpr:schema(), gdpr:entities(), file:name()) -> ok.
+to_csv_file(Filename, DataSchema, DataRows, TmpDir) ->
+    FilePath = <<(list_to_binary(TmpDir))/binary, "/", Filename/binary>>,
+    {ok, File} = file:open(FilePath, [write]),
     csv_gen:row(File, DataSchema),
     lists:foreach(fun(Row) -> csv_gen:row(File, Row) end, DataRows),
-    file:close(File),
-    Filename.
+    file:close(File).
 
 -spec get_modules() -> [module()].
 get_modules() ->
     [mod_vcard_mnesia].
 
 -spec user_exists(gdpr:username(), gdpr:domain()) -> boolean().
 user_exists(Username, Domain) ->
-    ejabberd_auth:is_user_exists(Username, Domain).
+    ejabberd_auth:is_user_exists(Username, Domain).
+
+-spec get_all_logs(gdpr:username(), gdpr:domain(), file:name()) -> [file:name()].
+get_all_logs(Username, Domain, TmpDir) ->
+    OtherNodes = ejabberd_config:other_cluster_nodes(),
+    LogFile = get_logs(Username, Domain, TmpDir),
+    LogFilesFromOtherNodes = [get_logs_from_node(Node, Username, Domain, TmpDir) || Node <- OtherNodes],
+    [LogFile | LogFilesFromOtherNodes].
+
+-spec get_logs(gdpr:username(), gdpr:domain(), file:name()) -> file:name().
+get_logs(Username, Domain, TmpDir) ->
+    FileList = [filename:absname(F) || F <- ejabberd_loglevel:get_log_files()],
+    Cmd = code:priv_dir(mongooseim) ++ "/parse_logs.sh",
+    FileName = "logs-" ++ atom_to_list(node()) ++ ".txt",
+    FilePath = TmpDir ++ "/" ++ FileName,
+    Args = [FilePath, Username, Domain | FileList],
+    0 = run(Cmd, Args, 300000),
+    FileName.
+
+-spec get_logs_from_node(node(), gdpr:username(), gdpr:domain(), file:name()) -> file:name().
+get_logs_from_node(Node, Username, Domain, TmpDir) ->
+    {ok, ZippedData} = rpc:call(Node, ?MODULE, retrieve_logs, [Username, Domain]),
+    {ok, [File]} = zip:unzip(ZippedData, [{cwd, TmpDir}]),
+    string:prefix(File, TmpDir ++ "/").
+
+-spec make_tmp_dir() -> file:name().
+make_tmp_dir() ->
+    TmpDirName = lists:flatten(io_lib:format("/tmp/gdpr-~4.36.0b", [rand:uniform(36#zzzz)])),
+    case file:make_dir(TmpDirName) of
+        ok -> TmpDirName;
+        {error, eexist} -> make_tmp_dir();
+        {error, Error} -> {error, Error}
+    end.
+
+-spec remove_tmp_dir(file:name()) -> ok.
+remove_tmp_dir(TmpDir) ->
+    {ok, FileNames} = file:list_dir(TmpDir),
+    [file:delete(TmpDir ++ "/" ++ File) || File <- FileNames],
+    file:del_dir(TmpDir).
+
+-type cmd() :: string() | binary().
+-spec run(cmd(), [cmd()], timeout()) -> non_neg_integer() | timeout.
+run(Cmd, Args, Timeout) ->
+    Port = erlang:open_port({spawn_executable, Cmd}, [exit_status, {args, Args}]),
+    receive
+        {Port, {exit_status, ExitStatus}} -> ExitStatus
+    after Timeout ->
+        timeout
+    end.

src/ejabberd_config.erl

@@ -54,6 +54,9 @@
 -export([config_state/0]).
 -export([config_states/0]).
 
+-export([all_cluster_nodes/0,
+         other_cluster_nodes/0]).
+
 -import(mongoose_config_parser, [can_be_ignored/1]).
 
 -export([apply_reloading_change/1]).

src/ejabberd_ctl.erl

@@ -152,7 +152,7 @@ process(["status"]) ->
                                  {mongoose_status, MongooseStatus},
                                  {os_pid, os:getpid()}, get_uptime(),
                                  {dist_proto, get_dist_proto()},
-                                 {logs, get_log_files()}])]),
+                                 {logs, ejabberd_loglevel:get_log_files()}])]),
     case MongooseStatus of
         not_running -> ?STATUS_ERROR;
         {running, _, _Version} -> ?STATUS_SUCCESS
@@ -923,17 +923,3 @@ get_dist_proto() ->
         _ -> "inet_tcp"
     end.
 
-%%-----------------------------
-%% Lager specific helpers
-%%-----------------------------
-
-get_log_files() ->
-    Handlers = case catch sys:get_state(lager_event) of
-                   {'EXIT', _} -> [];
-                   Hs when is_list(Hs) -> Hs
-               end,
-    [ file_backend_path(State)
-      || {lager_file_backend, _File, State} <- Handlers ].
-
-file_backend_path(LagerFileBackendState) when element(1, LagerFileBackendState) =:= state ->
-    element(2, LagerFileBackendState).

src/ejabberd_loglevel.erl

@@ -32,18 +32,20 @@
          set_custom/2,
          clear_custom/0, clear_custom/1]).
 
+-export([get_log_files/0]).
+
 -include("mongoose.hrl").
 
 -export_type([loglevel/0]).
 -type loglevel() :: none | critical | error | warning | info | debug.
 
 -define(LOG_LEVELS,
-        [{0, none},
-         {1, critical},
-         {2, error},
-         {3, warning},
-         {4, info},
-         {5, debug}]).
+    [{0, none},
+     {1, critical},
+     {2, error},
+     {3, warning},
+     {4, info},
+     {5, debug}]).
 
 -define(ETS_TRACE_TAB, ejabberd_lager_traces).
 
@@ -54,39 +56,39 @@ init() ->
 
 -spec get() -> [{{atom(), term()} | atom(), {non_neg_integer(), loglevel()}}].
 get() ->
-    Backends = gen_event:which_handlers(lager_event),
-    [ {Backend, lists:keyfind(lager:get_loglevel(Backend), 2, ?LOG_LEVELS)}
-      || Backend <- Backends, Backend /= lager_backend_throttle ].
+    Backends = [Backend || Sink <- lager:list_all_sinks(), Backend <- gen_event:which_handlers(Sink)],
+    [{Backend, lists:keyfind(lager:get_loglevel(Backend), 2, ?LOG_LEVELS)}
+     || Backend <- Backends, Backend /= lager_backend_throttle].
 
 -spec set(loglevel() | integer()) -> [Result] when
-      Result :: { LagerBackend, ok | {error, Reason} },
-      %% Yes, these are two different errors!
-      Reason :: bad_log_level | bad_loglevel,
-      LagerBackend :: lager_console_backend | {lager_file_backend, Path},
-      Path :: string().
+    Result :: {LagerBackend, ok | {error, Reason}},
+    %% Yes, these are two different errors!
+    Reason :: bad_log_level | bad_loglevel,
+    LagerBackend :: lager_console_backend | {lager_file_backend, Path},
+    Path :: string().
 set(Level) when is_integer(Level) ->
     {_, Name} = lists:keyfind(Level, 1, ?LOG_LEVELS),
     set(Name);
 set(Level) ->
-    Backends = gen_event:which_handlers(lager_event),
-    Files = [ { B, lager:set_loglevel(lager_file_backend, File, Level) }
-              || B = {lager_file_backend, File} <- Backends ],
-    Consoles = [ { B, lager:set_loglevel(lager_console_backend, Level) }
-                 || B = lager_console_backend <- Backends ],
+    Backends = [Backend || Sink <- lager:list_all_sinks(), Backend <- gen_event:which_handlers(Sink)],
+    Files = [{B, lager:set_loglevel(lager_file_backend, File, Level)}
+             || B = {lager_file_backend, File} <- Backends],
+    Consoles = [{B, lager:set_loglevel(lager_console_backend, Level)}
+                || B = lager_console_backend <- Backends],
     Files ++ Consoles.
 
 -spec set_custom(Module :: atom(), loglevel() | integer()) -> [Result] when
-      Result :: {lager_console_backend | {lager_file_backend, string()},
-                 ok | {error, any()}}.
+    Result :: {lager_console_backend | {lager_file_backend, string()},
+               ok | {error, any()}}.
 set_custom(Module, Level) when is_integer(Level) ->
     {_, Name} = lists:keyfind(Level, 1, ?LOG_LEVELS),
     set_custom(Module, Name);
 set_custom(Module, Level) when is_atom(Level) ->
     clear_custom(Module),
-    Backends = gen_event:which_handlers(lager_event),
-    [ {Backend, set_trace(Backend, Module, Level)}
-      || Backend <- Backends,
-         Backend /= lager_backend_throttle ].
+    Backends = [Backend || Sink <- lager:list_all_sinks(), Backend <- gen_event:which_handlers(Sink)],
+    [{Backend, set_trace(Backend, Module, Level)}
+     || Backend <- Backends,
+     Backend /= lager_backend_throttle].
 
 set_trace(Backend, Module, Level) ->
     case lager:trace(Backend, [{module, Module}], Level) of
@@ -111,8 +113,11 @@ clear_trace({_Module, Trace}, ok) ->
 clear_custom(Module) when is_atom(Module) ->
     case ets:lookup(?ETS_TRACE_TAB, Module) of
         [] -> ok;
-        [_|_] = Traces ->
+        [_ | _] = Traces ->
             ets:delete(?ETS_TRACE_TAB, Module),
-            [ lager:stop_trace(Trace) || {_, Trace} <- Traces ],
+            [lager:stop_trace(Trace) || {_, Trace} <- Traces],
             ok
     end.
+
+get_log_files() ->
+    [lager_util:expand_path(File) || {{lager_file_backend, File}, _, _} <- lager_config:global_get(handlers)].