GDPR retrieve inbox (clean) (#2299)

* Implement retrieve data from mod_inbox * Test retrieve perosnal data from mod_inbox * Change getting backends for mod_inbox * Check retreive from mod_inbox when module is disabled
esl · May 12, 2019 · ec387d4 · ec387d4
1 parent 8226429
commit ec387d4
Show file tree

Hide file tree

Showing 4 changed files with 115 additions and 19 deletions.
diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl
@@ -25,6 +25,7 @@
          dont_retrieve_other_user_private_xml/1,
          retrieve_multiple_private_xmls/1,
          retrieve_inbox/1,
+         retrieve_inbox_for_multiple_messages/1,
          retrieve_logs/1
         ]).
 -export([
@@ -59,7 +60,8 @@ groups() ->
                                            retrieve_roster,
                                            %retrieve_mam,
                                            %retrieve_offline,
-                                           %retrieve_inbox,
+                                           retrieve_inbox,
+                                           retrieve_inbox_for_multiple_messages,
                                            retrieve_logs,
                                            {group, retrieve_personal_data_pubsub},
                                            {group, retrieve_personal_data_private_xml}
@@ -73,6 +75,7 @@ groups() ->
                                          ]},
      {retrieve_personal_data_with_mods_disabled, [], [
                                                       retrieve_vcard,
+                                                      retrieve_inbox,
                                                       retrieve_logs,
                                                       retrieve_roster,
                                                       retrieve_all_pubsub_data,
@@ -111,14 +114,9 @@ end_per_group(_GN, Config) ->
     Config.
 
 init_per_testcase(retrieve_inbox = CN, Config) ->
-    case (not ct_helper:is_ct_running())
-         orelse mongoose_helper:is_rdbms_enabled(domain()) of
-        true ->
-            dynamic_modules:ensure_modules(domain(), inbox_required_modules()),
-            escalus:init_per_testcase(CN, Config);
-        false ->
-            {skip, require_rdbms}
-    end;
+    init_inbox(CN, Config);
+init_per_testcase(retrieve_inbox_for_multiple_messages = CN, Config) ->
+    init_inbox(CN, Config);
 init_per_testcase(retrieve_vcard = CN, Config) ->
     case vcard_update:is_vcard_ldap() of
         true ->
@@ -140,8 +138,25 @@ init_per_testcase(CN, Config) ->
 end_per_testcase(CN, Config) ->
     escalus:end_per_testcase(CN, Config).
 
+init_inbox(CN, Config) ->
+    case (not ct_helper:is_ct_running())
+         orelse mongoose_helper:is_rdbms_enabled(domain()) of
+        true ->
+            dynamic_modules:ensure_modules(domain(), inbox_required_modules()),
+            escalus:init_per_testcase(CN, Config);
+        false ->
+            {skip, require_rdbms}
+    end.
 inbox_required_modules() ->
-    [{mod_inbox, []}].
+    [
+     {mod_inbox, inbox_opts()}
+    ].
+
+inbox_opts() ->
+    [{aff_changes, true},
+     {remove_on_kicked, true},
+     {groupchat, [muclight]},
+     {markers, [displayed]}].
 
 pick_backend_for_mam() ->
     BackendsList = [
@@ -435,18 +450,50 @@ retrieve_multiple_private_xmls(Config) ->
 
 retrieve_inbox(Config) ->
     escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+            BobU = escalus_utils:jid_to_lower(escalus_client:username(Bob)),
+            BobS = escalus_utils:jid_to_lower(escalus_client:server(Bob)),
+            AliceU = escalus_utils:jid_to_lower(escalus_client:username(Alice)),
+            AliceS = escalus_utils:jid_to_lower(escalus_client:server(Alice)),
             Body = <<"With spam?">>,
-            escalus:send(Bob, escalus_stanza:chat_to(Alice, Body)),
-            Msg = escalus:wait_for_stanza(Alice),
-            escalus:assert(is_chat_message, [Body], Msg),
+            send_and_assert_is_chat_message(Bob, Alice, Body),
+            ExpectedHeader = ["jid", "content", "unread_count", "timestamp"],
+            ExpectedAliceItems = [
+                             #{ "content" => [{contains, Body}],
+                                "jid" => [{contains, BobS},
+                                          {contains, BobU}],
+                                "unread_count" => "1" }
+                            ],
+            ExpectedBobItems = [
+                             #{ "content" => [{contains, Body}],
+                                "jid" => [{contains, AliceS},
+                                          {contains, AliceU}],
+                                "unread_count" => "0" }
+                            ],
+            retrieve_and_validate_personal_data(
+              Alice, Config, "inbox", ExpectedHeader, ExpectedAliceItems),
+            retrieve_and_validate_personal_data(
+              Bob, Config, "inbox", ExpectedHeader, ExpectedBobItems)
+        end).
 
-            BobJid = escalus_client:short_jid(Bob),
-            ExpectedHeader = ["jid", "content", "unread_count", "msg_id", "timestamp"],
-            ExpectedItems = [
-                             #{ "content" => Body, "jid" => BobJid }
+retrieve_inbox_for_multiple_messages(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+            Bodies = [ <<"Nobody exists on purpose.">>,
+                       <<"Nobody belongs anywhere.">>,
+                       <<"We're all going to die.">>,
+                       <<"Come watch TV.">>],
+            lists:foreach(fun(Body) -> send_and_assert_is_chat_message(Bob, Alice, Body) end, Bodies),
+            BobU = escalus_utils:jid_to_lower(escalus_client:username(Bob)),
+            BobS = escalus_utils:jid_to_lower(escalus_client:server(Bob)),
+
+            ExpectedHeader = ["jid", "content", "unread_count", "timestamp"],
+            ExpectedAliceItems = [
+                             #{ "content" => [{contains, lists:last(Bodies)}],
+                                "jid" => [{contains, BobS},
+                                          {contains, BobU}],
+                                "unread_count" => integer_to_list(length(Bodies)) }
                             ],
             retrieve_and_validate_personal_data(
-              Alice, Config, "inbox", ExpectedHeader, ExpectedItems)
+              Alice, Config, "inbox", ExpectedHeader, ExpectedAliceItems)
         end).
 
 retrieve_logs(Config) ->
@@ -610,3 +657,8 @@ send_and_assert_private_stanza(User, NS, Content) ->
     escalus_client:send(User, PrivateStanza),
     escalus:assert(is_iq_result, [PrivateStanza], escalus_client:wait_for_stanza(User)).
 
+send_and_assert_is_chat_message(UserFrom, UserTo, Body) ->
+    escalus:send(UserFrom, escalus_stanza:chat_to(UserTo, Body)),
+    Msg = escalus:wait_for_stanza(UserTo),
+    escalus:assert(is_chat_message, [Body], Msg).
+
diff --git a/include/mod_inbox.hrl b/include/mod_inbox.hrl
@@ -18,7 +18,7 @@
 
 -type get_inbox_res() :: list(inbox_res()).
 
--type inbox_res() :: {RemoteUser :: username(),
+-type inbox_res() :: {RemoteBinJID :: binary(),
                       MsgContent :: content(),
                       UnreadCount :: count_bin(),
                       Timestamp :: erlang:timestamp()}.

diff --git a/src/inbox/mod_inbox.erl b/src/inbox/mod_inbox.erl
@@ -15,6 +15,10 @@
 -include("mongoose.hrl").
 -include("mongoose_logger.hrl").
 
+-behaviour(gdpr).
+
+-export([get_personal_data/2]).
+
 -export([start/2, stop/1, deps/2]).
 -export([process_iq/4, user_send_packet/4, filter_packet/1, inbox_unread_count/2]).
 -export([clear_inbox/2]).
@@ -78,6 +82,44 @@
 
 -export_type([get_inbox_params/0]).
 
+%%--------------------------------------------------------------------
+%% gdpr callbacks
+%%--------------------------------------------------------------------
+
+-spec get_personal_data(jid:username(), jid:server()) ->
+    [{gdpr:data_group(), gdpr:schema(), gdpr:entries()}].
+get_personal_data(Username, Server) ->
+    LUser = jid:nodeprep(Username),
+    LServer = jid:nameprep(Server),
+    Schema = ["jid", "content", "unread_count", "timestamp"],
+    InboxParams = #{
+        start => {0,0,0},
+        'end' => erlang:timestamp(),
+        order => asc,
+        hidden_read => false
+       },
+      Entries = lists:flatmap(fun(B) ->
+                          try B:get_inbox(LUser, LServer, InboxParams) of
+                              Entries when is_list(Entries) -> Entries;
+                              _ -> []
+                          catch
+                              C:R ->
+                                  log_get_personal_data_warning(B, C, R, erlang:get_stacktrace()),
+                                  []
+                          end
+                  end, mongoose_lib:find_behaviour_implementations(mod_inbox)),
+    ProcessedEntries = [{ RemJID, Content, UnreadCount, jlib:now_to_utc_string(Timestamp) } ||
+                        { RemJID, Content, UnreadCount, Timestamp } <- Entries],
+    [{inbox, Schema, ProcessedEntries}].
+
+log_get_personal_data_warning(Backend, Class, Reason, StackTrace) ->
+    ?WARNING_MSG("event=cannot_retrieve_personal_data,backend=~p,class=~p,reason=~p,stacktrace=~p",
+                 [Backend, Class, Reason, StackTrace]).
+
+%%--------------------------------------------------------------------
+%% inbox callbacks
+%%--------------------------------------------------------------------
+
 -spec deps(jid:lserver(), list()) -> list().
 deps(_Host, Opts) ->
     groupchat_deps(Opts).

diff --git a/src/inbox/mod_inbox_rdbms.erl b/src/inbox/mod_inbox_rdbms.erl
@@ -12,6 +12,8 @@
 -include("mongoose.hrl").
 -include("mod_inbox.hrl").
 
+-behaviour(mod_inbox).
+
 %% API
 -export([get_inbox/3,
          init/2,