Gdpr retrieve personal data from mod_private (#2288)

* Implement retreive personal data for mod_private with mnesia

* Enable test for retrive data from  mod_private

* Implement retreive personal data for mod_private with riak

* Implement retreive mod_privare for rdbms

* Test retrieving multiple private xmls

* Implement retreive for mod_private_mysql

* Refactor retreive personal xml tests

* Minor fixes in gdpr_SUITE

* Fix GDPR retrieval for mod_private_riak

* Move new mod_private callback to other behaviour defs

* Retrieve personal private XML from all backends

* Style fixes in private XML GDPR retrieval

* Fix style fixes in private XML GDPR retrieval

* Fix the fix of the style fixes

* Fix of the fix the fix of the style fixes

big_tests/tests/gdpr_SUITE.erl

@@ -22,6 +22,8 @@
          dont_retrieve_other_user_pubsub_payload/1,
          retrieve_pubsub_subscriptions/1,
          retrieve_private_xml/1,
+         dont_retrieve_other_user_private_xml/1,
+         retrieve_multiple_private_xmls/1,
          retrieve_inbox/1,
          retrieve_logs/1
         ]).
@@ -57,10 +59,10 @@ groups() ->
                                            retrieve_roster,
                                            %retrieve_mam,
                                            %retrieve_offline,
-                                           %retrieve_private_xml,
                                            %retrieve_inbox,
                                            retrieve_logs,
-                                           {group, retrieve_personal_data_pubsub}
+                                           {group, retrieve_personal_data_pubsub},
+                                           {group, retrieve_personal_data_private_xml}
                                           ]},
      {retrieve_personal_data_pubsub, [], [
                                           retrieve_pubsub_payloads,
@@ -73,8 +75,14 @@ groups() ->
                                                       retrieve_vcard,
                                                       retrieve_logs,
                                                       retrieve_roster,
-                                                      retrieve_all_pubsub_data
+                                                      retrieve_all_pubsub_data,
+                                                      retrieve_multiple_private_xmls
                                                      ]},
+     {retrieve_personal_data_private_xml, [], [
+                                               retrieve_private_xml,
+                                               dont_retrieve_other_user_private_xml,
+                                               retrieve_multiple_private_xmls
+                                              ]},
      {retrieve_negative, [], [
                               data_is_not_retrieved_for_missing_user
                              ]}
@@ -371,21 +379,60 @@ retrieve_private_xml(Config) ->
     escalus:fresh_story(Config, [{alice, 1}], fun(Alice) ->
             NS = <<"alice:gdpr:ns">>,
             Content = <<"dGhlcmUgYmUgZHJhZ29ucw==">>,
-            XML = #xmlel{ name = <<"fingerprint">>,
-                          attrs = [{<<"xmlns">>, NS}],
-                          children = [#xmlcdata{ content = Content }]},
-            PrivateStanza = escalus_stanza:private_set(XML),
-            escalus_client:send(Alice, PrivateStanza),
-            escalus:assert(is_iq_result, [PrivateStanza], escalus_client:wait_for_stanza(Alice)),
-            ExpectedHeader = ["ns", "xml"], % TODO?
-            ExpectedItems = [
-                             #{ "xml" => [{contains, "alice:gdpr:ns"},
+            send_and_assert_private_stanza(Alice, NS, Content),
+            ExpectedHeader = ["ns", "xml"],
+            ExpectedItems = [#{ "ns" => binary_to_list(NS),
+                                "xml" => [{contains, binary_to_list(NS)},
                                           {contains, binary_to_list(Content)}] }
                             ],
             retrieve_and_validate_personal_data(
               Alice, Config, "private", ExpectedHeader, ExpectedItems)
         end).
 
+dont_retrieve_other_user_private_xml(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+            AliceNS = <<"alice:gdpr:ns">>,
+            AliceContent = <<"To be or not to be">>,
+            BobNS = <<"bob:gdpr:ns">>,
+            BobContent = <<"This is the winter of our discontent">>,
+            send_and_assert_private_stanza(Alice, AliceNS, AliceContent),
+            send_and_assert_private_stanza(Bob, BobNS, BobContent),
+            ExpectedHeader = ["ns", "xml"],
+            ExpectedItems = [#{ "ns" => binary_to_list(AliceNS),
+                                "xml" => [{contains, binary_to_list(AliceNS)},
+                                          {contains, binary_to_list(AliceContent)}] }
+                            ],
+            retrieve_and_validate_personal_data(
+              Alice, Config, "private", ExpectedHeader, ExpectedItems)
+        end).
+
+retrieve_multiple_private_xmls(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}], fun(Alice) ->
+            NSsAndContents = [
+                              {<<"alice:gdpr:ns1">>, <<"You do not talk about FIGHT CLUB.">>},
+                              {<<"alice:gdpr:ns2">>, <<"You do not talk about FIGHT CLUB.">>},
+                              {<<"alice:gdpr:ns3">>, <<"If someone says stop or goes limp,"
+                                                       " taps out the fight is over.">>},
+                              {<<"alice:gdpr:ns4">>, <<"Only two guys to a fight.">>},
+                              {<<"alice:gdpr:ns5">>, <<"One fight at a time.">>}
+                             ],
+            lists:foreach(
+                fun({NS, Content}) ->
+                    send_and_assert_private_stanza(Alice, NS, Content)
+                end, NSsAndContents),
+            ExpectedHeader = ["ns", "xml"],
+            ExpectedItems = lists:map(
+                fun({NS, Content}) ->
+                    #{ "ns" => binary_to_list(NS),
+                       "xml" => [{contains, binary_to_list(NS)},
+                                 {contains, binary_to_list(Content)}]}
+                end, NSsAndContents),
+
+            maybe_stop_and_unload_module(mod_private, mod_private_backend, Config),
+            retrieve_and_validate_personal_data(
+              Alice, Config, "private", ExpectedHeader, ExpectedItems)
+        end).
+
 retrieve_inbox(Config) ->
     escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
             Body = <<"With spam?">>,
@@ -555,3 +602,11 @@ item_content_xml(Data) ->
            attrs = [{<<"xmlns">>, <<"http://www.w3.org/2005/Atom">>}],
            children = [#xmlcdata{content = Data}]}.
 
+send_and_assert_private_stanza(User, NS, Content) ->
+    XML = #xmlel{ name = <<"fingerprint">>,
+                  attrs = [{<<"xmlns">>, NS}],
+                  children = [#xmlcdata{ content = Content }]},
+    PrivateStanza = escalus_stanza:private_set(XML),
+    escalus_client:send(User, PrivateStanza),
+    escalus:assert(is_iq_result, [PrivateStanza], escalus_client:wait_for_stanza(User)).
+

src/mod_private.erl

@@ -28,13 +28,16 @@
 -author('alexey@process-one.net').
 
 -behaviour(gen_mod).
+-behaviour(gdpr).
 
 -export([start/2,
          stop/1,
          process_sm_iq/4,
          remove_user/3,
          remove_user/2]).
 
+-export([get_personal_data/2]).
+
 -include("mongoose.hrl").
 -include("jlib.hrl").
 -xep([{xep, 49}, {version, "1.2"}]).
@@ -67,6 +70,48 @@
     LUser   :: binary(),
     LServer :: binary().
 
+-callback get_all_nss(LUser, LServer) -> NSs when
+    LUser   :: binary(),
+    LServer :: binary(),
+    NSs     :: [binary()].
+
+%%--------------------------------------------------------------------
+%% gdpr callback
+%%--------------------------------------------------------------------
+
+-spec get_personal_data(jid:user(), jid:server()) ->
+    [{gdpr:data_group(), gdpr:schema(), gdpr:entries()}].
+get_personal_data(Username, Server) ->
+    LUser = jid:nodeprep(Username),
+    LServer = jid:nameprep(Server),
+    Schema = ["ns", "xml"],
+    %% TODO: Replace separate `mod_private_mysql` backend with a switch for `rdbms` one
+    %% or simply always use dedicated queries when the RDBMS backend type is `mysql`
+    %%
+    %% We remove `mod_private_mysql` from the list because calling all possible backends resulted
+    %% in duplicate entries with RDBMS.
+    Backends = mongoose_lib:find_behaviour_implementations(mod_private) -- [mod_private_mysql],
+    Entries =
+    lists:flatmap(fun(B) ->
+                          get_personal_data_from_backend(B, LUser, LServer)
+                  end, Backends),
+    [{private, Schema, Entries}].
+
+get_personal_data_from_backend(Backend, LUser, LServer) ->
+    try
+        NSs = Backend:get_all_nss(LUser, LServer),
+        lists:map(
+          fun(NS) ->
+                  { NS, exml:to_binary(Backend:multi_get_data(LUser, LServer, [{NS, default}])) }
+          end, NSs)
+    catch
+        C:R ->
+            ?WARNING_MSG("event=cannot_retrieve_personal_data,"
+                         "backend=~p,class=~p,reason=~p,stacktrace=~p",
+                         [Backend, C, R, erlang:get_stacktrace()]),
+            []
+    end.
+
 %% ------------------------------------------------------------------
 %% gen_mod callbacks
 

src/mod_private_mnesia.erl

@@ -36,6 +36,8 @@
          multi_get_data/3,
          remove_user/2]).
 
+-export([get_all_nss/2]).
+
 -include("mongoose.hrl").
 -include("jlib.hrl").
 
@@ -66,6 +68,13 @@ set_data_t(LUser, LServer, NS, XML) ->
 multi_get_data(LUser, LServer, NS2Def) ->
     [get_data(LUser, LServer, NS, Default) || {NS, Default} <- NS2Def].
 
+get_all_nss(LUser, LServer) ->
+    F = fun() ->
+        select_namespaces_t(LUser, LServer)
+    end,
+    {atomic, NSs} = mnesia:transaction(F),
+    NSs.
+
 %% @doc Return stored value or default.
 get_data(LUser, LServer, NS, Default) ->
     case mnesia:dirty_read(private_storage, {LUser, LServer, NS}) of

src/mod_private_mysql.erl

@@ -9,6 +9,8 @@
          multi_get_data/3,
          remove_user/2]).
 
+-export([get_all_nss/2]).
+
 -include("mongoose.hrl").
 -include("jlib.hrl").
 
@@ -52,3 +54,9 @@ select_value({NS, Def}, RowsDict) ->
 remove_user(LUser, LServer) ->
     SLUser = mongoose_rdbms:escape_string(LUser),
     rdbms_queries:del_user_private_storage(LServer, SLUser).
+
+get_all_nss(LUser, LServer) ->
+    EscLUser = mongoose_rdbms:escape_string(LUser),
+    {selected, Res} = rdbms_queries:get_all_private_namespaces(LServer, EscLUser),
+    lists:map(fun({R}) -> R end, Res).
+

src/mod_private_rdbms.erl

@@ -36,6 +36,8 @@
          multi_get_data/3,
          remove_user/2]).
 
+-export([get_all_nss/2]).
+
 -include("mongoose.hrl").
 -include("jlib.hrl").
 
@@ -75,6 +77,11 @@ get_data(LUser, LServer, NS, Default) ->
             Default
     end.
 
+get_all_nss(LUser, LServer) ->
+    EscLUser = mongoose_rdbms:escape_string(LUser),
+    {selected, Res} = rdbms_queries:get_all_private_namespaces(LServer, EscLUser),
+    lists:map(fun({R}) -> R end, Res).
+
 remove_user(LUser, LServer) ->
     SLUser = mongoose_rdbms:escape_string(LUser),
     rdbms_queries:del_user_private_storage(LServer, SLUser).

src/mod_private_riak.erl

@@ -23,6 +23,8 @@
          multi_get_data/3,
          remove_user/2]).
 
+-export([get_all_nss/2]).
+
 -include("mongoose.hrl").
 -include("jlib.hrl").
 
@@ -62,6 +64,18 @@ set_private_data(LUser, LServer, NS, XML) ->
     Obj = riakc_obj:new(bucket_type(LServer), key(LUser, NS), exml:to_binary(XML)),
     mongoose_riak:put(Obj).
 
+get_all_nss(LUser, LServer) ->
+    {ok, KeysWithUsername} = mongoose_riak:list_keys(bucket_type(LServer)),
+    lists:foldl(
+        fun(Key, Acc) ->
+            case binary:split(Key, <<"/">>) of
+                [LUser, ResultKey] -> [ResultKey | Acc];
+                _ -> Acc
+            end
+        end,
+        [], KeysWithUsername
+    ).
+
 get_private_data(LUser, LServer, NS, Default) ->
     case mongoose_riak:get(bucket_type(LServer), key(LUser, NS)) of
         {ok, Obj} ->

src/rdbms/rdbms_queries.erl

@@ -67,6 +67,7 @@
          get_subscription_t/3,
          set_private_data/4,
          set_private_data_sql/3,
+         get_all_private_namespaces/2,
          get_private_data/3,
          multi_get_private_data/3,
          multi_set_private_data/3,
@@ -641,6 +642,12 @@ set_private_data_sql(Username, LXMLNS, SData) ->
                       mongoose_rdbms:use_escaped_string(LXMLNS), ", ",
                       mongoose_rdbms:use_escaped_string(SData), ");"]].
 
+get_all_private_namespaces(LServer, Username) ->
+    mongoose_rdbms:sql_query(
+      LServer,
+      [<<"select namespace from private_storage where username=">>,
+       mongoose_rdbms:use_escaped_string(Username), " ;"]).
+
 get_private_data(LServer, Username, LXMLNS) ->
     mongoose_rdbms:sql_query(
       LServer,