Merge 040f949 into 2f1fb1a

big_tests/tests/gdpr_SUITE.erl

@@ -21,6 +21,7 @@
          dont_remove_other_user_private_xml/1,
          retrieve_roster/1,
          retrieve_mam_pm/1,
+         retrieve_mam_muc/1,
          retrieve_mam_muc_light/1,
          retrieve_mam_pm_and_muc_light_interfere/1,
          retrieve_mam_pm_and_muc_light_dont_interfere/1,
@@ -131,7 +132,7 @@ groups() ->
      {retrieve_personal_data_mam_rdbms, [], mam_testcases()},
      {retrieve_personal_data_mam_riak, [], mam_testcases()},
      {retrieve_personal_data_mam_cassandra, [], mam_testcases()},
-     {retrieve_personal_data_mam_elasticsearch, [], [retrieve_mam_pm]},
+     {retrieve_personal_data_mam_elasticsearch, [], [retrieve_mam_muc | mam_testcases()]},
      {remove_personal_data, [], removal_testcases()},
      {remove_personal_data_with_mods_disabled, [], removal_testcases()},
      {remove_personal_data_inbox, [], [remove_inbox, remove_inbox_muclight, remove_inbox_muc]}].
@@ -247,7 +248,8 @@ init_per_testcase(CN, Config) when CN =:= remove_private;
     private_started(),
     escalus:init_per_testcase(CN, Config);
 
-init_per_testcase(CN, Config) when CN =:= retrieve_mam_muc_light;
+init_per_testcase(CN, Config) when CN =:= retrieve_mam_muc;
+                                   CN =:= retrieve_mam_muc_light;
                                    CN =:= retrieve_mam_pm_and_muc_light_interfere;
                                    CN =:= retrieve_mam_pm_and_muc_light_dont_interfere;
                                    CN =:= retrieve_mam_pm ->
@@ -333,7 +335,12 @@ mam_required_modules(retrieve_mam_pm_and_muc_light_interfere, Backend) ->
                      {rdbms_message_format, simple}, %% ignored for any other than rdbms backend
                      {pm, [{archive_groupchats, true}]},
                      {muc, [{host, "muclight.@HOST@"}]}]},
-     {mod_muc_light, [{host, "muclight.@HOST@"}]}].
+     {mod_muc_light, [{host, "muclight.@HOST@"}]}];
+mam_required_modules(retrieve_mam_muc, Backend) ->
+    [{mod_mam_meta, [{backend, Backend},
+                     {pm, [{archive_groupchats, false}]},
+                     {muc, [{host, "muc.@HOST@"}]}]},
+     {mod_muc, [{host, "muc.@HOST@"}]}].
 
 pick_enabled_backend() ->
     BackendsList = [
@@ -594,6 +601,47 @@ retrieve_mam_pm(Config) ->
         end,
     escalus_fresh:story(Config, [{alice, 1}, {bob, 1}], F).
 
+retrieve_mam_muc(Config) ->
+    F = fun(Alice, Bob, Kate) ->
+            AliceUserCfg = escalus_users:get_user_by_name(alice),
+            RoomCfg = muc_helper:start_fresh_room([], AliceUserCfg, <<"someroom">>, []),
+            [Room, Domain] = [proplists:get_value(Key, RoomCfg) || Key <- [room, muc_host]],
+            AllRoomMembers = [Alice, Bob, Kate],
+
+            muc_helper:enter_room(RoomCfg, [{Alice, <<"Nancy">>},
+                                            {Bob, <<"Sid">>},
+                                            {Kate, <<"Johnny">>}]),
+
+            Body1 = <<"1some simple muc message">>,
+            Body2 = <<"2another one">>,
+            Body3 = <<"3third message">>,
+            muc_helper:send_to_room(RoomCfg, Alice, Body1),
+            muc_helper:verify_message_received(RoomCfg, AllRoomMembers, <<"Nancy">>, Body1),
+            muc_helper:send_to_room(RoomCfg, Alice, Body2),
+            muc_helper:verify_message_received(RoomCfg, AllRoomMembers, <<"Nancy">>, Body2),
+            muc_helper:send_to_room(RoomCfg, Bob, Body3),
+            muc_helper:verify_message_received(RoomCfg, AllRoomMembers, <<"Sid">>, Body3),
+
+            mam_helper:wait_for_room_archive_size(Domain, Room, 3),
+
+            ExpectedItemsAlice = [#{"message" => [{contains, binary_to_list(Body1)}]},
+                                  #{"message" => [{contains, binary_to_list(Body2)}]}],
+
+            ExpectedItemsBob = [#{"message" => [{contains, binary_to_list(Body3)}]}],
+
+            BackendModule = choose_mam_backend(Config, mam_muc),
+            maybe_stop_and_unload_module(mod_mam_muc, BackendModule, Config),
+
+            retrieve_and_validate_personal_data(
+                Alice, Config, "mam_muc", ["id", "message"], ExpectedItemsAlice, ["message"]),
+            retrieve_and_validate_personal_data(
+                Bob, Config, "mam_muc", ["id", "message"], ExpectedItemsBob, ["message"]),
+            refute_personal_data(Kate, Config, "mam_muc"),
+
+            muc_helper:destroy_room(RoomCfg)
+        end,
+    escalus_fresh:story(Config, [{alice, 1}, {bob, 1}, {kate, 1}], F).
+
 retrieve_mam_muc_light(Config) ->
     F = fun(Alice, Bob, Kate) ->
             RoomJid = muc_light_helper:given_muc_light_room(undefined, Alice, [{Bob, member}, {Kate, member}]),
@@ -1514,3 +1562,4 @@ given_fresh_muc_room(UserSpec, RoomOpts) ->
     From = muc_helper:generate_rpc_jid({user, UserSpec}),
     muc_helper:create_instant_room(<<"localhost">>, RoomName, From, Username, RoomOpts),
     {ok, RoomName}.
+

big_tests/tests/muc_helper.erl

@@ -70,7 +70,62 @@ muc_host() ->
 start_room(Config, User, Room, Nick, Opts) ->
     From = generate_rpc_jid(User),
     create_instant_room(<<"localhost">>, Room, From, Nick, Opts),
-    [{nick, Nick}, {room, Room} | Config].
+    RoomJID = room_address(Room),
+    [{nick, Nick}, {room, Room}, {room_jid, RoomJID}, {muc_host, muc_host()} | Config].
+
+start_fresh_room(Config, User, Nick, Opts) ->
+    Room = fresh_room_name(),
+    start_room(Config, User, Room, Nick, Opts).
+
+enter_room(Config, UsersAndNicks) ->
+    [Room, RoomJid] = [proplists:get_value(Key, Config) || Key <- [room, room_jid]],
+    lists:foldl(fun({User, Nick}, Acc) ->
+                        escalus:send(User, stanza_muc_enter_room(Room, Nick)),
+                        wait_for_presence(RoomJid, User, length(Acc)),
+                        foreach_recipient([User | Acc],
+                                          fun(Stanza) ->
+                                                  validate_presence(Stanza,
+                                                                    RoomJid,
+                                                                    Nick)
+                                          end),
+                        Subject = escalus:wait_for_stanza(User),
+                        validate_subject_message(Subject, RoomJid),
+                        [User | Acc]
+                end,
+                [],UsersAndNicks).
+
+wait_for_presence(_, _, 0) ->
+    ok;
+wait_for_presence(RoomJid, User, N) ->
+    Stanza = escalus:wait_for_stanza(User),
+    validate_presence(Stanza, RoomJid),
+    wait_for_presence(RoomJid, User, N - 1).
+
+validate_presence(Stanza, RoomJid) ->
+    escalus:assert(is_presence_stanza, [], Stanza),
+    [RoomJid, _] = binary:split(exml_query:attr(Stanza, <<"from">>), <<"/">>).
+
+validate_presence(Stanza, RoomJid, Nick) ->
+    [RoomJid, Nick] = validate_presence(Stanza, RoomJid).
+
+validate_subject_message(Stanza, RoomJid) ->
+    RoomJid = exml_query:attr(Stanza, <<"from">>),
+    #xmlel{} = exml_query:subelement(Stanza, <<"subject">>).
+
+verify_message_received(Config, Users, Nick, TextBody) ->
+    RoomJid = proplists:get_value(room_jid, Config),
+    foreach_recipient(Users, muc_msg_verify(RoomJid, Nick, TextBody)).
+
+muc_msg_verify(RoomBareJID, NickName, MsgText) ->
+    fun(Msg) ->
+        escalus:assert(is_groupchat_message, [MsgText], Msg),
+        [RoomBareJID, NickName] = binary:split(exml_query:attr(Msg, <<"from">>), <<"/">>)
+    end.
+
+send_to_room(RoomCfg, User, TextBody) ->
+    RoomJid = proplists:get_value(room_jid, RoomCfg),
+    Stanza = escalus_stanza:groupchat_to(RoomJid, TextBody),
+    escalus:send(User, Stanza).
 
 generate_rpc_jid({_,User}) ->
     {username, Username} = lists:keyfind(username, 1, User),

priv/elasticsearch/muc.json

@@ -8,6 +8,9 @@
 				"room": {
 					"type": "keyword"
 				},
+				"from_jid" : {
+					"type": "keyword"
+				},
 				"source_jid": {
 					"type": "keyword"
 				},

src/mam/mod_mam_muc.erl

@@ -242,7 +242,7 @@ archive_room_packet(Packet, FromNick, FromJID=#jid{}, RoomJID=#jid{}, Role, Affi
             MessID = generate_message_id(),
             Packet1 = replace_x_user_element(FromJID, Role, Affiliation, Packet),
             Result = archive_message(Host, MessID, ArcID,
-                                     RoomJID, SrcJID, SrcJID, incoming, Packet1),
+                                     RoomJID, FromJID, SrcJID, incoming, Packet1),
             %% Packet2 goes to archive, Packet to other users
             case Result of
                 ok ->

src/mam/mod_mam_muc_cassandra_arch.erl

@@ -156,7 +156,7 @@ insert_query_cql() ->
         "VALUES (?, ?, ?, ?, ?)".
 
 archive_message(Result, Host, MessID, _RoomID,
-                LocJID, NickName, NickName, Dir, Packet) ->
+                LocJID, _FromJID, NickName, Dir, Packet) ->
     try
         archive_message2(Result, Host, MessID,
                          LocJID, NickName, NickName, Dir, Packet)

src/mam/mod_mam_muc_elasticsearch_arch.erl

@@ -31,6 +31,9 @@
 -export([remove_archive/4]).
 -export([archive_size/4]).
 
+%gdpr
+-export([get_mam_muc_gdpr_data/2]).
+
 -include("mongoose.hrl").
 -include("mongoose_rsm.hrl").
 -include("mod_mam.hrl").
@@ -56,12 +59,29 @@ stop(Host) ->
 %%-------------------------------------------------------------------
 %% ejabberd_gen_mam_archive callbacks
 %%-------------------------------------------------------------------
+-spec get_mam_muc_gdpr_data(jid:username(), jid:server()) ->
+    {ok, ejabberd_gen_mam_archive:mam_muc_gdpr_data()}.
+get_mam_muc_gdpr_data(User, Host) ->
+    Source = jid:make(User, Host, <<"">>),
+    BinSource = mod_mam_utils:bare_jid(Source),
+    Filter = #{term => #{from_jid => BinSource}},
+    Sorting = #{mam_id => #{order => asc}},
+    SearchQuery = #{query => #{bool => #{filter => Filter}},
+                    sort => Sorting},
+    case mongoose_elasticsearch:search(?INDEX_NAME, ?TYPE_NAME, SearchQuery) of
+        {ok, #{<<"hits">> := #{<<"hits">> := Hits}}} ->
+            Messages = lists:map(fun hit_to_gdpr_mam_message/1, Hits),
+            {ok, Messages};
+        {error, _} ->
+            {ok, []}
+    end.
 
-archive_message(_Result, Host, MessageId, _UserId, RoomJid, _SourceJid, SourceJid, _Dir, Packet) ->
+archive_message(_Result, Host, MessageId, _UserId, RoomJid, FromJID, SourceJid, _Dir, Packet) ->
     Room = mod_mam_utils:bare_jid(RoomJid),
     SourceBinJid = mod_mam_utils:full_jid(SourceJid),
+    From = mod_mam_utils:bare_jid(FromJID),
     DocId = make_document_id(Room, MessageId),
-    Doc = make_document(MessageId, Room, SourceBinJid, Packet),
+    Doc = make_document(MessageId, Room, SourceBinJid, Packet, From),
     case mongoose_elasticsearch:insert_document(?INDEX_NAME, ?TYPE_NAME, DocId, Doc) of
         {ok, _} ->
             ok;
@@ -144,10 +164,11 @@ hooks(Host) ->
 make_document_id(Room, MessageId) ->
     <<Room/binary, $$, (integer_to_binary(MessageId))/binary>>.
 
--spec make_document(mod_mam:message_id(), binary(), binary(), exml:element()) ->
-    map().
-make_document(MessageId, Room, SourceBinJid, Packet) ->
+-spec make_document(mod_mam:message_id(), binary(), binary(), exml:element(),
+                    binary()) -> map().
+make_document(MessageId, Room, SourceBinJid, Packet, FromJID) ->
     #{mam_id     => MessageId,
+      from_jid   => FromJID,
       room       => Room,
       source_jid => SourceBinJid,
       message    => exml:to_binary(Packet),
@@ -257,6 +278,11 @@ hit_to_mam_message(#{<<"_source">> := JSON}) ->
     {ok, Stanza} = exml:parse(Packet),
     {MessageId, jid:from_binary(SourceJid), Stanza}.
 
+hit_to_gdpr_mam_message(#{<<"_source">> := JSON}) ->
+    MessageId = maps:get(<<"mam_id">>, JSON),
+    Packet = maps:get(<<"message">>, JSON),
+    {integer_to_binary(MessageId), Packet}.
+
 %% Usage of RSM affects the `"total"' value returned by ElasticSearch. Per RSM spec, the count
 %% returned by the query should represent the size of the whole result set, which in case of MAM
 %% is bound only by the MAM filters.

src/mam/mod_mam_rdbms_arch.erl

@@ -20,6 +20,7 @@
 
 -export([archive_size/4,
          archive_message/9,
+         archive_message_muc/9,
          lookup_messages/3,
          remove_archive/4]).
 
@@ -162,7 +163,7 @@ start_muc(Host, _Opts) ->
         true ->
             ok;
         false ->
-            ejabberd_hooks:add(mam_muc_archive_message, Host, ?MODULE, archive_message, 50)
+            ejabberd_hooks:add(mam_muc_archive_message, Host, ?MODULE, archive_message_muc, 50)
     end,
     ejabberd_hooks:add(mam_muc_archive_size, Host, ?MODULE, archive_size, 50),
     ejabberd_hooks:add(mam_muc_lookup_messages, Host, ?MODULE, lookup_messages, 50),
@@ -176,7 +177,7 @@ stop_muc(Host) ->
         true ->
             ok;
         false ->
-            ejabberd_hooks:delete(mam_muc_archive_message, Host, ?MODULE, archive_message, 50)
+            ejabberd_hooks:delete(mam_muc_archive_message, Host, ?MODULE, archive_message_muc, 50)
     end,
     ejabberd_hooks:delete(mam_muc_archive_size, Host, ?MODULE, archive_size, 50),
     ejabberd_hooks:delete(mam_muc_lookup_messages, Host, ?MODULE, lookup_messages, 50),
@@ -208,6 +209,14 @@ index_hint_sql(Host) ->
     end.
 
 
+-spec archive_message_muc(_Result, Host :: jid:server(),
+                          MessID :: mod_mam:message_id(), UserID :: mod_mam:archive_id(),
+                          LocJID :: jid:jid(), RemJID :: jid:jid(),
+                          SrcJID :: jid:jid(), Dir :: atom(), Packet :: any()) -> ok.
+archive_message_muc(Result, Host, MessID, UserID, LocJID, _RemJID, SrcJID, Dir, Packet) ->
+    archive_message(Result, Host, MessID, UserID, LocJID, SrcJID, SrcJID, Dir, Packet).
+
+
 -spec archive_message(_Result, Host :: jid:server(),
                       MessID :: mod_mam:message_id(), UserID :: mod_mam:archive_id(),
                       LocJID :: jid:jid(), RemJID :: jid:jid(),

src/mam/mod_mam_riak_timed_arch_yz.erl

@@ -132,8 +132,8 @@ archive_message(_Result, Host, MessId, _UserID, LocJID, RemJID, SrcJID, _Dir, Pa
             {error, Reason}
     end.
 
-archive_message_muc(_Result, Host, MessId, _UserID, LocJID, RemJID, SrcJID, _Dir, Packet) ->
-    RemJIDMuc = maybe_muc_jid(RemJID),
+archive_message_muc(_Result, Host, MessId, _UserID, LocJID, _FromJID, SrcJID, _Dir, Packet) ->
+    RemJIDMuc = maybe_muc_jid(SrcJID),
     try
         archive_message(Host, MessId, LocJID, RemJIDMuc, SrcJID, Packet, muc)
     catch _Type:Reason ->