From 1560bd6a175c60d8b6915322ded93782ed93b26a Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Mon, 29 Apr 2019 13:37:43 +0200 Subject: [PATCH 01/17] Improve mod offline retrival test --- big_tests/tests/gdpr_SUITE.erl | 55 ++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index dd9f5362bd0..0feb5ef76b7 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -201,10 +201,14 @@ retrieve_mam(_Config) -> ok. retrieve_offline(Config) -> - escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) -> + escalus:fresh_story(Config, [{alice, 1}, {bob, 1}, {kate, 1}], fun(Alice, Bob, Kate) -> mongoose_helper:logout_user(Config, Alice), - Body = <<"Here's Johnny!">>, - escalus:send(Bob, escalus_stanza:chat_to(Alice, Body)), + Body1 = <<"Hey!">>, + Body2 = <<"Here is Johnny!">>, + Body3 = <<"Where is Johnny ?">>, + escalus:send(Bob, escalus_stanza:chat_to(Alice, Body1)), + escalus:send(Bob, escalus_stanza:chat_to(Alice, Body2)), + escalus:send(Kate, escalus_stanza:chat_to(Alice, Body3)), %% Well, jid_to_lower works for any binary :) AliceU = escalus_utils:jid_to_lower(escalus_client:username(Alice)), AliceS = escalus_utils:jid_to_lower(escalus_client:server(Alice)), @@ -212,12 +216,25 @@ retrieve_offline(Config) -> fun() -> mongoose_helper:successful_rpc(mod_offline_backend, count_offline_messages, [AliceU, AliceS, 1]) - end, 1), + end, 3), BobJid = escalus_client:short_jid(Bob), + AliceJid = escalus_client:short_jid(Alice), + KateJid = escalus_client:short_jid(Kate), ExpectedHeader = ["timestamp", "from", "to", "packet"], ExpectedItems = [ - #{ "packet" => [{contains, Body}], "from" => BobJid } + #{ "packet" => [{contains, Body1}], + "from" => binary_to_list(BobJid), + "to" => binary_to_list(AliceJid), + "timestamp" => [{validate, fun validate_datetime/1}]}, + #{ "packet" => [{contains, Body2}], + "from" => binary_to_list(BobJid), + "to" => binary_to_list(AliceJid), + "timestamp" => [{validate, fun validate_datetime/1}]}, + #{ "packet" => [{contains, Body3}], + "from" => binary_to_list(KateJid), + "to" => binary_to_list(AliceJid), + "timestamp" => [{validate, fun validate_datetime/1}]} ], maybe_stop_and_unload_module(mod_offline, mod_offline_backend, Config), retrieve_and_validate_personal_data( @@ -469,7 +486,11 @@ validate_sorted_personal_maps([Map | RMaps], [Checks | RChecks]) -> maps:fold(fun(K, Conditions, _) -> validate_personal_item(maps:get(K, Map), Conditions) end, ok, Checks), +<<<<<<< 0b85d95a6467c76e9c3cc66562fc6ff8a5839525 validate_sorted_personal_maps(RMaps, RChecks). +======= + validate_sorted_personal_maps(RMaps, RChecks). +>>>>>>> Improve mod offline retrival test validate_personal_item(_Value, []) -> ok; @@ -477,6 +498,9 @@ validate_personal_item(ExactValue, ExactValue) -> ok; validate_personal_item(Value, [{contains, String} | RConditions]) -> {match, _} = re:run(Value, String), + validate_personal_item(Value, RConditions); +validate_personal_item(Value, [{validate, Validator} | RConditions]) when is_function(Validator) -> + true = Validator(Value), validate_personal_item(Value, RConditions). retrieve_and_decode_personal_data(Client, Config, FilePrefix) -> @@ -551,3 +575,24 @@ item_content_xml(Data) -> attrs = [{<<"xmlns">>, <<"http://www.w3.org/2005/Atom">>}], children = [#xmlcdata{content = Data}]}. +validate_datetime(TimeStr) -> + [Date, Time] = string:tokens(TimeStr, "T"), + validate_date(Date), + validate_time(Time). + +validate_date(Date) -> + [Y, M, D] = string:tokens(Date, "-"), + Date1 = {list_to_integer(Y), list_to_integer(M), list_to_integer(D)}, + calendar:valid_date(Date1). + +validate_time(Time) -> + [T | _] = string:tokens(Time, "Z"), + validate_time1(T). + + +validate_time1(Time) -> + [H, M, S] = string:tokens(Time, ":"), + check_list([{H, 24}, {M, 60}, {S, 60}]). + +check_list(List) -> + lists:all(fun({V, L}) -> I = list_to_integer(V), I >= 0 andalso I < L end, List). From b84d63a464eff9d9d4945954880bff314d9b2f91 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Mon, 29 Apr 2019 13:37:55 +0200 Subject: [PATCH 02/17] Retrieve offline data from mnesia --- src/mod_offline_mnesia.erl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/mod_offline_mnesia.erl b/src/mod_offline_mnesia.erl index 821b597c61a..c9671c3c2be 100644 --- a/src/mod_offline_mnesia.erl +++ b/src/mod_offline_mnesia.erl @@ -36,6 +36,8 @@ remove_old_messages/2, remove_user/2]). +-export([get_personal_data/2]). + -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_offline.hrl"). @@ -162,3 +164,19 @@ remove_old_message(TimeStamp, Rec) -> is_old_message(MaxAllowedTimeStamp, #offline_msg{timestamp=TimeStamp}) -> TimeStamp < MaxAllowedTimeStamp. + +get_personal_data(Username, Server) -> + LUser = jid:nodeprep(Username), + LServer = jid:nodeprep(Server), + US = {LUser, LServer}, + {atomic, Messages} = mnesia:transaction(fun() -> mnesia:wread({offline_msg, US}) end), + [{offline, ["timestamp", "from", "to", "packet"], process_offline_messages(Messages)}]. + +process_offline_messages(MsgList) -> + [process_offline_msg(Msg) || Msg <- MsgList]. + +process_offline_msg(#offline_msg{timestamp = Timestamp, from = From, to = To, packet = Packet}) -> + NowUniversal = calendar:now_to_universal_time(Timestamp), + {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), + UTC = list_to_binary(UTCTime ++ UTCDiff), + {UTC, jid:to_binary(jid:to_bare(From)), jid:to_binary(jid:to_bare(To)), exml:to_binary(Packet)}. From b1ce329d467f68f0b6aa6ff966176c33b058e620 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Mon, 29 Apr 2019 14:00:21 +0200 Subject: [PATCH 03/17] Add support for RDBMS --- src/mod_offline_mnesia.erl | 8 ++++---- src/mod_offline_rdbms.erl | 22 ++++++++++++++++++++++ src/rdbms/rdbms_queries.erl | 4 ++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/mod_offline_mnesia.erl b/src/mod_offline_mnesia.erl index c9671c3c2be..596d29972a2 100644 --- a/src/mod_offline_mnesia.erl +++ b/src/mod_offline_mnesia.erl @@ -170,12 +170,12 @@ get_personal_data(Username, Server) -> LServer = jid:nodeprep(Server), US = {LUser, LServer}, {atomic, Messages} = mnesia:transaction(fun() -> mnesia:wread({offline_msg, US}) end), - [{offline, ["timestamp", "from", "to", "packet"], process_offline_messages(Messages)}]. + [{offline, ["timestamp", "from", "to", "packet"], offline_messages_to_gdpr_format(Messages)}]. -process_offline_messages(MsgList) -> - [process_offline_msg(Msg) || Msg <- MsgList]. +offline_messages_to_gdpr_format(MsgList) -> + [offline_msg_to_gdpr_format(Msg) || Msg <- MsgList]. -process_offline_msg(#offline_msg{timestamp = Timestamp, from = From, to = To, packet = Packet}) -> +offline_msg_to_gdpr_format(#offline_msg{timestamp = Timestamp, from = From, to = To, packet = Packet}) -> NowUniversal = calendar:now_to_universal_time(Timestamp), {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), UTC = list_to_binary(UTCTime ++ UTCDiff), diff --git a/src/mod_offline_rdbms.erl b/src/mod_offline_rdbms.erl index 7b4ed16a3cd..783c8fd1b51 100644 --- a/src/mod_offline_rdbms.erl +++ b/src/mod_offline_rdbms.erl @@ -34,6 +34,8 @@ remove_old_messages/2, remove_user/2]). +-export([get_personal_data/2]). + -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_offline.hrl"). @@ -151,3 +153,23 @@ maybe_encode_timestamp(never) -> mongoose_rdbms:escape_null(); maybe_encode_timestamp(TimeStamp) -> encode_timestamp(TimeStamp). + +get_personal_data(Username, Server) -> + LUser = jid:nodeprep(Username), + LServer = jid:nodeprep(Server), + SUser = mongoose_rdbms:escape_string(LUser), + SServer = mongoose_rdbms:escape_string(LServer), + TimeStamp = p1_time_compat:timestamp(), + STimeStamp = encode_timestamp(TimeStamp), + User = jid:to_binary({LUser, LServer}), + {atomic, {selected, Rows}} = rdbms_queries:fetch_offline_messages(LServer, SUser, SServer, STimeStamp), + [{offline, ["timestamp", "from", "to", "packet"], rows_to_gdpr_format(User, Rows)}]. + + +rows_to_gdpr_format(User, Rows) -> + [row_to_gdpr_format(User, Row) || Row <- Rows]. + +row_to_gdpr_format(User, {STimeStamp, SFrom, SPacket}) -> + [STimeStamp, SFrom, User, SPacket]. + + diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index 27e2ab6d496..f282263d043 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -96,6 +96,7 @@ prepare_offline_message/6, push_offline_messages/2, pop_offline_messages/4, + fetch_offline_messages/4, count_offline_messages/4, remove_old_offline_messages/2, remove_expired_offline_messages/2, @@ -896,6 +897,9 @@ pop_offline_messages(LServer, SUser, SServer, STimeStamp) -> end, mongoose_rdbms:sql_transaction(LServer, F). +fetch_offline_messages(LServer, SUser, SServer, STimeStamp) -> + mongoose_rdbms:sql_query(LServer, select_offline_messages_sql(SUser, SServer, STimeStamp)). + select_offline_messages_sql(SUser, SServer, STimeStamp) -> [<<"select timestamp, from_jid, packet from offline_message " "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " From 6faa87f757dd7f9c0ad30db4b9699dd40ba9d89a Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Mon, 29 Apr 2019 14:22:19 +0200 Subject: [PATCH 04/17] Change timestamp format in rdbms return --- src/mod_offline_rdbms.erl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/mod_offline_rdbms.erl b/src/mod_offline_rdbms.erl index 783c8fd1b51..9aab0850706 100644 --- a/src/mod_offline_rdbms.erl +++ b/src/mod_offline_rdbms.erl @@ -161,8 +161,8 @@ get_personal_data(Username, Server) -> SServer = mongoose_rdbms:escape_string(LServer), TimeStamp = p1_time_compat:timestamp(), STimeStamp = encode_timestamp(TimeStamp), - User = jid:to_binary({LUser, LServer}), - {atomic, {selected, Rows}} = rdbms_queries:fetch_offline_messages(LServer, SUser, SServer, STimeStamp), + User = jid:to_binary({Username, LServer}), + {selected, Rows} = rdbms_queries:fetch_offline_messages(LServer, SUser, SServer, STimeStamp), [{offline, ["timestamp", "from", "to", "packet"], rows_to_gdpr_format(User, Rows)}]. @@ -170,6 +170,10 @@ rows_to_gdpr_format(User, Rows) -> [row_to_gdpr_format(User, Row) || Row <- Rows]. row_to_gdpr_format(User, {STimeStamp, SFrom, SPacket}) -> - [STimeStamp, SFrom, User, SPacket]. + Timestamp = usec:to_now(mongoose_rdbms:result_to_integer(STimeStamp)), + NowUniversal = calendar:now_to_universal_time(Timestamp), + {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), + UTC = list_to_binary(UTCTime ++ UTCDiff), + [UTC, jid:to_binary(jid:binary_to_bare(SFrom)), User, SPacket]. From 22a771774bd8a89e28ae167a7cd2c617d472b443 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Tue, 30 Apr 2019 14:52:07 +0200 Subject: [PATCH 05/17] Fetch personal data for offline in riak backend --- big_tests/tests/gdpr_SUITE.erl | 2 +- src/mod_offline_riak.erl | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 0feb5ef76b7..565911a4b0d 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -215,7 +215,7 @@ retrieve_offline(Config) -> mongoose_helper:wait_until( fun() -> mongoose_helper:successful_rpc(mod_offline_backend, count_offline_messages, - [AliceU, AliceS, 1]) + [AliceU, AliceS, 10]) end, 3), BobJid = escalus_client:short_jid(Bob), diff --git a/src/mod_offline_riak.erl b/src/mod_offline_riak.erl index 9abd814f8a2..2b3dc60e278 100644 --- a/src/mod_offline_riak.erl +++ b/src/mod_offline_riak.erl @@ -31,6 +31,7 @@ -export([remove_old_messages/2]). -export([remove_user/2]). -export([count_offline_messages/3]). +-export([get_personal_data/2]). -include("mongoose.hrl"). -include("jlib.hrl"). @@ -184,3 +185,37 @@ maybe_decode_timestamp(?INFINITY) -> maybe_decode_timestamp(TS) -> usec:to_now(TS). + +get_personal_data(Username, Server) -> + LUser = jid:nodeprep(Username), + LServer = jid:nodeprep(Server), + [{offline, ["timestamp", "from", "to", "packet"], []}]. + +fetch_messages(LUser, LServer) -> + Keys = read_user_idx(LUser, LServer), + To = jid:to_binary(jid:make(LUser, LServer, <<>>)), + Msgs = [fetch_msg(Key, LServer, To) || Key <- Keys], + {ok, lists:flatten(Msgs)}. + +fetch_msg(Key, LServer, To) -> + try + {ok, Obj} = mongoose_riak:get(bucket_type(LServer), Key), + + PacketRaw = riakc_obj:get_value(Obj), + {ok, Packet} = exml:parse(PacketRaw), + MD = riakc_obj:get_update_metadata(Obj), + [Timestamp] = riakc_obj:get_secondary_index(MD, ?TIMESTAMP_IDX), + From = riakc_obj:get_user_metadata_entry(MD, <<"from">>), + + NowUniversal = calendar:now_to_universal_time(usec:to_now(Timestamp)), + {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), + UTC = list_to_binary(UTCTime ++ UTCDiff), + {UTC, jid:to_binary(jid:binary_to_bare(From)), To, exml:to_binary(Packet)} + + catch + Error:Reason -> + ?WARNING_MSG("issue=~p, action=reading_key, host=~s, reason=~p, stack_trace=~p", + [Error, LServer, Reason, erlang:get_stacktrace()]), + [] + end. + From e5fe5f1c373a96dd1d9407fbf34971492e00aee5 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Mon, 6 May 2019 12:33:44 +0200 Subject: [PATCH 06/17] Riak support for mod offline --- src/mod_offline_riak.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mod_offline_riak.erl b/src/mod_offline_riak.erl index 2b3dc60e278..e2e0d5e1b9c 100644 --- a/src/mod_offline_riak.erl +++ b/src/mod_offline_riak.erl @@ -187,15 +187,15 @@ maybe_decode_timestamp(TS) -> get_personal_data(Username, Server) -> - LUser = jid:nodeprep(Username), LServer = jid:nodeprep(Server), - [{offline, ["timestamp", "from", "to", "packet"], []}]. + [{offline, ["timestamp", "from", "to", "packet"], fetch_messages(Username, LServer)}]. -fetch_messages(LUser, LServer) -> +fetch_messages(Username, LServer) -> + LUser = jid:nodeprep(Username), Keys = read_user_idx(LUser, LServer), - To = jid:to_binary(jid:make(LUser, LServer, <<>>)), - Msgs = [fetch_msg(Key, LServer, To) || Key <- Keys], - {ok, lists:flatten(Msgs)}. + User = jid:to_binary({Username, LServer}), + Msgs = [fetch_msg(Key, LServer, User) || Key <- Keys], + lists:flatten(Msgs). fetch_msg(Key, LServer, To) -> try From f2990101ff281effa1ca66dc55e3eed88a0a86c7 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Mon, 6 May 2019 18:39:54 +0200 Subject: [PATCH 07/17] dont do useless parsing, fix var names in mod_offline_riak --- src/mod_offline_riak.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/mod_offline_riak.erl b/src/mod_offline_riak.erl index e2e0d5e1b9c..e741e782293 100644 --- a/src/mod_offline_riak.erl +++ b/src/mod_offline_riak.erl @@ -193,16 +193,15 @@ get_personal_data(Username, Server) -> fetch_messages(Username, LServer) -> LUser = jid:nodeprep(Username), Keys = read_user_idx(LUser, LServer), - User = jid:to_binary({Username, LServer}), - Msgs = [fetch_msg(Key, LServer, User) || Key <- Keys], + ToJid = jid:to_binary({Username, LServer}), + Msgs = [fetch_msg(Key, LServer, ToJid) || Key <- Keys], lists:flatten(Msgs). -fetch_msg(Key, LServer, To) -> +fetch_msg(Key, LServer, ToJid) -> try {ok, Obj} = mongoose_riak:get(bucket_type(LServer), Key), PacketRaw = riakc_obj:get_value(Obj), - {ok, Packet} = exml:parse(PacketRaw), MD = riakc_obj:get_update_metadata(Obj), [Timestamp] = riakc_obj:get_secondary_index(MD, ?TIMESTAMP_IDX), From = riakc_obj:get_user_metadata_entry(MD, <<"from">>), @@ -210,7 +209,7 @@ fetch_msg(Key, LServer, To) -> NowUniversal = calendar:now_to_universal_time(usec:to_now(Timestamp)), {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), UTC = list_to_binary(UTCTime ++ UTCDiff), - {UTC, jid:to_binary(jid:binary_to_bare(From)), To, exml:to_binary(Packet)} + {UTC, jid:to_binary(jid:binary_to_bare(From)), ToJid, PacketRaw} catch Error:Reason -> From eccb4e91cb029add65ea0287f812951c296abba7 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Mon, 6 May 2019 18:41:29 +0200 Subject: [PATCH 08/17] Call generic mod_offline instead of backend implementations --- src/mod_offline_rdbms.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mod_offline_rdbms.erl b/src/mod_offline_rdbms.erl index 9aab0850706..2dc39a0e628 100644 --- a/src/mod_offline_rdbms.erl +++ b/src/mod_offline_rdbms.erl @@ -161,19 +161,19 @@ get_personal_data(Username, Server) -> SServer = mongoose_rdbms:escape_string(LServer), TimeStamp = p1_time_compat:timestamp(), STimeStamp = encode_timestamp(TimeStamp), - User = jid:to_binary({Username, LServer}), + ToJid = jid:to_binary({Username, LServer}), {selected, Rows} = rdbms_queries:fetch_offline_messages(LServer, SUser, SServer, STimeStamp), - [{offline, ["timestamp", "from", "to", "packet"], rows_to_gdpr_format(User, Rows)}]. + [{offline, ["timestamp", "from", "to", "packet"], rows_to_gdpr_format(ToJid, Rows)}]. -rows_to_gdpr_format(User, Rows) -> - [row_to_gdpr_format(User, Row) || Row <- Rows]. +rows_to_gdpr_format(ToJid, Rows) -> + [row_to_gdpr_format(ToJid, Row) || Row <- Rows]. -row_to_gdpr_format(User, {STimeStamp, SFrom, SPacket}) -> +row_to_gdpr_format(ToJid, {STimeStamp, SFrom, SPacket}) -> Timestamp = usec:to_now(mongoose_rdbms:result_to_integer(STimeStamp)), NowUniversal = calendar:now_to_universal_time(Timestamp), {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), UTC = list_to_binary(UTCTime ++ UTCDiff), - [UTC, jid:to_binary(jid:binary_to_bare(SFrom)), User, SPacket]. + [UTC, jid:to_binary(jid:binary_to_bare(SFrom)), ToJid, SPacket]. From 15e2e6f530036af7ea99bf4619ee89eb480da5e1 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Wed, 8 May 2019 14:43:44 +0200 Subject: [PATCH 09/17] Move GDPR logic to generic mod_offline --- src/mod_offline.erl | 24 +++++++++++++++++++++++ src/mod_offline_mnesia.erl | 29 +++++++++++----------------- src/mod_offline_rdbms.erl | 39 +++++++++++++++----------------------- src/mod_offline_riak.erl | 27 +++++++++++++------------- 4 files changed, 63 insertions(+), 56 deletions(-) diff --git a/src/mod_offline.erl b/src/mod_offline.erl index cf985dc71f9..79aacd906cb 100644 --- a/src/mod_offline.erl +++ b/src/mod_offline.erl @@ -32,6 +32,7 @@ -xep([{xep, 22}, {version, "1.4"}]). -xep([{xep, 85}, {version, "2.1"}]). -behaviour(gen_mod). +-behaviour(gdpr). %% gen_mod handlers -export([start/2, stop/1]). @@ -57,6 +58,9 @@ %% helpers to be used from backend moudules -export([is_expired_message/2]). +%% GDPR related +-export([get_personal_data/2]). + -include("mongoose.hrl"). -include("jlib.hrl"). -include("amp.hrl"). @@ -94,6 +98,11 @@ LServer :: jid:lserver(), Reason :: term(), Result :: list(#offline_msg{}). +-callback fetch_messages(LUser, LServer) -> {ok, Result} | {error, Reason} when + LUser :: jid:luser(), + LServer :: jid:lserver(), + Reason :: term(), + Result :: list(#offline_msg{}). -callback write_messages(LUser, LServer, Msgs) -> ok | {error, Reason} when LUser :: jid:luser(), @@ -510,6 +519,21 @@ pop_messages(LUser, LServer) -> Other end. +get_personal_data(Username, Server) -> + LUser = jid:nodeprep(Username), + LServer = jid:nodeprep(Server), + {ok, Messages} = mod_offline_backend:fetch_messages(LUser, LServer), + [{offline, ["timestamp", "from", "to", "packet"], offline_messages_to_gdpr_format(Messages)}]. + +offline_messages_to_gdpr_format(MsgList) -> + [offline_msg_to_gdpr_format(Msg) || Msg <- MsgList]. + +offline_msg_to_gdpr_format(#offline_msg{timestamp = Timestamp, from = From, to = To, packet = Packet}) -> + NowUniversal = calendar:now_to_universal_time(Timestamp), + {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), + UTC = list_to_binary(UTCTime ++ UTCDiff), + {UTC, jid:to_binary(jid:to_bare(From)), jid:to_binary(jid:to_bare(To)), exml:to_binary(Packet)}. + skip_expired_messages(TimeStamp, Rs) -> [R || R <- Rs, not is_expired_message(TimeStamp, R)]. diff --git a/src/mod_offline_mnesia.erl b/src/mod_offline_mnesia.erl index 596d29972a2..cb3cf4e960d 100644 --- a/src/mod_offline_mnesia.erl +++ b/src/mod_offline_mnesia.erl @@ -30,14 +30,13 @@ -behaviour(mod_offline). -export([init/2, pop_messages/2, + fetch_messages/2, write_messages/3, count_offline_messages/3, remove_expired_messages/1, remove_old_messages/2, remove_user/2]). --export([get_personal_data/2]). - -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_offline.hrl"). @@ -67,6 +66,16 @@ pop_messages(LUser, LServer) -> {error, Reason} end. +fetch_messages(LUser, LServer) -> + US = {LUser, LServer}, + F = fun() -> mnesia:wread({offline_msg, US}) end, + case mnesia:transaction(F) of + {atomic, Rs} -> + {ok, Rs}; + {aborted, Reason} -> + {error, Reason} + end. + write_messages(_LUser, _LServer, Msgs) -> F = fun() -> write_messages_t(Msgs) end, case mnesia:transaction(F) of @@ -164,19 +173,3 @@ remove_old_message(TimeStamp, Rec) -> is_old_message(MaxAllowedTimeStamp, #offline_msg{timestamp=TimeStamp}) -> TimeStamp < MaxAllowedTimeStamp. - -get_personal_data(Username, Server) -> - LUser = jid:nodeprep(Username), - LServer = jid:nodeprep(Server), - US = {LUser, LServer}, - {atomic, Messages} = mnesia:transaction(fun() -> mnesia:wread({offline_msg, US}) end), - [{offline, ["timestamp", "from", "to", "packet"], offline_messages_to_gdpr_format(Messages)}]. - -offline_messages_to_gdpr_format(MsgList) -> - [offline_msg_to_gdpr_format(Msg) || Msg <- MsgList]. - -offline_msg_to_gdpr_format(#offline_msg{timestamp = Timestamp, from = From, to = To, packet = Packet}) -> - NowUniversal = calendar:now_to_universal_time(Timestamp), - {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), - UTC = list_to_binary(UTCTime ++ UTCDiff), - {UTC, jid:to_binary(jid:to_bare(From)), jid:to_binary(jid:to_bare(To)), exml:to_binary(Packet)}. diff --git a/src/mod_offline_rdbms.erl b/src/mod_offline_rdbms.erl index 2dc39a0e628..6f9bd5d03bd 100644 --- a/src/mod_offline_rdbms.erl +++ b/src/mod_offline_rdbms.erl @@ -28,6 +28,7 @@ -behaviour(mod_offline). -export([init/2, pop_messages/2, + fetch_messages/2, write_messages/3, count_offline_messages/3, remove_expired_messages/1, @@ -61,6 +62,20 @@ pop_messages(LUser, LServer) -> {error, Reason} end. +fetch_messages(LUser, LServer) -> + US = {LUser, LServer}, + To = jid:make(LUser, LServer, <<>>), + TimeStamp = p1_time_compat:timestamp(), + SUser = mongoose_rdbms:escape_string(LUser), + SServer = mongoose_rdbms:escape_string(LServer), + STimeStamp = encode_timestamp(TimeStamp), + case rdbms_queries:fetch_offline_messages(LServer, SUser, SServer, STimeStamp) of + {selected, Rows} -> + {ok, rows_to_records(US, To, Rows)}; + {error, Reason} -> + {error, Reason} + end. + rows_to_records(US, To, Rows) -> [row_to_record(US, To, Row) || Row <- Rows]. @@ -153,27 +168,3 @@ maybe_encode_timestamp(never) -> mongoose_rdbms:escape_null(); maybe_encode_timestamp(TimeStamp) -> encode_timestamp(TimeStamp). - -get_personal_data(Username, Server) -> - LUser = jid:nodeprep(Username), - LServer = jid:nodeprep(Server), - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - TimeStamp = p1_time_compat:timestamp(), - STimeStamp = encode_timestamp(TimeStamp), - ToJid = jid:to_binary({Username, LServer}), - {selected, Rows} = rdbms_queries:fetch_offline_messages(LServer, SUser, SServer, STimeStamp), - [{offline, ["timestamp", "from", "to", "packet"], rows_to_gdpr_format(ToJid, Rows)}]. - - -rows_to_gdpr_format(ToJid, Rows) -> - [row_to_gdpr_format(ToJid, Row) || Row <- Rows]. - -row_to_gdpr_format(ToJid, {STimeStamp, SFrom, SPacket}) -> - Timestamp = usec:to_now(mongoose_rdbms:result_to_integer(STimeStamp)), - NowUniversal = calendar:now_to_universal_time(Timestamp), - {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), - UTC = list_to_binary(UTCTime ++ UTCDiff), - [UTC, jid:to_binary(jid:binary_to_bare(SFrom)), ToJid, SPacket]. - - diff --git a/src/mod_offline_riak.erl b/src/mod_offline_riak.erl index e741e782293..75716670396 100644 --- a/src/mod_offline_riak.erl +++ b/src/mod_offline_riak.erl @@ -26,12 +26,12 @@ -export([init/2]). -export([pop_messages/2]). +-export([fetch_messages/2]). -export([write_messages/3]). -export([remove_expired_messages/1]). -export([remove_old_messages/2]). -export([remove_user/2]). -export([count_offline_messages/3]). --export([get_personal_data/2]). -include("mongoose.hrl"). -include("jlib.hrl"). @@ -186,30 +186,29 @@ maybe_decode_timestamp(TS) -> usec:to_now(TS). -get_personal_data(Username, Server) -> - LServer = jid:nodeprep(Server), - [{offline, ["timestamp", "from", "to", "packet"], fetch_messages(Username, LServer)}]. - -fetch_messages(Username, LServer) -> - LUser = jid:nodeprep(Username), +fetch_messages(LUser, LServer) -> Keys = read_user_idx(LUser, LServer), - ToJid = jid:to_binary({Username, LServer}), - Msgs = [fetch_msg(Key, LServer, ToJid) || Key <- Keys], + To = jid:make({LUser, LServer, <<>>}), + Msgs = [fetch_msg(Key, LUser, LServer, To) || Key <- Keys], lists:flatten(Msgs). -fetch_msg(Key, LServer, ToJid) -> +fetch_msg(Key, LUser, LServer, To) -> try {ok, Obj} = mongoose_riak:get(bucket_type(LServer), Key), PacketRaw = riakc_obj:get_value(Obj), + {ok, Packet} = exml:parse(PacketRaw), MD = riakc_obj:get_update_metadata(Obj), [Timestamp] = riakc_obj:get_secondary_index(MD, ?TIMESTAMP_IDX), From = riakc_obj:get_user_metadata_entry(MD, <<"from">>), + [Expire] = riakc_obj:get_secondary_index(MD, ?EXPIRE_IDX), - NowUniversal = calendar:now_to_universal_time(usec:to_now(Timestamp)), - {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), - UTC = list_to_binary(UTCTime ++ UTCDiff), - {UTC, jid:to_binary(jid:binary_to_bare(From)), ToJid, PacketRaw} + #offline_msg{us = {LUser, LServer}, + timestamp = usec:to_now(Timestamp), + expire = maybe_decode_timestamp(Expire), + from = jid:from_binary(From), + to = To, + packet = Packet} catch Error:Reason -> From 76920ade73e684e47c9de2fdf0f4245afb09ac70 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Wed, 8 May 2019 14:49:37 +0200 Subject: [PATCH 10/17] Retrieve full jid --- big_tests/tests/gdpr_SUITE.erl | 4 ++-- src/mod_offline.erl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 565911a4b0d..8f2c55a9b5e 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -218,9 +218,9 @@ retrieve_offline(Config) -> [AliceU, AliceS, 10]) end, 3), - BobJid = escalus_client:short_jid(Bob), + BobJid = escalus_client:full_jid(Bob), AliceJid = escalus_client:short_jid(Alice), - KateJid = escalus_client:short_jid(Kate), + KateJid = escalus_client:full_jid(Kate), ExpectedHeader = ["timestamp", "from", "to", "packet"], ExpectedItems = [ #{ "packet" => [{contains, Body1}], diff --git a/src/mod_offline.erl b/src/mod_offline.erl index 79aacd906cb..a571ccc4e9d 100644 --- a/src/mod_offline.erl +++ b/src/mod_offline.erl @@ -532,7 +532,7 @@ offline_msg_to_gdpr_format(#offline_msg{timestamp = Timestamp, from = From, to = NowUniversal = calendar:now_to_universal_time(Timestamp), {UTCTime, UTCDiff} = jlib:timestamp_to_iso(NowUniversal, utc), UTC = list_to_binary(UTCTime ++ UTCDiff), - {UTC, jid:to_binary(jid:to_bare(From)), jid:to_binary(jid:to_bare(To)), exml:to_binary(Packet)}. + {UTC, jid:to_binary(From), jid:to_binary(jid:to_bare(To)), exml:to_binary(Packet)}. skip_expired_messages(TimeStamp, Rs) -> [R || R <- Rs, not is_expired_message(TimeStamp, R)]. From 99bfd7b5fd63bfaf7d76fefa2894338cd9c2c7a5 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Wed, 8 May 2019 14:57:28 +0200 Subject: [PATCH 11/17] Shorten assertions in tests --- big_tests/tests/gdpr_SUITE.erl | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 8f2c55a9b5e..2c2ada9dbab 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -222,21 +222,17 @@ retrieve_offline(Config) -> AliceJid = escalus_client:short_jid(Alice), KateJid = escalus_client:full_jid(Kate), ExpectedHeader = ["timestamp", "from", "to", "packet"], - ExpectedItems = [ - #{ "packet" => [{contains, Body1}], - "from" => binary_to_list(BobJid), - "to" => binary_to_list(AliceJid), - "timestamp" => [{validate, fun validate_datetime/1}]}, - #{ "packet" => [{contains, Body2}], - "from" => binary_to_list(BobJid), - "to" => binary_to_list(AliceJid), - "timestamp" => [{validate, fun validate_datetime/1}]}, - #{ "packet" => [{contains, Body3}], - "from" => binary_to_list(KateJid), - "to" => binary_to_list(AliceJid), - "timestamp" => [{validate, fun validate_datetime/1}]} - ], + Expected = [{Body1, BobJid, AliceJid}, {Body2, BobJid, AliceJid}, {Body3, KateJid, AliceJid}], + + ExpectedItems = lists:map(fun({Body, From ,To}) -> + #{ "packet" => [{contains, Body}], + "from" => binary_to_list(From), + "to" => binary_to_list(To), + "timestamp" => [{validate, fun validate_datetime/1}]} + end, Expected), + maybe_stop_and_unload_module(mod_offline, mod_offline_backend, Config), + retrieve_and_validate_personal_data( Alice, Config, "offline", ExpectedHeader, ExpectedItems) end). @@ -486,11 +482,7 @@ validate_sorted_personal_maps([Map | RMaps], [Checks | RChecks]) -> maps:fold(fun(K, Conditions, _) -> validate_personal_item(maps:get(K, Map), Conditions) end, ok, Checks), -<<<<<<< 0b85d95a6467c76e9c3cc66562fc6ff8a5839525 -validate_sorted_personal_maps(RMaps, RChecks). -======= validate_sorted_personal_maps(RMaps, RChecks). ->>>>>>> Improve mod offline retrival test validate_personal_item(_Value, []) -> ok; From f53028097de5423552b0e48a1497412e39180a36 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Wed, 8 May 2019 15:00:39 +0200 Subject: [PATCH 12/17] Remove unimplemented function --- src/mod_offline_rdbms.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mod_offline_rdbms.erl b/src/mod_offline_rdbms.erl index 6f9bd5d03bd..6b9802c2332 100644 --- a/src/mod_offline_rdbms.erl +++ b/src/mod_offline_rdbms.erl @@ -35,8 +35,6 @@ remove_old_messages/2, remove_user/2]). --export([get_personal_data/2]). - -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_offline.hrl"). From f938cb64eaa15635d2a773246aac7761b94e00e8 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Wed, 8 May 2019 15:18:35 +0200 Subject: [PATCH 13/17] fix fetching offline for riak --- src/mod_offline_riak.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mod_offline_riak.erl b/src/mod_offline_riak.erl index 75716670396..216c3455d6a 100644 --- a/src/mod_offline_riak.erl +++ b/src/mod_offline_riak.erl @@ -189,8 +189,7 @@ maybe_decode_timestamp(TS) -> fetch_messages(LUser, LServer) -> Keys = read_user_idx(LUser, LServer), To = jid:make({LUser, LServer, <<>>}), - Msgs = [fetch_msg(Key, LUser, LServer, To) || Key <- Keys], - lists:flatten(Msgs). + {ok, [fetch_msg(Key, LUser, LServer, To) || Key <- Keys]}. fetch_msg(Key, LUser, LServer, To) -> try From 1bd8de2311febe0a6691452a000f6cdfa7a5699d Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Wed, 8 May 2019 15:28:18 +0200 Subject: [PATCH 14/17] fix upper/lower case of jid issue --- src/mod_offline.erl | 4 +--- src/mod_offline_mnesia.erl | 4 +++- src/mod_offline_rdbms.erl | 6 ++++-- src/mod_offline_riak.erl | 6 ++++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/mod_offline.erl b/src/mod_offline.erl index a571ccc4e9d..0c602991e3c 100644 --- a/src/mod_offline.erl +++ b/src/mod_offline.erl @@ -520,9 +520,7 @@ pop_messages(LUser, LServer) -> end. get_personal_data(Username, Server) -> - LUser = jid:nodeprep(Username), - LServer = jid:nodeprep(Server), - {ok, Messages} = mod_offline_backend:fetch_messages(LUser, LServer), + {ok, Messages} = mod_offline_backend:fetch_messages(Username, Server), [{offline, ["timestamp", "from", "to", "packet"], offline_messages_to_gdpr_format(Messages)}]. offline_messages_to_gdpr_format(MsgList) -> diff --git a/src/mod_offline_mnesia.erl b/src/mod_offline_mnesia.erl index cb3cf4e960d..4beac230964 100644 --- a/src/mod_offline_mnesia.erl +++ b/src/mod_offline_mnesia.erl @@ -66,7 +66,9 @@ pop_messages(LUser, LServer) -> {error, Reason} end. -fetch_messages(LUser, LServer) -> +fetch_messages(User, Server) -> + LUser = jid:nodeprep(User), + LServer = jid:nodeprep(Server), US = {LUser, LServer}, F = fun() -> mnesia:wread({offline_msg, US}) end, case mnesia:transaction(F) of diff --git a/src/mod_offline_rdbms.erl b/src/mod_offline_rdbms.erl index 6b9802c2332..afbd9638e78 100644 --- a/src/mod_offline_rdbms.erl +++ b/src/mod_offline_rdbms.erl @@ -60,9 +60,11 @@ pop_messages(LUser, LServer) -> {error, Reason} end. -fetch_messages(LUser, LServer) -> +fetch_messages(User, Server) -> + LUser = jid:nodeprep(User), + LServer = jid:nodeprep(Server), US = {LUser, LServer}, - To = jid:make(LUser, LServer, <<>>), + To = jid:make(User, LServer, <<>>), TimeStamp = p1_time_compat:timestamp(), SUser = mongoose_rdbms:escape_string(LUser), SServer = mongoose_rdbms:escape_string(LServer), diff --git a/src/mod_offline_riak.erl b/src/mod_offline_riak.erl index 216c3455d6a..edb47af93c6 100644 --- a/src/mod_offline_riak.erl +++ b/src/mod_offline_riak.erl @@ -186,9 +186,11 @@ maybe_decode_timestamp(TS) -> usec:to_now(TS). -fetch_messages(LUser, LServer) -> +fetch_messages(User, Server) -> + LUser = jid:nodeprep(User), + LServer = jid:nodeprep(Server), Keys = read_user_idx(LUser, LServer), - To = jid:make({LUser, LServer, <<>>}), + To = jid:make({User, LServer, <<>>}), {ok, [fetch_msg(Key, LUser, LServer, To) || Key <- Keys]}. fetch_msg(Key, LUser, LServer, To) -> From 962d936ca60133630515384798027f5e5104ca24 Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Fri, 10 May 2019 11:00:47 +0200 Subject: [PATCH 15/17] Enable mod_offline retrieve tests --- big_tests/tests/gdpr_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 2c2ada9dbab..56d8ee11bbf 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -56,7 +56,7 @@ groups() -> retrieve_vcard, %retrieve_roster, %retrieve_mam, - %retrieve_offline, + retrieve_offline, %retrieve_private_xml, %retrieve_inbox, retrieve_logs, From a4cd8bae12056c7d760e4847c0be3e298205161e Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Fri, 10 May 2019 11:00:59 +0200 Subject: [PATCH 16/17] merge reults from disabled backends as well --- src/mod_offline.erl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/mod_offline.erl b/src/mod_offline.erl index 0c602991e3c..2b0d82953e4 100644 --- a/src/mod_offline.erl +++ b/src/mod_offline.erl @@ -520,8 +520,17 @@ pop_messages(LUser, LServer) -> end. get_personal_data(Username, Server) -> - {ok, Messages} = mod_offline_backend:fetch_messages(Username, Server), - [{offline, ["timestamp", "from", "to", "packet"], offline_messages_to_gdpr_format(Messages)}]. + AllMessages = lists:flatmap(fun(B) -> + try B:fetch_messages(Username, Server) of + {ok, Messages} -> + Messages; + _ -> [] + catch + _:_ -> + [] + end + end, mongoose_lib:find_behaviour_implementations(mod_offline)), + [{offline, ["timestamp", "from", "to", "packet"], offline_messages_to_gdpr_format(AllMessages)}]. offline_messages_to_gdpr_format(MsgList) -> [offline_msg_to_gdpr_format(Msg) || Msg <- MsgList]. From ea799deb6a721285fc84d46cca0b2a4dd50a52bc Mon Sep 17 00:00:00 2001 From: Ludwik Bukowski Date: Fri, 10 May 2019 13:40:18 +0200 Subject: [PATCH 17/17] test offline retrieve with unloading module --- big_tests/tests/gdpr_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 56d8ee11bbf..e9ecb6a62a6 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -71,6 +71,7 @@ groups() -> ]}, {retrieve_personal_data_with_mods_disabled, [], [ retrieve_vcard, + retrieve_offline, retrieve_logs, retrieve_all_pubsub_data ]},