Merge branch 'master' into feature/instrument

big_tests/rebar.config

@@ -16,11 +16,11 @@
         {proper, "1.4.0"},
         {gun, "2.1.0"},
         {fusco, "0.1.1"},
-        {escalus, {git, "git@github.com:esl/escalus.git", {branch, "ws-stream-management"}}},
-%        {escalus, "4.2.12"},
+        {escalus, "4.2.15"},
         {cowboy, "2.12.0"},
         {csv, "3.0.3", {pkg, csve}},
-        {amqp_client, "3.12.14"},
+        {amqp_client, "3.13.0-rc.2"},
+        {rabbit_common, "3.13.4"},
         {esip, "1.0.52"},
         {jid, "2.1.0", {pkg, mongoose_jid}}
 ]}.

big_tests/rebar.lock

@@ -1,5 +1,5 @@
 {"1.2.0",
-[{<<"amqp_client">>,{pkg,<<"amqp_client">>,<<"3.12.14">>},0},
+[{<<"amqp_client">>,{pkg,<<"amqp_client">>,<<"3.13.0-rc.2">>},0},
  {<<"base16">>,{pkg,<<"base16">>,<<"2.0.1">>},0},
  {<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.12.2">>},0},
  {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.12.0">>},0},
@@ -8,15 +8,12 @@
   {pkg,<<"credentials_obfuscation">>,<<"3.4.0">>},
   1},
  {<<"csv">>,{pkg,<<"csve">>,<<"3.0.3">>},0},
- {<<"escalus">>,
-  {git,"git@github.com:esl/escalus.git",
-       {ref,"f99d6efe1028f645d8a17c9ab3fb8e4adab69a76"}},
-  0},
+ {<<"escalus">>,{pkg,<<"escalus">>,<<"4.2.15">>},0},
  {<<"esip">>,{pkg,<<"esip">>,<<"1.0.52">>},0},
  {<<"exml">>,{pkg,<<"hexml">>,<<"3.4.1">>},0},
  {<<"fast_pbkdf2">>,{pkg,<<"fast_pbkdf2">>,<<"1.0.5">>},2},
  {<<"fast_scram">>,{pkg,<<"fast_scram">>,<<"0.6.0">>},1},
- {<<"fast_tls">>,{pkg,<<"fast_tls">>,<<"1.1.19">>},1},
+ {<<"fast_tls">>,{pkg,<<"fast_tls">>,<<"1.1.21">>},0},
  {<<"fusco">>,{pkg,<<"fusco">>,<<"0.1.1">>},0},
  {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1},
  {<<"gun">>,{pkg,<<"gun">>,<<"2.1.0">>},0},
@@ -27,28 +24,29 @@
  {<<"p1_utils">>,{pkg,<<"p1_utils">>,<<"1.0.25">>},1},
  {<<"proper">>,{pkg,<<"proper">>,<<"1.4.0">>},0},
  {<<"quickrand">>,{pkg,<<"quickrand">>,<<"2.0.7">>},2},
- {<<"rabbit_common">>,{pkg,<<"rabbit_common">>,<<"3.12.14">>},1},
+ {<<"rabbit_common">>,{pkg,<<"rabbit_common">>,<<"3.13.4">>},0},
  {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},1},
- {<<"recon">>,{pkg,<<"recon">>,<<"2.5.3">>},2},
+ {<<"recon">>,{pkg,<<"recon">>,<<"2.5.3">>},1},
  {<<"stringprep">>,{pkg,<<"stringprep">>,<<"1.0.29">>},1},
  {<<"stun">>,{pkg,<<"stun">>,<<"1.2.12">>},1},
- {<<"thoas">>,{pkg,<<"thoas">>,<<"1.0.0">>},2},
+ {<<"thoas">>,{pkg,<<"thoas">>,<<"1.0.0">>},1},
  {<<"uuid">>,{pkg,<<"uuid_erl">>,<<"2.0.7">>},1},
- {<<"worker_pool">>,{pkg,<<"worker_pool">>,<<"6.2.0">>},1}]}.
+ {<<"worker_pool">>,{pkg,<<"worker_pool">>,<<"6.2.1">>},1}]}.
 [
 {pkg_hash,[
- {<<"amqp_client">>, <<"2B677BC3F2E2234BA7517042B25D72071A79735042E91F9116BD3C176854B622">>},
+ {<<"amqp_client">>, <<"2C9975DA52319D8E613B5DBD3E931374B41AF6F571BD7A3F96CE50AD6C27441B">>},
  {<<"base16">>, <<"F0549F732E03BE8124ED0D19FD5EE52146CC8BE24C48CBC3F23AB44B157F11A2">>},
  {<<"bbmustache">>, <<"0CABDCE0DB9FE6D3318131174B9F2B351328A4C0AFBEB3E6E99BB0E02E9B621D">>},
  {<<"cowboy">>, <<"F276D521A1FF88B2B9B4C54D0E753DA6C66DD7BE6C9FCA3D9418B561828A3731">>},
  {<<"cowlib">>, <<"DB8F7505D8332D98EF50A3EF34B34C1AFDDEC7506E4EE4DD4A3A266285D282CA">>},
  {<<"credentials_obfuscation">>, <<"34E18B126B3AEFD6E8143776FBE1CECEEA6792307C99AC5EE8687911F048CFD7">>},
  {<<"csv">>, <<"69E7D9B3FDC72016644368762C6A3E6CBFEB85BCCADBF1BD99AB6C827E360E04">>},
+ {<<"escalus">>, <<"1F153CFBEAA67B012F7A34D0AD61A8CD68ED7C0BF9F02B94606B339FBCDCD617">>},
  {<<"esip">>, <<"A2840287C493A4280E6FBA57A257706843B025C315875E38B03FD07190E22DBA">>},
  {<<"exml">>, <<"9581FE6512D9772C61BBE611CD4A8E5BB90B4D4481275325EC520F7A931A9393">>},
  {<<"fast_pbkdf2">>, <<"6045138C4C209FC8222A0B18B2CB1D7BD7407EF4ADAD0F14C5E0F7F4726E3E41">>},
  {<<"fast_scram">>, <<"70724F584A118DA147A51EE38DEE56203F217D58AD61E0BB2C2EF834C16B35B8">>},
- {<<"fast_tls">>, <<"F52731A4B35259FA06CF23E2A0732920AD9EFCE7C3D68377F129A474998747BB">>},
+ {<<"fast_tls">>, <<"65D7D547A09EEFB37A1C0D04D8601FAC4F3E6E2C1EDE859A7787081670F9648D">>},
  {<<"fusco">>, <<"3DD6A90151DFEF30EA1937CC44E9A59177C0094918388D9BCAA2F2DC5E2AE4AA">>},
  {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>},
  {<<"gun">>, <<"B4E4CBBF3026D21981C447E9E7CA856766046EFF693720BA43114D7F5DE36E87">>},
@@ -59,27 +57,28 @@
  {<<"p1_utils">>, <<"2D39B5015A567BBD2CC7033EEB93A7C60D8C84EFE1EF69A3473FAA07FA268187">>},
  {<<"proper">>, <<"89A44B8C39D28BB9B4BE8E4D715D534905B325470F2E0EC5E004D12484A79434">>},
  {<<"quickrand">>, <<"D2BD76676A446E6A058D678444B7FDA1387B813710D1AF6D6E29BB92186C8820">>},
- {<<"rabbit_common">>, <<"466123EE7346A3CDAC078C0C302BCD36DA4523E8ACD678C1B992F7B4DF1F7914">>},
+ {<<"rabbit_common">>, <<"68650FBCB8CE204CE28C116451C030E294BF916ED722B43A369D7586DA2A5605">>},
  {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>},
  {<<"recon">>, <<"739107B9050EA683C30E96DE050BC59248FD27EC147696F79A8797FF9FA17153">>},
  {<<"stringprep">>, <<"02F23E8C3A219A3DFE40A22E908BECE3A2F68AF0FF599EA8A7B714ECB21E62EE">>},
  {<<"stun">>, <<"A65DF67A8AAAECB6A94D687977B2E9F161820819910CB97BBE26A3525356525B">>},
  {<<"thoas">>, <<"567C03902920827A18A89F05B79A37B5BF93553154B883E0131801600CF02CE0">>},
  {<<"uuid">>, <<"B2078D2CC814F53AFA52D36C91E08962C7E7373585C623F4C0EA6DFB04B2AF94">>},
- {<<"worker_pool">>, <<"506DE38C528A81ED2C6A80A419B83DDE6DA5E295BD320BDF4D35A69AFEB0247A">>}]},
+ {<<"worker_pool">>, <<"BD98A0BE1D20057AE9967CBE73D263AEA5BE14BBE4C73CAFEB1378572FF14561">>}]},
 {pkg_hash_ext,[
- {<<"amqp_client">>, <<"5F70B6C3B1A739790080DA4FDDC94A867E99F033C4B1EDC20D6FF8B8FB4BD160">>},
+ {<<"amqp_client">>, <<"E86FD4B95291799BDE1571AF81077B5A5DB9653824FBE429F5BADAD831C52642">>},
  {<<"base16">>, <<"06EA2D48343282E712160BA89F692B471DB8B36ABE8394F3445FF9032251D772">>},
  {<<"bbmustache">>, <<"688B33A4D5CC2D51F575ADF0B3683FC40A38314A2F150906EDCFC77F5B577B3B">>},
  {<<"cowboy">>, <<"8A7ABE6D183372CEB21CAA2709BEC928AB2B72E18A3911AA1771639BEF82651E">>},
  {<<"cowlib">>, <<"E1E1284DC3FC030A64B1AD0D8382AE7E99DA46C3246B815318A4B848873800A4">>},
  {<<"credentials_obfuscation">>, <<"738ACE0ED5545D2710D3F7383906FC6F6B582D019036E5269C4DBD85DBCED566">>},
  {<<"csv">>, <<"741D1A55AABADAA3E0FE13051050101A73E90C4570B9F9403A939D9546813521">>},
+ {<<"escalus">>, <<"048CEB922E64F23DD05890A60256EDC28B86969E48B0BE5FDE202F62315C9E4F">>},
  {<<"esip">>, <<"6F00165395900500AA262CE0297162D93931C78C1464D89FD0EDC6E3D6BC011F">>},
  {<<"exml">>, <<"D8E7894E2544402B4986EEB2443C15B51B14F686266F091DBF2777D1D99A2FA2">>},
  {<<"fast_pbkdf2">>, <<"BC3B5A3CAB47AD114FF8BB815FEDE62A6187ACD14D8B37412F2AF8236A089CEF">>},
  {<<"fast_scram">>, <<"771D034341599CFC6A6C5E56CF924B68D2C7478088CAF17419E3147B66914667">>},
- {<<"fast_tls">>, <<"DB34322C8782D4C5139CCB80709D8EC8C38089B44262EDD0C2F660AC495BD389">>},
+ {<<"fast_tls">>, <<"131542913937025E48CD80AA81F00359686D5501B75621E72026A87B5229505B">>},
  {<<"fusco">>, <<"6343551BD1E824F2A6CA85E1158C5B37C320FD449FBFEC7450A73F192AAF9022">>},
  {<<"goldrush">>, <<"99CB4128CFFCB3227581E5D4D803D5413FA643F4EB96523F77D9E6937D994CEB">>},
  {<<"gun">>, <<"52FC7FC246BFC3B00E01AEA1C2854C70A366348574AB50C57DFE796D24A0101D">>},
@@ -90,12 +89,12 @@
  {<<"p1_utils">>, <<"9219214428F2C6E5D3187FF8EB9A8783695C2427420BE9A259840E07ADA32847">>},
  {<<"proper">>, <<"18285842185BD33EFBDA97D134A5CB5A0884384DB36119FEE0E3CFA488568CBB">>},
  {<<"quickrand">>, <<"B8ACBF89A224BC217C3070CA8BEBC6EB236DBE7F9767993B274084EA044D35F0">>},
- {<<"rabbit_common">>, <<"70C31A51F7401CC0204DDEF2745D98680C2E0DF67E3B0C9E198916881FDE3293">>},
+ {<<"rabbit_common">>, <<"C0E8F2909EAECABC732990D26B75DE092E265D14E8AC9EAFD4813438F1CC2233">>},
  {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>},
  {<<"recon">>, <<"6C6683F46FD4A1DFD98404B9F78DCABC7FCD8826613A89DCB984727A8C3099D7">>},
  {<<"stringprep">>, <<"928EBA304C3006EB1512110EBD7B87DB163B00859A09375A1E4466152C6C462A">>},
  {<<"stun">>, <<"A2055032B6D338D0454142004BCB12FAFB0C64AB1F273F1D0C6923EBBC8EDE40">>},
  {<<"thoas">>, <<"FC763185B932ECB32A554FB735EE03C3B6B1B31366077A2427D2A97F3BD26735">>},
  {<<"uuid">>, <<"4E4C5CA3461DC47C5E157ED42AA3981A053B7A186792AF972A27B14A9489324E">>},
- {<<"worker_pool">>, <<"E1DED160797FBE656AD683109DBD741B520097DF17BDF7B51E3E4697073A6E62">>}]}
+ {<<"worker_pool">>, <<"64E560DE08CA5E7DB8BD4CDCC7B744B0659696194E3BC9E56239BA4A0F7E24F9">>}]}
 ].

big_tests/run_common_test.erl

@@ -359,21 +359,22 @@ analyze_coverage(_, _) ->
 
 prepare(Props) ->
     Nodes = get_mongoose_nodes(Props),
-    maybe_compile_cover(Nodes).
+    maybe_compile_cover(Nodes),
+    block_nodes(Props).
 
 maybe_compile_cover([]) ->
     io:format("cover: skip cover compilation~n", []),
     ok;
-maybe_compile_cover(Nodes) ->
+maybe_compile_cover([CoverNode|_] = Nodes) ->
     io:format("cover: compiling modules for nodes ~p~n", [Nodes]),
     import_code_paths(hd(Nodes)),
 
-    cover:start(Nodes),
+    cover_start(CoverNode, Nodes),
     Dir = call(hd(Nodes), code, lib_dir, [mongooseim, ebin]),
 
     %% Time is in microseconds
     {Time, Compiled} = timer:tc(fun() ->
-                            Results = cover:compile_beam_directory(Dir),
+                            Results = cover_compile_dir(CoverNode, Dir),
                             Ok = [X || X = {ok, _} <- Results],
                             NotOk = Results -- Ok,
                             #{ok => length(Ok), failed => NotOk}
@@ -385,26 +386,26 @@ maybe_compile_cover(Nodes) ->
     ok.
 
 analyze(Props, CoverOpts) ->
+    unblock_nodes(Props),
     io:format("Coverage analyzing~n"),
     Nodes = get_mongoose_nodes(Props),
     analyze(Props, CoverOpts, Nodes).
 
 analyze(_Props, _CoverOpts, []) ->
     ok;
-analyze(_Props, CoverOpts, Nodes) ->
-    deduplicate_cover_server_console_prints(),
+analyze(_Props, CoverOpts, [CoverNode|_] = Nodes) ->
     %% Import small tests cover
     Files = filelib:wildcard(repo_dir() ++ "/_build/**/cover/*.coverdata"),
     io:format("Files: ~p", [Files]),
     report_time("Import cover data into run_common_test node", fun() ->
-            [cover:import(File) || File <- Files]
+            [cover_import(CoverNode, File) || File <- Files]
         end),
     report_time("Export merged cover data", fun() ->
-			cover:export("/tmp/mongoose_combined.coverdata")
+                        cover_export(CoverNode, "/tmp/mongoose_combined.coverdata")
 		end),
     case os:getenv("GITHUB_RUN_ID") of
         false ->
-            make_html(modules_to_analyze(CoverOpts));
+            make_html(CoverNode, modules_to_analyze(CoverNode, CoverOpts));
         _ ->
             ok
     end,
@@ -414,11 +415,11 @@ analyze(_Props, CoverOpts, Nodes) ->
             ok;
         _ ->
             report_time("Stopping cover on MongooseIM nodes", fun() ->
-                        cover:stop([node()|Nodes])
+                        cover_stop(CoverNode, Nodes)
                     end)
     end.
 
-make_html(Modules) ->
+make_html(CoverNode, Modules) ->
     {ok, Root} = file:get_cwd(),
     SortScript = Root ++ "/priv/sorttable.js",
     os:cmd("cp " ++ SortScript ++ " " ++ ?CT_REPORT),
@@ -437,11 +438,11 @@ make_html(Modules) ->
                   FileName = lists:flatten(io_lib:format("~s.COVER.html",[Module])),
 
                   %% We assume that import_code_paths/1 was called earlier
-                  case cover:analyse(Module, module) of
+                  case cover_analyse(CoverNode, Module) of
                       {ok, {Module, {C, NC}}} ->
                           file:write(File, row(atom_to_list(Module), C, NC, percent(C,NC),"coverage/"++FileName)),
                           FilePathC = filename:join([CoverageDir, FileName]),
-                          catch cover:analyse_to_file(Module, FilePathC, [html]),
+                          cover_analyse_to_html_file(CoverNode, Module, FilePathC),
                           {CAcc + C, NCAcc + NC};
                       Reason ->
                           error_logger:error_msg("issue=cover_analyse_failed module=~p reason=~p",
@@ -503,9 +504,9 @@ module_list(undefined) ->
 module_list(ModuleList) ->
     [ list_to_atom(L) || L <- string:tokens(ModuleList, ", ") ].
 
-modules_to_analyze(true) ->
-    lists:usort(cover:imported_modules() ++ cover:modules());
-modules_to_analyze(ModuleList) when is_list(ModuleList) ->
+modules_to_analyze(CoverNode, true) ->
+    lists:usort(cover_all_modules(CoverNode));
+modules_to_analyze(_CoverNode, ModuleList) when is_list(ModuleList) ->
     ModuleList.
 
 add({X1, X2, X3, X4},
@@ -581,6 +582,9 @@ host_param(Name, {_, Params}) ->
     {Name, Param} = lists:keyfind(Name, 1, Params),
     Param.
 
+host_param(Name, {_, Params}, Default) ->
+    proplists:get_value(Name, Params, Default).
+
 report_time(Description, Fun) ->
 	report_progress("~nExecuting ~ts~n", [Description]),
 	Start = os:timestamp(),
@@ -630,16 +634,6 @@ handle_file_error(_FileName, Other) ->
 
 %% ------------------------------------------------------------------
 
-%% cover_server process is using io:format too much.
-%% This code removes duplicate io:formats.
-%%
-%% Example of a message we want to write only once:
-%% "Analysis includes data from imported files" from cover.erl in Erlang/R19
-deduplicate_cover_server_console_prints() ->
-    %% Set a new group leader for cover_server
-    CoverPid = whereis(cover_server),
-    dedup_proxy_group_leader:start_proxy_group_leader_for(CoverPid).
-
 ct_run_dirs() ->
     filelib:wildcard("ct_report/ct_run*").
 
@@ -721,3 +715,91 @@ assert_preset_present(Preset, PresetConfs) ->
             error_logger:error_msg("Preset not found ~p~n", [Preset]),
             error({preset_not_found, Preset})
     end.
+
+assert_list(X) when is_list(X) -> X.
+
+%% We use mim1 as a main node.
+%% Only the main node supports meck
+%% (other nodes should not use meck for the cover compiled modules).
+cover_start(CoverNode, Nodes) ->
+    {ok, _} = cover_call(CoverNode, start, [Nodes]),
+    CoverNode = cover_call(CoverNode, get_main_node, []),
+    ok.
+
+cover_stop(CoverNode, Nodes) ->
+    cover_call(CoverNode, stop, [Nodes]).
+
+cover_all_modules(CoverNode) ->
+    List1 = assert_list(cover_call(CoverNode, imported_modules, [])),
+    List2 = assert_list(cover_call(CoverNode, modules, [])),
+    List1 ++ List2.
+
+cover_analyse_to_html_file(CoverNode, Module, FilePathC) ->
+    catch cover_call(CoverNode, analyse_to_file, [Module, FilePathC, [html]]).
+
+cover_analyse(CoverNode, Module) ->
+    cover_call(CoverNode, analyse, [Module, module]).
+
+cover_export(CoverNode, ToFile) ->
+    cover_call(CoverNode, export, [ToFile]).
+
+cover_import(CoverNode, FromFile) ->
+    cover_call(CoverNode, import, [FromFile]).
+
+cover_compile_dir(CoverNode, Dir) ->
+    cover_call(CoverNode, compile_beam_directory, [Dir]).
+
+cover_call(CoverNode, Fun, Args) ->
+    rpc:call(CoverNode, cover, Fun, Args).
+
+block_nodes(Props) ->
+   [block_node(Node, BlockNode, Props) || {Node, BlockNode} <- block_nodes_specs(Props)],
+   ok.
+
+unblock_nodes(Props) ->
+   [unblock_node(Node, BlockNode, Props) || {Node, BlockNode} <- block_nodes_specs(Props)],
+   ok.
+
+%% Reads `blocks_hosts' parameter for the host from `test.config'.
+%% Returns a list of blocks to do like `[{mim, fed}]'.
+block_nodes_specs(Props) ->
+    EnabledHosts = [ H || H <- get_all_hosts(Props), is_test_host_enabled(host_name(H)) ],
+    [{host_name(H), BlockName}
+     || H <- EnabledHosts, BlockName <- host_param(blocks_hosts, H, []),
+        is_test_host_enabled(BlockName)].
+
+host_name_to_node(Name, Props) ->
+    Hosts = get_all_hosts(Props),
+    Host = proplists:get_value(Name, Hosts, []),
+    case proplists:get_value(node, Host) of
+        undefined ->
+            error({host_name_to_node_failed, Name, Props});
+        Node ->
+            Node
+    end.
+
+%% Do not allow node Name to talk to node BlockName
+block_node(Name, BlockName, Props) ->
+    Node = host_name_to_node(Name, Props),
+    BlockNode = host_name_to_node(BlockName, Props),
+    rpc_call(Node, erlang, set_cookie, [BlockNode, make_bad_cookie(Name, BlockNode)]),
+    rpc_call(Node, erlang, disconnect_node, [BlockNode]),
+    Cond = fun() -> lists:member(BlockNode, rpc_call(Node, erlang, nodes, [])) end,
+    mongoose_helper:wait_until(Cond, false).
+
+unblock_node(Name, BlockName, Props) ->
+    Node = host_name_to_node(Name, Props),
+    BlockNode = host_name_to_node(BlockName, Props),
+    DefCookie = rpc_call(Node, erlang, get_cookie, []),
+    rpc_call(Node, erlang, set_cookie, [BlockNode, DefCookie]).
+
+make_bad_cookie(Name, BlockName) ->
+    list_to_atom(atom_to_list(Name) ++ "_blocks_" ++ atom_to_list(BlockName)).
+
+rpc_call(Node, M, F, Args) ->
+    case rpc:call(Node, M, F, Args) of
+        {badrpc, Reason} ->
+            error({rpc_call_failed, Reason, Node, {M, F, Args}});
+        Res ->
+            Res
+    end.