Dist blocker

rebar.lock

@@ -8,7 +8,7 @@
  {<<"certifi">>,{pkg,<<"certifi">>,<<"2.9.0">>},1},
  {<<"cets">>,
   {git,"https://github.com/esl/cets.git",
-       {ref,"0321fa355d6bfb0f28ff244e34b22deec707dbee"}},
+       {ref,"0f69ec9eaed6e9567337b845661045d2ce04ad20"}},
   0},
  {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},0},
  {<<"cowboy_swagger">>,{pkg,<<"cowboy_swagger">>,<<"2.5.1">>},0},

src/ejabberd_sup.erl

@@ -38,6 +38,7 @@ start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->
+    DistBlocker = worker_spec(cets_dist_blocker),
     Hooks = worker_spec(gen_hook),
     Cleaner = worker_spec(mongoose_cleaner),
     Router = worker_spec(ejabberd_router),
@@ -64,7 +65,8 @@ init([]) ->
     IQSupervisor =
         ejabberd_tmp_sup_spec(ejabberd_iq_sup, [ejabberd_iq_sup, mongoose_iq_worker]),
     {ok, {{one_for_one, 10, 1},
-          [StartIdServer,
+          [DistBlocker,
+           StartIdServer,
            PG,
            Hooks,
            Cleaner,

src/mongoose_cleaner.erl

@@ -34,6 +34,7 @@
 %%%===================================================================
 
 init([]) ->
+    cets_dist_blocker:add_cleaner(self()),
     case net_kernel:monitor_nodes(true) of
         ok ->
             {ok, #state{}};
@@ -56,6 +57,7 @@
                    text => <<"mongoose_cleaner received nodenown event">>,
                    down_node => Node}),
     cleanup_modules(Node),
+    cets_dist_blocker:cleaning_done(self(), Node),
     {noreply, State};
 handle_info(_Info, State) ->
     {noreply, State}.
@@ -75,15 +77,15 @@
     LockRequest = {LockKey, self()},
     C = fun () -> run_node_cleanup(Node) end,
     Nodes = [node() | nodes()],
-    Retries = 1,
+    Retries = 10,
     case global:trans(LockRequest, C, Nodes, Retries) of
         aborted ->
-            ?LOG_INFO(#{what => cleaner_trans_aborted,
-                        text => <<"mongoose_cleaner failed to get global lock">>,
-                        lock_key => LockKey}),
-            {ok, aborted};
+            ?LOG_ERROR(#{what => cleaner_trans_aborted,
+                         text => <<"mongoose_cleaner failed to get the global lock, run cleanup anyway">>,
+                         remote_node => Node, lock_key => LockKey, retries => Retries}),
+            C();
         Result ->
-            {ok, Result}
+            Result
     end.
 
 run_node_cleanup(Node) ->

test/mongoose_cleanup_SUITE.erl

@@ -124,10 +124,10 @@ opts() ->
       {auth, ?HOST} => config_parser_helper:extra_auth(),
       {modules, ?HOST} => #{}}.
 
-meck_mods(bosh) -> [exometer, mod_bosh_socket];
-meck_mods(s2s) -> [exometer, mongoose_bin];
-meck_mods(component) -> [exometer];
-meck_mods(_) -> [exometer, ejabberd_sm, ejabberd_local].
+meck_mods(bosh) -> [exometer, mod_bosh_socket, cets_dist_blocker];
+meck_mods(s2s) -> [exometer, mongoose_bin, cets_dist_blocker];
+meck_mods(component) -> [exometer, cets_dist_blocker];
+meck_mods(_) -> [exometer, ejabberd_sm, ejabberd_local, cets_dist_blocker].
 
 %% -----------------------------------------------------
 %% Tests
@@ -306,6 +306,11 @@ setup_meck([mod_bosh_socket | R]) ->
     meck:new(mod_bosh_socket, [passthrough]),
     meck:expect(mod_bosh_socket, start_supervisor, fun() -> {ok, self()} end),
     setup_meck(R);
+setup_meck([cets_dist_blocker | R]) ->
+    meck:new(cets_dist_blocker, [passthrough]),
+    meck:expect(cets_dist_blocker, add_cleaner, fun(_Pid) -> ok end),
+    meck:expect(cets_dist_blocker, cleaning_done, fun(_Pid, _Node) -> ok end),
+    setup_meck(R);
 setup_meck([]) ->
     ok.