diff --git a/deps/rabbitmq_management/src/rabbit_mgmt_dispatcher.erl b/deps/rabbitmq_management/src/rabbit_mgmt_dispatcher.erl index 891963148a19..d54567320e97 100644 --- a/deps/rabbitmq_management/src/rabbit_mgmt_dispatcher.erl +++ b/deps/rabbitmq_management/src/rabbit_mgmt_dispatcher.erl @@ -202,6 +202,10 @@ dispatcher() -> {"/health/checks/port-listener/:port", rabbit_mgmt_wm_health_check_port_listener, []}, {"/health/checks/protocol-listener/:protocol", rabbit_mgmt_wm_health_check_protocol_listener, []}, {"/health/checks/virtual-hosts", rabbit_mgmt_wm_health_check_virtual_hosts, []}, + {"/health/checks/quorum-queues-without-elected-leaders/all-vhosts/", rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts, []}, + {"/health/checks/quorum-queues-without-elected-leaders/vhost/:vhost/", rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders, []}, + {"/health/checks/quorum-queues-without-elected-leaders/all-vhosts/pattern/:pattern", rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts, []}, + {"/health/checks/quorum-queues-without-elected-leaders/vhost/:vhost/pattern/:pattern", rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders, []}, {"/health/checks/node-is-quorum-critical", rabbit_mgmt_wm_health_check_node_is_quorum_critical, []}, {"/reset", rabbit_mgmt_wm_reset, []}, {"/reset/:node", rabbit_mgmt_wm_reset, []}, diff --git a/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders.erl b/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders.erl new file mode 100644 index 000000000000..950351f4ca6c --- /dev/null +++ b/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders.erl @@ -0,0 +1,68 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2025 Broadcom. All Rights Reserved. The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. All rights reserved. +%% + +%% An HTTP API counterpart of 'rabbitmq-diagnostics check_for_quorum_queues_without_an_elected_leader' +-module(rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders). + +-export([init/2, to_json/2, content_types_provided/2, is_authorized/2]). +-export([resource_exists/2]). +-export([variances/2]). + +-include_lib("rabbitmq_management_agent/include/rabbit_mgmt_records.hrl"). + +-define(DEFAULT_PATTERN, <<".*">>). + +%%-------------------------------------------------------------------- + +init(Req, _State) -> + {cowboy_rest, rabbit_mgmt_headers:set_common_permission_headers(Req, ?MODULE), #context{}}. + +variances(Req, Context) -> + {[<<"accept-encoding">>, <<"origin">>], Req, Context}. + +content_types_provided(ReqData, Context) -> + {rabbit_mgmt_util:responder_map(to_json), ReqData, Context}. + +resource_exists(ReqData, Context) -> + Result = case {vhost(ReqData), pattern(ReqData)} of + {none, _} -> false; + {_, none} -> false; + _ -> true + end, + {Result, ReqData, Context}. + +to_json(ReqData, Context) -> + case rabbit_quorum_queue:leader_health_check(pattern(ReqData), vhost(ReqData)) of + [] -> + rabbit_mgmt_util:reply(#{status => ok}, ReqData, Context); + Qs when length(Qs) > 0 -> + Msg = <<"Detected quorum queues without an elected leader">>, + failure(Msg, Qs, ReqData, Context) + end. + +failure(Message, Qs, ReqData, Context) -> + Body = #{status => failed, + reason => Message, + queues => Qs}, + {Response, ReqData1, Context1} = rabbit_mgmt_util:reply(Body, ReqData, Context), + {stop, cowboy_req:reply(503, #{}, Response, ReqData1), Context1}. + +is_authorized(ReqData, Context) -> + rabbit_mgmt_util:is_authorized(ReqData, Context). + +%% +%% Implementation +%% + +vhost(ReqData) -> + rabbit_mgmt_util:id(vhost, ReqData). + +pattern(ReqData) -> + case rabbit_mgmt_util:id(pattern, ReqData) of + none -> ?DEFAULT_PATTERN; + Other -> Other + end. diff --git a/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts.erl b/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts.erl new file mode 100644 index 000000000000..f56beb677c6d --- /dev/null +++ b/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts.erl @@ -0,0 +1,61 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2025 Broadcom. All Rights Reserved. The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. All rights reserved. +%% + +%% An HTTP API counterpart of 'rabbitmq-diagnostics check_for_quorum_queues_without_an_elected_leader --across-all-vhosts' +-module(rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts). + +-export([init/2, to_json/2, content_types_provided/2, is_authorized/2]). +-export([resource_exists/2]). +-export([variances/2]). + +-include_lib("rabbitmq_management_agent/include/rabbit_mgmt_records.hrl"). + +-define(ACROSS_ALL_VHOSTS, across_all_vhosts). +-define(DEFAULT_PATTERN, <<".*">>). + +%%-------------------------------------------------------------------- + +init(Req, _State) -> + {cowboy_rest, rabbit_mgmt_headers:set_common_permission_headers(Req, ?MODULE), #context{}}. + +variances(Req, Context) -> + {[<<"accept-encoding">>, <<"origin">>], Req, Context}. + +content_types_provided(ReqData, Context) -> + {rabbit_mgmt_util:responder_map(to_json), ReqData, Context}. + +resource_exists(ReqData, Context) -> + {true, ReqData, Context}. + +to_json(ReqData, Context) -> + case rabbit_quorum_queue:leader_health_check(pattern(ReqData), ?ACROSS_ALL_VHOSTS) of + [] -> + rabbit_mgmt_util:reply(#{status => ok}, ReqData, Context); + Qs when length(Qs) > 0 -> + Msg = <<"Detected quorum queues without an elected leader">>, + failure(Msg, Qs, ReqData, Context) + end. + +failure(Message, Qs, ReqData, Context) -> + Body = #{status => failed, + reason => Message, + queues => Qs}, + {Response, ReqData1, Context1} = rabbit_mgmt_util:reply(Body, ReqData, Context), + {stop, cowboy_req:reply(503, #{}, Response, ReqData1), Context1}. + +is_authorized(ReqData, Context) -> + rabbit_mgmt_util:is_authorized(ReqData, Context). + +%% +%% Implementation +%% + +pattern(ReqData) -> + case rabbit_mgmt_util:id(pattern, ReqData) of + none -> ?DEFAULT_PATTERN; + Other -> Other + end. diff --git a/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl b/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl index 9cf2ae71f89b..96a34bb5859e 100644 --- a/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl +++ b/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl @@ -37,7 +37,10 @@ groups() -> local_alarms_test, metadata_store_initialized_test, metadata_store_initialized_with_data_test, - is_quorum_critical_single_node_test]} + is_quorum_critical_single_node_test, + quorum_queues_without_elected_leader_single_node_test, + quorum_queues_without_elected_leader_across_all_virtual_hosts_single_node_test + ]} ]. all_tests() -> [ @@ -165,7 +168,8 @@ local_alarms_test(Config) -> is_quorum_critical_single_node_test(Config) -> - Check0 = http_get(Config, "/health/checks/node-is-quorum-critical", ?OK), + EndpointPath = "/health/checks/node-is-quorum-critical", + Check0 = http_get(Config, EndpointPath, ?OK), ?assertEqual(<<"single node cluster">>, maps:get(reason, Check0)), ?assertEqual(<<"ok">>, maps:get(status, Check0)), @@ -178,13 +182,14 @@ is_quorum_critical_single_node_test(Config) -> durable = true, auto_delete = false, arguments = Args})), - Check1 = http_get(Config, "/health/checks/node-is-quorum-critical", ?OK), + Check1 = http_get(Config, EndpointPath, ?OK), ?assertEqual(<<"single node cluster">>, maps:get(reason, Check1)), passed. is_quorum_critical_test(Config) -> - Check0 = http_get(Config, "/health/checks/node-is-quorum-critical", ?OK), + EndpointPath = "/health/checks/node-is-quorum-critical", + Check0 = http_get(Config, EndpointPath, ?OK), ?assertEqual(false, maps:is_key(reason, Check0)), ?assertEqual(<<"ok">>, maps:get(status, Check0)), @@ -198,7 +203,7 @@ is_quorum_critical_test(Config) -> durable = true, auto_delete = false, arguments = Args})), - Check1 = http_get(Config, "/health/checks/node-is-quorum-critical", ?OK), + Check1 = http_get(Config, EndpointPath, ?OK), ?assertEqual(false, maps:is_key(reason, Check1)), RaName = binary_to_atom(<<"%2F_", QName/binary>>, utf8), @@ -207,7 +212,104 @@ is_quorum_critical_test(Config) -> ok = rabbit_ct_broker_helpers:stop_node(Config, Server2), ok = rabbit_ct_broker_helpers:stop_node(Config, Server3), - Body = http_get_failed(Config, "/health/checks/node-is-quorum-critical"), + Body = http_get_failed(Config, EndpointPath), + ?assertEqual(<<"failed">>, maps:get(<<"status">>, Body)), + ?assertEqual(true, maps:is_key(<<"reason">>, Body)), + Queues = maps:get(<<"queues">>, Body), + ?assert(lists:any( + fun(Item) -> + QName =:= maps:get(<<"name">>, Item) + end, Queues)), + + passed. + +quorum_queues_without_elected_leader_single_node_test(Config) -> + EndpointPath = "/health/checks/quorum-queues-without-elected-leaders/all-vhosts/", + Check0 = http_get(Config, EndpointPath, ?OK), + ?assertEqual(false, maps:is_key(reason, Check0)), + ?assertEqual(<<"ok">>, maps:get(status, Check0)), + + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Args = [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-quorum-initial-group-size">>, long, 3}], + QName = <<"quorum_queues_without_elected_leader">>, + ?assertEqual({'queue.declare_ok', QName, 0, 0}, + amqp_channel:call(Ch, #'queue.declare'{ + queue = QName, + durable = true, + auto_delete = false, + arguments = Args + })), + + Check1 = http_get(Config, EndpointPath, ?OK), + ?assertEqual(false, maps:is_key(reason, Check1)), + + RaSystem = quorum_queues, + QResource = rabbit_misc:r(<<"/">>, queue, QName), + {ok, Q1} = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_db_queue, get, [QResource]), + + _ = rabbit_ct_broker_helpers:rpc(Config, 0, ra, stop_server, [RaSystem, amqqueue:get_pid(Q1)]), + + Body = http_get_failed(Config, EndpointPath), + ?assertEqual(<<"failed">>, maps:get(<<"status">>, Body)), + ?assertEqual(true, maps:is_key(<<"reason">>, Body)), + Queues = maps:get(<<"queues">>, Body), + ?assert(lists:any( + fun(Item) -> + QName =:= maps:get(<<"name">>, Item) + end, Queues)), + + _ = rabbit_ct_broker_helpers:rpc(Config, 0, ra, restart_server, [RaSystem, amqqueue:get_pid(Q1)]), + rabbit_ct_helpers:await_condition( + fun() -> + try + Check2 = http_get(Config, EndpointPath, ?OK), + false =:= maps:is_key(reason, Check2) + catch _:_ -> + false + end + end), + + passed. + +quorum_queues_without_elected_leader_across_all_virtual_hosts_single_node_test(Config) -> + VH2 = <<"vh-2">>, + rabbit_ct_broker_helpers:add_vhost(Config, VH2), + + EndpointPath1 = "/health/checks/quorum-queues-without-elected-leaders/vhost/%2f/", + EndpointPath2 = "/health/checks/quorum-queues-without-elected-leaders/vhost/vh-2/", + %% ^other + EndpointPath3 = "/health/checks/quorum-queues-without-elected-leaders/vhost/vh-2/pattern/%5Eother", + + Check0 = http_get(Config, EndpointPath1, ?OK), + Check0 = http_get(Config, EndpointPath2, ?OK), + ?assertEqual(false, maps:is_key(reason, Check0)), + ?assertEqual(<<"ok">>, maps:get(status, Check0)), + + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Args = [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-quorum-initial-group-size">>, long, 3}], + QName = <<"quorum_queues_without_elected_leader_across_all_virtual_hosts_single_node_test">>, + ?assertEqual({'queue.declare_ok', QName, 0, 0}, + amqp_channel:call(Ch, #'queue.declare'{ + queue = QName, + durable = true, + auto_delete = false, + arguments = Args + })), + + Check1 = http_get(Config, EndpointPath1, ?OK), + ?assertEqual(false, maps:is_key(reason, Check1)), + + RaSystem = quorum_queues, + QResource = rabbit_misc:r(<<"/">>, queue, QName), + {ok, Q1} = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_db_queue, get, [QResource]), + + _ = rabbit_ct_broker_helpers:rpc(Config, 0, ra, stop_server, [RaSystem, amqqueue:get_pid(Q1)]), + + Body = http_get_failed(Config, EndpointPath1), ?assertEqual(<<"failed">>, maps:get(<<"status">>, Body)), ?assertEqual(true, maps:is_key(<<"reason">>, Body)), Queues = maps:get(<<"queues">>, Body), @@ -216,8 +318,30 @@ is_quorum_critical_test(Config) -> QName =:= maps:get(<<"name">>, Item) end, Queues)), + %% virtual host vh-2 is still fine + Check2 = http_get(Config, EndpointPath2, ?OK), + ?assertEqual(false, maps:is_key(reason, Check2)), + + %% a different queue name pattern succeeds + Check3 = http_get(Config, EndpointPath3, ?OK), + ?assertEqual(false, maps:is_key(reason, Check3)), + + _ = rabbit_ct_broker_helpers:rpc(Config, 0, ra, restart_server, [RaSystem, amqqueue:get_pid(Q1)]), + rabbit_ct_helpers:await_condition( + fun() -> + try + Check4 = http_get(Config, EndpointPath1, ?OK), + false =:= maps:is_key(reason, Check4) + catch _:_ -> + false + end + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VH2), + passed. + virtual_hosts_test(Config) -> VHost1 = <<"vhost1">>, VHost2 = <<"vhost2">>,