feat: allow getting health check status via control API (#3345)

Fix #2798

apisix/control/v1.lua

@@ -16,6 +16,13 @@
 --
 local core = require("apisix.core")
 local plugin = require("apisix.plugin")
+local get_routes = require("apisix.router").http_routes
+local get_services = require("apisix.http.service").services
+local upstream_mod = require("apisix.upstream")
+local get_upstreams = upstream_mod.upstreams
+local ipairs = ipairs
+local str_format = string.format
+local ngx_var = ngx.var
 
 
 local _M = {}
@@ -49,11 +56,120 @@ function _M.schema()
 end
 
 
+local function extra_checker_info(value, src_type)
+    local checker = value.checker
+    local upstream = value.checker_upstream
+    local host = upstream.checks and upstream.checks.active and upstream.checks.active.host
+    local port = upstream.checks and upstream.checks.active and upstream.checks.active.port
+    local nodes = upstream.nodes
+    local healthy_nodes = core.table.new(#nodes, 0)
+    for _, node in ipairs(nodes) do
+        local ok = checker:get_target_status(node.host, port or node.port, host)
+        if ok then
+            core.table.insert(healthy_nodes, node)
+        end
+    end
+
+    local conf = value.value
+    return {
+        name = upstream_mod.get_healthchecker_name(value),
+        src_id = conf.id,
+        src_type = src_type,
+        nodes = nodes,
+        healthy_nodes = healthy_nodes,
+    }
+end
+
+
+local function iter_and_add_healthcheck_info(infos, values, src_type)
+    if not values then
+        return
+    end
+
+    for _, value in core.config_util.iterate_values(values) do
+        if value.checker then
+            core.table.insert(infos, extra_checker_info(value, src_type))
+        end
+    end
+end
+
+
+function _M.get_health_checkers()
+    local infos = {}
+    local routes = get_routes()
+    iter_and_add_healthcheck_info(infos, routes, "routes")
+    local services = get_services()
+    iter_and_add_healthcheck_info(infos, services, "services")
+    local upstreams = get_upstreams()
+    iter_and_add_healthcheck_info(infos, upstreams, "upstreams")
+    return 200, infos
+end
+
+
+local function iter_and_find_healthcheck_info(values, src_type, src_id)
+    if not values then
+        return nil, str_format("%s[%s] not found", src_type, src_id)
+    end
+
+    for _, value in core.config_util.iterate_values(values) do
+        if value.value.id == src_id then
+            if not value.checker then
+                return nil, str_format("no checker for %s[%s]", src_type, src_id)
+            end
+
+            return extra_checker_info(value, src_type)
+        end
+    end
+
+    return nil, str_format("%s[%s] not found", src_type, src_id)
+end
+
+
+function _M.get_health_checker()
+    local uri_segs = core.utils.split_uri(ngx_var.uri)
+    core.log.info("healthcheck uri: ", core.json.delay_encode(uri_segs))
+
+    local src_type, src_id = uri_segs[4], uri_segs[5]
+    if not src_id then
+        return 404, {error_msg = str_format("missing src id for src type %s", src_type)}
+    end
+
+    local values
+    if src_type == "routes" then
+        values = get_routes()
+    elseif src_type == "services" then
+        values = get_services()
+    elseif src_type == "upstreams" then
+        values = get_upstreams()
+    else
+        return 400, {error_msg = str_format("invalid src type %s", src_type)}
+    end
+
+    local info, err = iter_and_find_healthcheck_info(values, src_type, src_id)
+    if not info then
+        return 404, {error_msg = err}
+    end
+    return 200, info
+end
+
+
 return {
     -- /v1/schema
     {
         methods = {"GET"},
         uris = {"/schema"},
         handler = _M.schema,
+    },
+    -- /v1/healthcheck
+    {
+        methods = {"GET"},
+        uris = {"/healthcheck"},
+        handler = _M.get_health_checkers,
+    },
+    -- /v1/healthcheck/{src_type}/{src_id}
+    {
+        methods = {"GET"},
+        uris = {"/healthcheck/*"},
+        handler = _M.get_health_checker,
     }
 }

apisix/upstream.lua

@@ -60,6 +60,12 @@ local function release_checker(healthcheck_parent)
 end
 
 
+local function get_healthchecker_name(value)
+    return "upstream#" .. value.key
+end
+_M.get_healthchecker_name = get_healthchecker_name
+
+
 local function create_checker(upstream)
     if healthcheck == nil then
         healthcheck = require("resty.healthcheck")
@@ -71,7 +77,7 @@ local function create_checker(upstream)
     end
 
     local checker, err = healthcheck.new({
-        name = "upstream#" .. healthcheck_parent.key,
+        name = get_healthchecker_name(healthcheck_parent),
         shm_name = "upstream-healthcheck",
         checks = upstream.checks,
     })

doc/control-api.md

@@ -79,3 +79,102 @@ Return the jsonschema used by this APISIX instance in the format below:
 For `plugins` part, only enabled plugins will be returned. Some plugins may lack
 of fields like `consumer_schema` or `type`, it is dependended by the plugin's
 definition.
+
+### GET /v1/healthcheck
+
+Introduced since `v2.3`.
+
+Return current [health check](health-check.md) status in the format below:
+
+```json
+[
+    {
+        "healthy_nodes": [
+            {
+                "host": "127.0.0.1",
+                "port": 1980,
+                "weight": 1
+            }
+        ],
+        "name": "upstream#/upstreams/1",
+        "nodes": [
+            {
+                "host": "127.0.0.1",
+                "port": 1980,
+                "weight": 1
+            },
+            {
+                "host": "127.0.0.2",
+                "port": 1988,
+                "weight": 1
+            }
+        ],
+        "src_id": "1",
+        "src_type": "upstreams"
+    },
+    {
+        "healthy_nodes": [
+            {
+                "host": "127.0.0.1",
+                "port": 1980,
+                "weight": 1
+            }
+        ],
+        "name": "upstream#/routes/1",
+        "nodes": [
+            {
+                "host": "127.0.0.1",
+                "port": 1980,
+                "weight": 1
+            },
+            {
+                "host": "127.0.0.1",
+                "port": 1988,
+                "weight": 1
+            }
+        ],
+        "src_id": "1",
+        "src_type": "routes"
+    }
+]
+```
+
+Each entry contains fields below:
+
+* src_type: where the health checker comes from. The value is one of `["routes", "services", "upstreams"]`.
+* src_id: the id of object which creates the health checker. For example, if Upstream
+object with id 1 creates a health checker, the `src_type` is `upstreams` and the `src_id` is `1`.
+* name: the name of the health checker.
+* nodes: the target nodes of the health checker.
+* healthy_nodes: the healthy node known by the health checker.
+
+User can also use `/v1/healthcheck/$src_type/$src_id` can get the status of a health checker.
+
+For example, `GET /v1/healthcheck/upstreams/1` returns:
+
+```json
+{
+    "healthy_nodes": [
+        {
+            "host": "127.0.0.1",
+            "port": 1980,
+            "weight": 1
+        }
+    ],
+    "name": "upstream#/upstreams/1",
+    "nodes": [
+        {
+            "host": "127.0.0.1",
+            "port": 1980,
+            "weight": 1
+        },
+        {
+            "host": "127.0.0.2",
+            "port": 1988,
+            "weight": 1
+        }
+    ],
+    "src_id": "1",
+    "src_type": "upstreams"
+}
+```

doc/health-check.md

@@ -22,6 +22,9 @@
 Health Check of APISIX is based on [lua-resty-healthcheck](https://github.com/Kong/lua-resty-healthcheck),
 you can use it for upstream.
 
+Note that we only start the health check when the upstream is hit by a request.
+There won't be any health check if an upstream is configured but isn't in used.
+
 The following is an example of health check:
 
 ```shell
@@ -105,3 +108,5 @@ contains: `active` or `passive`.
   * `passive.unhealthy.tcp_failures`: Number of TCP failures in proxied traffic to consider a target unhealthy, as observed by passive health checks.
   * `passive.unhealthy.timeouts`: Number of timeouts in proxied traffic to consider a target unhealthy, as observed by passive health checks.
   * `passive.unhealthy.http_failures`: Number of HTTP failures in proxied traffic (as defined by `passive.unhealthy.http_statuses`) to consider a target unhealthy, as observed by passive health checks.
+
+The health check status can be fetched via `GET /v1/healthcheck` in [control API](./control-api.md).

doc/zh-cn/health-check.md

@@ -23,6 +23,9 @@
 
 APISIX的健康检查使用[lua-resty-healthcheck](https://github.com/Kong/lua-resty-healthcheck)实现，你可以在upstream中使用它。
 
+注意只有在 upstream 被请求时才会开始健康检查。
+如果一个 upstream 被配置但没有被请求，那么就不会有健康检查。
+
 下面是一个检查检查的例子：
 
 ```shell
@@ -105,3 +108,5 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
     * `passive.unhealthy.tcp_failures`: 如果TCP通讯失败次数超过 `tcp_failures` 次，则将upstream节点设置为 `unhealthy` 状态。
     * `passive.unhealthy.timeouts`: 如果被动健康检查超时次数超过 `timeouts` 次，则将upstream节点设置为 `unhealthy` 状态。
     * `passive.unhealthy.http_failures`: 如果被动健康检查的HTTP请求失败（由 `passive.unhealthy.http_statuses` 定义）的次数超过 `http_failures`次，则将upstream节点设置为 `unhealthy` 状态。
+
+健康检查信息可以通过 [控制接口](./control_api.md) 中的 `GET /v1/healthcheck` 接口得到。