feat: add dump for consul_kv (#3848)

Co-authored-by: nieyong <nieyong@staff.weibo.com>

apisix/control/router.lua

@@ -31,6 +31,41 @@ local get_method = ngx.req.get_method
 local _M = {}
 
 
+local function format_dismod_uri(mod_name, uri)
+    if core.string.has_prefix(uri, "/v1/") then
+        return uri
+    end
+
+    local tmp = {"/v1/discovery/", mod_name}
+    if not core.string.has_prefix(uri, "/") then
+        core.table.insert(tmp, "/")
+    end
+    core.table.insert(tmp, uri)
+
+    return core.table.concat(tmp, "")
+end
+
+-- we do not hardcode the discovery module's control api uri
+local function format_dismod_control_api_uris(mod_name, api_route)
+    if not api_route or #api_route == 0 then
+        return api_route
+    end
+
+    local clone_route = core.table.clone(api_route)
+    for _, v in ipairs(clone_route) do
+        local uris = v.uris
+        local target_uris = core.table.new(#uris, 0)
+        for _, uri in ipairs(uris) do
+            local target_uri = format_dismod_uri(mod_name, uri)
+            core.table.insert(target_uris, target_uri)
+        end
+        v.uris = target_uris
+    end
+
+    return clone_route
+end
+
+
 local fetch_control_api_router
 do
     local function register_api_routes(routes, api_routes)
@@ -78,14 +113,16 @@ function fetch_control_api_router()
             local api_fun = dis_mod.control_api
             if api_fun then
                 local api_route = api_fun()
-                register_api_routes(routes, api_route)
+                local format_route = format_dismod_control_api_uris(key, api_route)
+                register_api_routes(routes, format_route)
             end
 
             local dump_data = dis_mod.dump_data
             if dump_data then
+                local target_uri = format_dismod_uri(key, "/dump")
                 local item = {
                     methods = {"GET"},
-                    uris = {"/v1/discovery/" .. key .. "/dump"},
+                    uris = {target_uri},
                     handler = function()
                         return 200, dump_data()
                     end

apisix/discovery/consul_kv.lua

@@ -20,11 +20,12 @@ local core               = require("apisix.core")
 local resty_consul       = require('resty.consul')
 local cjson              = require('cjson')
 local http               = require('resty.http')
+local util               = require("apisix.cli.util")
 local ipairs             = ipairs
 local error              = error
 local ngx                = ngx
 local unpack             = unpack
-local ngx_re_match      = ngx.re.match
+local ngx_re_match       = ngx.re.match
 local tonumber           = tonumber
 local pairs              = pairs
 local ipairs             = ipairs
@@ -33,13 +34,14 @@ local ngx_timer_every    = ngx.timer.every
 local log                = core.log
 local ngx_decode_base64  = ngx.decode_base64
 local json_delay_encode  = core.json.delay_encode
-local cjson_null          = cjson.null
+local cjson_null         = cjson.null
 
 local applications = core.table.new(0, 5)
 local default_service
 local default_weight
 local default_prefix_rule
 local skip_keys_map = core.table.new(0, 1)
+local dump_params
 
 local events
 local events_list
@@ -82,6 +84,15 @@ local schema = {
                 type = "string",
             }
         },
+        dump = {
+            type = "object",
+            properties = {
+                path = {type = "string", minLength = 1},
+                load_on_init = {type = "boolean", default = true},
+                expire = {type = "integer", default = 0},
+            },
+            required = {"path"},
+        },
         default_service = {
             type = "object",
             properties = {
@@ -230,6 +241,72 @@ local function update_application(server_name_prefix, data)
 end
 
 
+local function read_dump_srvs()
+    local data, err = util.read_file(dump_params.path)
+    if not data then
+        log.notice("read dump file get error: ", err)
+        return
+    end
+
+    log.info("read dump file: ", data)
+    data = util.trim(data)
+    if #data == 0 then
+        log.error("dump file is empty")
+        return
+    end
+
+    local entity, err  = core.json.decode(data)
+    if err then
+        log.error("decoded dump data got error: ", err, ", file content: ", data)
+        return
+    end
+
+    if not entity.services or not entity.last_update then
+        log.warn("decoded dump data miss fields, file content: ", data)
+        return
+    end
+
+    local now_time = ngx.time()
+    log.info("dump file last_update: ", entity.last_update, ", dump_params.expire: ",
+        dump_params.expire, ", now_time: ", now_time)
+    if dump_params.expire ~= 0  and (entity.last_update + dump_params.expire) < now_time then
+       log.warn("dump file: ", dump_params.path, " had expired, ignored it")
+       return
+    end
+
+    applications = entity.services
+    log.info("load dump file into memory success")
+end
+
+
+local function write_dump_srvs()
+    local entity = {
+        services = applications,
+        last_update = ngx.time(),
+        expire = dump_params.expire, -- later need handle it
+    }
+    local data = core.json.encode(entity)
+    local succ, err =  util.write_file(dump_params.path, data)
+    if not succ then
+        log.error("write dump into file got error: ", err)
+    end
+end
+
+
+local function show_dump_file()
+    if not dump_params then
+        return 503, "dump params is nil"
+    end
+
+    local data, err = util.read_file(dump_params.path)
+    if not data then
+        return 503, err
+    end
+
+    return 200, data
+end
+
+
 function _M.connect(premature, consul_server)
     if premature then
         return
@@ -283,6 +360,10 @@ function _M.connect(premature, consul_server)
                 log.error("post_event failure with ", events_list._source,
                     ", update application error: ", err)
             end
+
+            if dump_params then
+                ngx_timer_at(0, write_dump_srvs)
+            end
         end
     end
 
@@ -356,6 +437,15 @@ function _M.init_worker()
         return
     end
 
+    if consul_conf.dump then
+      local dump = consul_conf.dump
+      dump_params = dump
+
+      if dump.load_on_init then
+          read_dump_srvs()
+      end
+    end
+
     events = require("resty.worker.events")
     events_list = events.event_list(
         "discovery_consul_update_application",
@@ -411,4 +501,15 @@ function _M.dump_data()
 end
 
 
+function _M.control_api()
+    return {
+        {
+            methods = {"GET"},
+            uris = {"/show_dump_file"},
+            handler = show_dump_file,
+        }
+    }
+end
+
+
 return _M

docs/en/latest/discovery/consul_kv.md

@@ -57,6 +57,9 @@ discovery:
         fail_timeout: 1           # default 1 ms
         weight: 1                 # default 1
         max_fails: 1              # default 1
+    dump:                         # if you need, when registered nodes updated can dump into file
+       path: "logs/consul_kv.dump"
+       expire: 2592000      # unit sec, here is 30 day
 ```
 
 And you can config it in short by default value:
@@ -73,6 +76,31 @@ The `keepalive` has two optional values:
 - `true`, default and recommend value, use the long pull way to query consul servers
 - `false`, not recommend, it would use the short pull way to query consul servers, then you can set the `fetch_interval` for fetch interval
 
+#### Dump Data
+
+When we need reload `apisix` online, as the `consul_kv` module maybe loads data from CONSUL slower than load routes from ETCD, and would get the log at the moment before load successfully from consul:
+
+```
+ http_access_phase(): failed to set upstream: no valid upstream node
+```
+
+So, we import the `dump` function for `consul_kv` module. When reload, would load the dump file before from consul; when the registered nodes in consul been updated, would dump the upstream nodes into file automatically.
+
+The `dump` has three optional values now:
+
+- `path`, the dump file save path
+    - support relative path, eg: `logs/consul_kv.dump`
+    - support absolute path, eg: `/tmp/consul_kv.bin`
+    - make sure the dump file's parent path exist
+    - make sure the `apisix` has the dump file's read-write access permission,eg: `chown  www:root conf/upstream.d/`
+- `load_on_init`, default value is `true`
+    - if `true`, just try to load the data from the dump file before loading data from  consul when starting, does not care the dump file exists or not
+    - if `false`, ignore loading data from the dump file
+    - Whether `true` or `false`, we don't need to prepare a dump file for apisix at anytime
+- `expire`, unit sec, avoiding load expired dump data when load
+    - default `0`, it is unexpired forever
+    - recommend 2592000, which is 30 days(equals 3600 \* 24 \* 30)
+
 ### Register Http API Services
 
 Service register Key&Value template:
@@ -147,7 +175,9 @@ You could find more usage in the `apisix/t/discovery/consul_kv.t` file.
 
 ## Debugging API
 
-It also offers control api for debugging:
+It also offers control api for debugging.
+
+### Memory Dump API
 
 ```shell
 GET /v1/discovery/consul_kv/dump
@@ -220,3 +250,35 @@ For example:
   }
 }
 ```
+
+### Show Dump File API
+
+It offers another control api for dump file view now. Maybe would add more api for debugging in future.
+
+```shell
+GET /v1/discovery/consul_kv/show_dump_file
+```
+
+For example:
+
+```shell
+curl http://127.0.0.1:9090/v1/discovery/consul_kv/show_dump_file | jq
+{
+  "services": {
+    "http://172.19.5.31:8500/v1/kv/upstreams/1614480/webpages/": [
+      {
+        "host": "172.19.5.12",
+        "port": 8000,
+        "weight": 120
+      },
+      {
+        "host": "172.19.5.13",
+        "port": 8000,
+        "weight": 120
+      }
+    ]
+  },
+  "expire": 0,
+  "last_update": 1615877468
+}
+```