From 1896cb1826255aefebe989c54a6dfd7e6feef53d Mon Sep 17 00:00:00 2001 From: yongboy Date: Tue, 23 Mar 2021 10:07:32 +0800 Subject: [PATCH] feat: add dump for consul_kv (#3848) Co-authored-by: nieyong --- apisix/control/router.lua | 41 ++- apisix/discovery/consul_kv.lua | 105 ++++++- docs/en/latest/discovery/consul_kv.md | 64 ++++- t/discovery/consul_kv_dump.t | 386 ++++++++++++++++++++++++++ 4 files changed, 591 insertions(+), 5 deletions(-) create mode 100644 t/discovery/consul_kv_dump.t diff --git a/apisix/control/router.lua b/apisix/control/router.lua index 49fdc23a47b7..43fb7e213989 100644 --- a/apisix/control/router.lua +++ b/apisix/control/router.lua @@ -31,6 +31,41 @@ local get_method = ngx.req.get_method local _M = {} +local function format_dismod_uri(mod_name, uri) + if core.string.has_prefix(uri, "/v1/") then + return uri + end + + local tmp = {"/v1/discovery/", mod_name} + if not core.string.has_prefix(uri, "/") then + core.table.insert(tmp, "/") + end + core.table.insert(tmp, uri) + + return core.table.concat(tmp, "") +end + +-- we do not hardcode the discovery module's control api uri +local function format_dismod_control_api_uris(mod_name, api_route) + if not api_route or #api_route == 0 then + return api_route + end + + local clone_route = core.table.clone(api_route) + for _, v in ipairs(clone_route) do + local uris = v.uris + local target_uris = core.table.new(#uris, 0) + for _, uri in ipairs(uris) do + local target_uri = format_dismod_uri(mod_name, uri) + core.table.insert(target_uris, target_uri) + end + v.uris = target_uris + end + + return clone_route +end + + local fetch_control_api_router do local function register_api_routes(routes, api_routes) @@ -78,14 +113,16 @@ function fetch_control_api_router() local api_fun = dis_mod.control_api if api_fun then local api_route = api_fun() - register_api_routes(routes, api_route) + local format_route = format_dismod_control_api_uris(key, api_route) + register_api_routes(routes, format_route) end local dump_data = dis_mod.dump_data if dump_data then + local target_uri = format_dismod_uri(key, "/dump") local item = { methods = {"GET"}, - uris = {"/v1/discovery/" .. key .. "/dump"}, + uris = {target_uri}, handler = function() return 200, dump_data() end diff --git a/apisix/discovery/consul_kv.lua b/apisix/discovery/consul_kv.lua index e94f04365c63..bd61e17e2713 100644 --- a/apisix/discovery/consul_kv.lua +++ b/apisix/discovery/consul_kv.lua @@ -20,11 +20,12 @@ local core = require("apisix.core") local resty_consul = require('resty.consul') local cjson = require('cjson') local http = require('resty.http') +local util = require("apisix.cli.util") local ipairs = ipairs local error = error local ngx = ngx local unpack = unpack -local ngx_re_match = ngx.re.match +local ngx_re_match = ngx.re.match local tonumber = tonumber local pairs = pairs local ipairs = ipairs @@ -33,13 +34,14 @@ local ngx_timer_every = ngx.timer.every local log = core.log local ngx_decode_base64 = ngx.decode_base64 local json_delay_encode = core.json.delay_encode -local cjson_null = cjson.null +local cjson_null = cjson.null local applications = core.table.new(0, 5) local default_service local default_weight local default_prefix_rule local skip_keys_map = core.table.new(0, 1) +local dump_params local events local events_list @@ -82,6 +84,15 @@ local schema = { type = "string", } }, + dump = { + type = "object", + properties = { + path = {type = "string", minLength = 1}, + load_on_init = {type = "boolean", default = true}, + expire = {type = "integer", default = 0}, + }, + required = {"path"}, + }, default_service = { type = "object", properties = { @@ -230,6 +241,72 @@ local function update_application(server_name_prefix, data) end +local function read_dump_srvs() + local data, err = util.read_file(dump_params.path) + if not data then + log.notice("read dump file get error: ", err) + return + end + + log.info("read dump file: ", data) + data = util.trim(data) + if #data == 0 then + log.error("dump file is empty") + return + end + + local entity, err = core.json.decode(data) + if err then + log.error("decoded dump data got error: ", err, ", file content: ", data) + return + end + + if not entity.services or not entity.last_update then + log.warn("decoded dump data miss fields, file content: ", data) + return + end + + local now_time = ngx.time() + log.info("dump file last_update: ", entity.last_update, ", dump_params.expire: ", + dump_params.expire, ", now_time: ", now_time) + if dump_params.expire ~= 0 and (entity.last_update + dump_params.expire) < now_time then + log.warn("dump file: ", dump_params.path, " had expired, ignored it") + return + end + + applications = entity.services + log.info("load dump file into memory success") +end + + +local function write_dump_srvs() + local entity = { + services = applications, + last_update = ngx.time(), + expire = dump_params.expire, -- later need handle it + } + local data = core.json.encode(entity) + local succ, err = util.write_file(dump_params.path, data) + if not succ then + log.error("write dump into file got error: ", err) + end +end + + +local function show_dump_file() + if not dump_params then + return 503, "dump params is nil" + end + + local data, err = util.read_file(dump_params.path) + if not data then + return 503, err + end + + return 200, data +end + + function _M.connect(premature, consul_server) if premature then return @@ -283,6 +360,10 @@ function _M.connect(premature, consul_server) log.error("post_event failure with ", events_list._source, ", update application error: ", err) end + + if dump_params then + ngx_timer_at(0, write_dump_srvs) + end end end @@ -356,6 +437,15 @@ function _M.init_worker() return end + if consul_conf.dump then + local dump = consul_conf.dump + dump_params = dump + + if dump.load_on_init then + read_dump_srvs() + end + end + events = require("resty.worker.events") events_list = events.event_list( "discovery_consul_update_application", @@ -411,4 +501,15 @@ function _M.dump_data() end +function _M.control_api() + return { + { + methods = {"GET"}, + uris = {"/show_dump_file"}, + handler = show_dump_file, + } + } +end + + return _M diff --git a/docs/en/latest/discovery/consul_kv.md b/docs/en/latest/discovery/consul_kv.md index b5af505b9f91..f7291543c81f 100644 --- a/docs/en/latest/discovery/consul_kv.md +++ b/docs/en/latest/discovery/consul_kv.md @@ -57,6 +57,9 @@ discovery: fail_timeout: 1 # default 1 ms weight: 1 # default 1 max_fails: 1 # default 1 + dump: # if you need, when registered nodes updated can dump into file + path: "logs/consul_kv.dump" + expire: 2592000 # unit sec, here is 30 day ``` And you can config it in short by default value: @@ -73,6 +76,31 @@ The `keepalive` has two optional values: - `true`, default and recommend value, use the long pull way to query consul servers - `false`, not recommend, it would use the short pull way to query consul servers, then you can set the `fetch_interval` for fetch interval +#### Dump Data + +When we need reload `apisix` online, as the `consul_kv` module maybe loads data from CONSUL slower than load routes from ETCD, and would get the log at the moment before load successfully from consul: + +``` + http_access_phase(): failed to set upstream: no valid upstream node +``` + +So, we import the `dump` function for `consul_kv` module. When reload, would load the dump file before from consul; when the registered nodes in consul been updated, would dump the upstream nodes into file automatically. + +The `dump` has three optional values now: + +- `path`, the dump file save path + - support relative path, eg: `logs/consul_kv.dump` + - support absolute path, eg: `/tmp/consul_kv.bin` + - make sure the dump file's parent path exist + - make sure the `apisix` has the dump file's read-write access permission,eg: `chown www:root conf/upstream.d/` +- `load_on_init`, default value is `true` + - if `true`, just try to load the data from the dump file before loading data from consul when starting, does not care the dump file exists or not + - if `false`, ignore loading data from the dump file + - Whether `true` or `false`, we don't need to prepare a dump file for apisix at anytime +- `expire`, unit sec, avoiding load expired dump data when load + - default `0`, it is unexpired forever + - recommend 2592000, which is 30 days(equals 3600 \* 24 \* 30) + ### Register Http API Services Service register Key&Value template: @@ -147,7 +175,9 @@ You could find more usage in the `apisix/t/discovery/consul_kv.t` file. ## Debugging API -It also offers control api for debugging: +It also offers control api for debugging. + +### Memory Dump API ```shell GET /v1/discovery/consul_kv/dump @@ -220,3 +250,35 @@ For example: } } ``` + +### Show Dump File API + +It offers another control api for dump file view now. Maybe would add more api for debugging in future. + +```shell +GET /v1/discovery/consul_kv/show_dump_file +``` + +For example: + +```shell +curl http://127.0.0.1:9090/v1/discovery/consul_kv/show_dump_file | jq +{ + "services": { + "http://172.19.5.31:8500/v1/kv/upstreams/1614480/webpages/": [ + { + "host": "172.19.5.12", + "port": 8000, + "weight": 120 + }, + { + "host": "172.19.5.13", + "port": 8000, + "weight": 120 + } + ] + }, + "expire": 0, + "last_update": 1615877468 +} +``` diff --git a/t/discovery/consul_kv_dump.t b/t/discovery/consul_kv_dump.t new file mode 100644 index 000000000000..7d4e276ad54c --- /dev/null +++ b/t/discovery/consul_kv_dump.t @@ -0,0 +1,386 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); +log_level("info"); + + +add_block_preprocessor(sub { + my ($block) = @_; + + my $http_config = $block->http_config // <<_EOC_; + + server { + listen 30511; + + location /hello { + content_by_lua_block { + ngx.say("server 1") + } + } + } +_EOC_ + + $block->set_value("http_config", $http_config); +}); + +our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + enable_control: true + +discovery: + consul_kv: + servers: + - "http://127.0.0.1:8500" + dump: + path: "consul_kv.dump" + load_on_init: true +_EOC_ + + +run_tests(); + +__DATA__ + +=== TEST 1: prepare nodes +--- config +location /v1/kv { + proxy_pass http://127.0.0.1:8500; +} +--- request eval +[ + "DELETE /v1/kv/upstreams/?recurse=true", + "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 1, \"fail_timeout\": 1}", +] +--- response_body eval +[ + 'true', + 'true', + 'true', +] + + + +=== TEST 2: show dump services +--- yaml_config eval: $::yaml_config +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + ngx.sleep(2) + + local code, body, res = t.test('/v1/discovery/consul_kv/show_dump_file', + ngx.HTTP_GET) + local entity = json.decode(res) + ngx.say(json.encode(entity.services)) + } + } +--- timeout: 3 +--- request +GET /t +--- response_body +{"http://127.0.0.1:8500/v1/kv/upstreams/webpages/":[{"host":"127.0.0.1","port":30511,"weight":1}]} + + + +=== TEST 3: prepare dump file for next test +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + enable_control: true + +discovery: + consul_kv: + servers: + - "http://127.0.0.1:8500" + dump: + path: "/tmp/consul_kv.dump" + load_on_init: true +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul_kv + type: roundrobin +#END +--- request +GET /hello +--- response_body +server 1 + + + +=== TEST 4: clean registered nodes +--- config +location /v1/kv { + proxy_pass http://127.0.0.1:8500; +} +--- request eval +[ + "DELETE /v1/kv/upstreams/?recurse=true", +] +--- response_body eval +[ + 'true' +] + + + +=== TEST 5: test load dump on init +Configure the invalid consul server addr, and loading the last test 3 generated /tmp/consul_kv.dump file into memory when initializing +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + enable_control: true + +discovery: + consul_kv: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul_kv.dump" + load_on_init: true +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul_kv + type: roundrobin +#END +--- request +GET /hello +--- response_body +server 1 + + + +=== TEST 6: delete dump file +--- config + location /t { + content_by_lua_block { + local util = require("apisix.cli.util") + local succ, err = util.execute_cmd("rm -f /tmp/consul_kv.dump") + ngx.say(succ and "success" or err) + } + } +--- request +GET /t +--- response_body +success + + + +=== TEST 7: miss load dump on init +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + enable_control: true + +discovery: + consul_kv: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul_kv.dump" + load_on_init: true +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul_kv + type: roundrobin +#END +--- request +GET /hello +--- error_code: 503 + + + +=== TEST 8: prepare expired dump file +--- config + location /t { + content_by_lua_block { + local util = require("apisix.cli.util") + local json = require("toolkit.json") + + local applications = json.decode('{"http://127.0.0.1:8500/v1/kv/upstreams/webpages/":[{"host":"127.0.0.1","port":30511,"weight":1}]}') + local entity = { + services = applications, + last_update = ngx.time(), + expire = 10, + } + local succ, err = util.write_file("/tmp/consul_kv.dump", json.encode(entity)) + + ngx.sleep(2) + ngx.say(succ and "success" or err) + } + } +--- timeout: 3 +--- request +GET /t +--- response_body +success + + + +=== TEST 9: unexpired dump +test load unexpired /tmp/consul_kv.dump file generated by upper test when initializing + when initializing +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + enable_control: true + +discovery: + consul_kv: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul_kv.dump" + load_on_init: true + expire: 5 +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul_kv + type: roundrobin +#END +--- request +GET /hello +--- response_body +server 1 + + + +=== TEST 10: expired dump +test load expired ( by check: (dump_file.last_update + dump.expire) < ngx.time ) ) /tmp/consul_kv.dump file generated by upper test when initializing + when initializing +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + enable_control: true + +discovery: + consul_kv: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul_kv.dump" + load_on_init: true + expire: 1 +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul_kv + type: roundrobin +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +dump file: /tmp/consul_kv.dump had expired, ignored it + + + +=== TEST 11: delete dump file +--- config + location /t { + content_by_lua_block { + local util = require("apisix.cli.util") + local succ, err = util.execute_cmd("rm -f /tmp/consul_kv.dump") + ngx.say(succ and "success" or err) + } + } +--- request +GET /t +--- response_body +success + + + +=== TEST 12: dump file inexistence +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + enable_control: true + +discovery: + consul_kv: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul_kv.dump" +#END +--- request +GET /v1/discovery/consul_kv/show_dump_file +--- error_code: 503 + + + +=== TEST 13: no dump config +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + enable_control: true + +discovery: + consul_kv: + servers: + - "http://127.0.0.1:38500" +#END +--- request +GET /v1/discovery/consul_kv/show_dump_file +--- error_code: 503