From f99b7df2032de20e329191f89099fe10eb2f4d9e Mon Sep 17 00:00:00 2001 From: spacewander Date: Mon, 28 Nov 2022 10:29:10 +0800 Subject: [PATCH] feat: interact via gRPC in APISIX Admin API Signed-off-by: spacewander --- apisix/admin/init.lua | 11 +++++ apisix/cli/schema.lua | 7 ++- apisix/core/config_etcd.lua | 6 +-- apisix/core/etcd.lua | 69 +++++++++++++++++++---------- apisix/core/pubsub.lua | 4 +- apisix/plugins/server-info.lua | 6 ++- ci/linux_openresty_common_runner.sh | 2 +- ci/linux_openresty_runner.sh | 1 + rockspec/apisix-master-0.rockspec | 4 +- t/APISIX.pm | 11 +++++ t/plugin/grpc-transcode2.t | 12 +++-- 11 files changed, 97 insertions(+), 36 deletions(-) diff --git a/apisix/admin/init.lua b/apisix/admin/init.lua index 5827f83978fb..b21507ce350e 100644 --- a/apisix/admin/init.lua +++ b/apisix/admin/init.lua @@ -25,6 +25,7 @@ local ngx_time = ngx.time local ngx_timer_at = ngx.timer.at local ngx_worker_id = ngx.worker.id local tonumber = tonumber +local tostring = tostring local str_lower = string.lower local reload_event = "/apisix/admin/plugins/reload" local ipairs = ipairs @@ -111,6 +112,16 @@ local function strip_etcd_resp(data) data.node.createdIndex = nil data.node.modifiedIndex = nil end + + data.count = nil + data.more = nil + data.prev_kvs = nil + + if data.deleted then + -- We used to treat the type incorrectly. But for compatibility we follow + -- the existing type. + data.deleted = tostring(data.deleted) + end end return data diff --git a/apisix/cli/schema.lua b/apisix/cli/schema.lua index e1c1a0920812..5f1f789addda 100644 --- a/apisix/cli/schema.lua +++ b/apisix/cli/schema.lua @@ -61,7 +61,12 @@ local etcd_schema = { default = 30, minimum = 1, description = "etcd connection timeout in seconds", - } + }, + use_grpc = { + type = "boolean", + -- TODO: set true by default in v3.2 + default = false, + }, }, required = {"prefix", "host"} } diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua index 80d6d23e2493..08da8735af96 100644 --- a/apisix/core/config_etcd.lua +++ b/apisix/core/config_etcd.lua @@ -506,8 +506,8 @@ do return etcd_cli end - local err - etcd_cli, err = etcd_apisix.switch_proxy() + local _, err + etcd_cli, _, err = etcd_apisix.switch_proxy(true) return etcd_cli, err end end @@ -693,7 +693,7 @@ function _M.new(key, opts) else local etcd_cli, err = get_etcd() if not etcd_cli then - return nil, "failed to start a etcd instance: " .. err + return nil, "failed to start an etcd instance: " .. err end obj.etcd_cli = etcd_cli end diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua index d75a4191650f..a9077584537c 100644 --- a/apisix/core/etcd.lua +++ b/apisix/core/etcd.lua @@ -19,6 +19,7 @@ -- -- @module core.etcd +local require = require local fetch_local_conf = require("apisix.core.config_local").local_conf local array_mt = require("apisix.core.json").array_mt local v3_adapter = require("apisix.admin.v3_adapter") @@ -27,6 +28,7 @@ local clone_tab = require("table.clone") local health_check = require("resty.etcd.health_check") local pl_path = require("pl.path") local ipairs = ipairs +local pcall = pcall local setmetatable = setmetatable local string = string local tonumber = tonumber @@ -70,6 +72,18 @@ local function _new(etcd_conf) end end + if etcd_conf.use_grpc then + -- TODO: let lua-resty-etcd support more use cases + if etcd.user or ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then + etcd_conf.use_grpc = false + else + local ok = pcall(require, "resty.grpc") + if not ok then + etcd_conf.use_grpc = false + end + end + end + local etcd_cli, err = etcd.new(etcd_conf) if not etcd_cli then return nil, nil, err @@ -79,13 +93,41 @@ local function _new(etcd_conf) end -local function new() +--- +-- Create an etcd client which will connect to etcd without being proxyed by conf server. +-- This method is used in init_worker phase when the conf server is not ready. +-- +-- @function core.etcd.new_without_proxy +-- @treturn table|nil the etcd client, or nil if failed. +-- @treturn string|nil the configured prefix of etcd keys, or nil if failed. +-- @treturn nil|string the error message. +local function new_without_proxy() + local local_conf, err = fetch_local_conf() + if not local_conf then + return nil, nil, err + end + + local etcd_conf = clone_tab(local_conf.etcd) + return _new(etcd_conf) +end +_M.new_without_proxy = new_without_proxy + + +local function new(use_http) local local_conf, err = fetch_local_conf() if not local_conf then return nil, nil, err end local etcd_conf = clone_tab(local_conf.etcd) + if use_http then + etcd_conf.use_grpc = false + end + if etcd_conf.use_grpc then + -- TODO: add grpc proxy support later + return new_without_proxy() + end + local proxy_by_conf_server = false if local_conf.deployment then @@ -151,32 +193,13 @@ end _M.new = new ---- --- Create an etcd client which will connect to etcd without being proxyed by conf server. --- This method is used in init_worker phase when the conf server is not ready. --- --- @function core.etcd.new_without_proxy --- @treturn table|nil the etcd client, or nil if failed. --- @treturn string|nil the configured prefix of etcd keys, or nil if failed. --- @treturn nil|string the error message. -local function new_without_proxy() - local local_conf, err = fetch_local_conf() - if not local_conf then - return nil, nil, err - end - - local etcd_conf = clone_tab(local_conf.etcd) - return _new(etcd_conf) -end -_M.new_without_proxy = new_without_proxy - - -local function switch_proxy() +-- TODO: add grpc proxy support later, then we can remove "use_http" scaffold +local function switch_proxy(use_http) if ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then return new_without_proxy() end - local etcd_cli, prefix, err = new() + local etcd_cli, prefix, err = new(use_http) if not etcd_cli or err then return etcd_cli, prefix, err end diff --git a/apisix/core/pubsub.lua b/apisix/core/pubsub.lua index d6bcafad11f4..98db52cca2ea 100644 --- a/apisix/core/pubsub.lua +++ b/apisix/core/pubsub.lua @@ -59,7 +59,9 @@ local function get_cmd(data) for key, value in pairs(data) do -- There are sequence and command properties in the data, -- select the handler according to the command value. - if key ~= "sequence" then + if key ~= "sequence" and key ~= "req" then + -- new version of lua-protobuf will add a new field 'oneof_name = oneof_type' + -- so we also need to filter it out (in this case, the 'req' key) return key, value end end diff --git a/apisix/plugins/server-info.lua b/apisix/plugins/server-info.lua index b7cd67793d75..9ced84cd4042 100644 --- a/apisix/plugins/server-info.lua +++ b/apisix/plugins/server-info.lua @@ -172,7 +172,11 @@ local function report(premature, report_ttl) return else - server_info.etcd_version = res.body.etcdcluster + if res.body.etcdcluster == "" then + server_info.etcd_version = res.body.etcdserver + else + server_info.etcd_version = res.body.etcdcluster + end end end diff --git a/ci/linux_openresty_common_runner.sh b/ci/linux_openresty_common_runner.sh index 1ce9b7782900..9d48f8a32dc8 100755 --- a/ci/linux_openresty_common_runner.sh +++ b/ci/linux_openresty_common_runner.sh @@ -38,7 +38,7 @@ do_install() { # sudo apt-get install tree -y # tree deps - git clone https://github.com/openresty/test-nginx.git test-nginx + git clone --depth 1 https://github.com/openresty/test-nginx.git test-nginx make utils mkdir -p build-cache diff --git a/ci/linux_openresty_runner.sh b/ci/linux_openresty_runner.sh index 2e39224efc59..2cdc87b218f3 100755 --- a/ci/linux_openresty_runner.sh +++ b/ci/linux_openresty_runner.sh @@ -18,4 +18,5 @@ export OPENRESTY_VERSION=source +export TEST_CI_USE_GRPC=true . ./ci/linux_openresty_common_runner.sh diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index 227d6007125f..d42396634abb 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -34,7 +34,7 @@ dependencies = { "lua-resty-ctxdump = 0.1-0", "lua-resty-dns-client = 6.0.2", "lua-resty-template = 2.0", - "lua-resty-etcd = 1.9.0", + "lua-resty-etcd = 1.10.0", "api7-lua-resty-http = 0.2.0", "lua-resty-balancer = 0.04", "lua-resty-ngxvar = 0.5.2", @@ -46,7 +46,7 @@ dependencies = { "lua-resty-session = 3.10", "opentracing-openresty = 0.1", "lua-resty-radixtree = 2.8.2", - "lua-protobuf = 0.3.4", + "api7-lua-protobuf = 0.1.1", "lua-resty-openidc = 1.7.5", "luafilesystem = 1.7.0-2", "api7-lua-tinyyaml = 0.4.2", diff --git a/t/APISIX.pm b/t/APISIX.pm index 940e1ee869b5..9223df0ae775 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -837,6 +837,17 @@ deployment: _EOC_ if ($yaml_config !~ m/deployment:/) { + # TODO: remove this temporary option once we have using gRPC by default + if ($ENV{TEST_CI_USE_GRPC}) { + $default_deployment .= <<_EOC_; + etcd: + host: + - "http://127.0.0.1:2379" + prefix: /apisix + use_grpc: true +_EOC_ + } + $yaml_config = $default_deployment . $yaml_config; } diff --git a/t/plugin/grpc-transcode2.t b/t/plugin/grpc-transcode2.t index fe99671439c8..7dc42789a3bf 100644 --- a/t/plugin/grpc-transcode2.t +++ b/t/plugin/grpc-transcode2.t @@ -565,7 +565,10 @@ qr/request log: \{.*body":\"\{\\"result\\":3}/ local pb = require("pb") local old_f = pb.option pb.option = function(o) - ngx.log(ngx.WARN, "set protobuf option: ", o) + if o ~= "int64_as_string" then + -- filter out options set by other components + ngx.log(ngx.WARN, "set protobuf option: ", o) + end return old_f(o) end --- config @@ -669,13 +672,11 @@ qr/request log: \{.*body":\"\{\\"result\\":3}/ qr/set protobuf option: \w+/ --- grep_error_log_out set protobuf option: enum_as_name -set protobuf option: int64_as_string set protobuf option: auto_default_values set protobuf option: disable_hooks set protobuf option: enum_as_name set protobuf option: int64_as_number set protobuf option: enum_as_name -set protobuf option: int64_as_string @@ -684,7 +685,10 @@ set protobuf option: int64_as_string local pb = require("pb") local old_f = pb.option pb.option = function(o) - ngx.log(ngx.WARN, "set protobuf option: ", o) + if o ~= "int64_as_string" then + -- filter out options set by other components + ngx.log(ngx.WARN, "set protobuf option: ", o) + end return old_f(o) end --- config