diff --git a/.travis.yml b/.travis.yml index ebb3e6929969..f5335b838cd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,6 @@ addons: - libreadline-dev - libssl-dev - perl - - etcd homebrew: update: true diff --git a/.travis/linux_apisix_current_luarocks_runner.sh b/.travis/linux_apisix_current_luarocks_runner.sh index 5a48932e8a3a..894542a6e4ad 100755 --- a/.travis/linux_apisix_current_luarocks_runner.sh +++ b/.travis/linux_apisix_current_luarocks_runner.sh @@ -39,6 +39,8 @@ do_install() { sudo make install > build.log 2>&1 || (cat build.log && exit 1) cd .. rm -rf luarocks-2.4.4 + + ./utils/install-etcd.sh } script() { @@ -48,8 +50,8 @@ script() { sudo service etcd start sudo service etcd stop mkdir -p ~/etcd-data - /usr/bin/etcd --listen-client-urls 'http://0.0.0.0:2379' --advertise-client-urls='http://0.0.0.0:2379' --data-dir ~/etcd-data > /dev/null 2>&1 & - etcd --version + etcd --listen-client-urls 'http://0.0.0.0:2379' --advertise-client-urls='http://0.0.0.0:2379' --data-dir ~/etcd-data > /dev/null 2>&1 & + etcdctl version sleep 5 sudo rm -rf /usr/local/apisix diff --git a/.travis/linux_openresty_mtls_runner.sh b/.travis/linux_openresty_mtls_runner.sh index 876c868f9b58..a3b0ab0bf466 100755 --- a/.travis/linux_openresty_mtls_runner.sh +++ b/.travis/linux_openresty_mtls_runner.sh @@ -59,6 +59,7 @@ do_install() { sudo luarocks install luacheck > build.log 2>&1 || (cat build.log && exit 1) + ./utils/install-etcd.sh if [ ! -f "build-cache/apisix-master-0.rockspec" ]; then create_lua_deps @@ -92,8 +93,8 @@ script() { openresty -V sudo service etcd stop mkdir -p ~/etcd-data - /usr/bin/etcd --listen-client-urls 'http://0.0.0.0:2379' --advertise-client-urls='http://0.0.0.0:2379' --data-dir ~/etcd-data > /dev/null 2>&1 & - etcd --version + etcd --listen-client-urls 'http://0.0.0.0:2379' --advertise-client-urls='http://0.0.0.0:2379' --data-dir ~/etcd-data > /dev/null 2>&1 & + etcdctl version sleep 5 diff --git a/.travis/linux_openresty_runner.sh b/.travis/linux_openresty_runner.sh index b37aaea42315..5c9aec471b44 100755 --- a/.travis/linux_openresty_runner.sh +++ b/.travis/linux_openresty_runner.sh @@ -74,6 +74,8 @@ do_install() { sudo luarocks install luacheck > build.log 2>&1 || (cat build.log && exit 1) + ./utils/install-etcd.sh + if [ ! -f "build-cache/apisix-master-0.rockspec" ]; then create_lua_deps @@ -127,8 +129,8 @@ script() { openresty -V sudo service etcd stop mkdir -p ~/etcd-data - /usr/bin/etcd --listen-client-urls 'http://0.0.0.0:2379' --advertise-client-urls='http://0.0.0.0:2379' --data-dir ~/etcd-data > /dev/null 2>&1 & - etcd --version + etcd --listen-client-urls 'http://0.0.0.0:2379' --advertise-client-urls='http://0.0.0.0:2379' --data-dir ~/etcd-data > /dev/null 2>&1 & + etcdctl version sleep 5 ./build-cache/grpc_server_example & diff --git a/.travis/linux_tengine_runner.sh b/.travis/linux_tengine_runner.sh index 12b6819f0b39..a6b2408b1d8b 100755 --- a/.travis/linux_tengine_runner.sh +++ b/.travis/linux_tengine_runner.sh @@ -259,6 +259,8 @@ do_install() { sudo luarocks install luacheck > build.log 2>&1 || (cat build.log && exit 1) + ./utils/install-etcd.sh + git clone https://github.com/iresty/test-nginx.git test-nginx make utils @@ -280,8 +282,8 @@ script() { openresty -V sudo service etcd stop mkdir -p ~/etcd-data - /usr/bin/etcd --listen-client-urls 'http://0.0.0.0:2379' --advertise-client-urls='http://0.0.0.0:2379' --data-dir ~/etcd-data > /dev/null 2>&1 & - etcd --version + etcd --listen-client-urls 'http://0.0.0.0:2379' --advertise-client-urls='http://0.0.0.0:2379' --data-dir ~/etcd-data > /dev/null 2>&1 & + etcdctl version sleep 5 ./build-cache/grpc_server_example & diff --git a/.travis/osx_openresty_runner.sh b/.travis/osx_openresty_runner.sh index 808359d3e088..f2e9b83d4a9e 100755 --- a/.travis/osx_openresty_runner.sh +++ b/.travis/osx_openresty_runner.sh @@ -65,7 +65,7 @@ script() { export_or_prefix export PATH=$OPENRESTY_PREFIX/nginx/sbin:$OPENRESTY_PREFIX/luajit/bin:$OPENRESTY_PREFIX/bin:$PATH - etcd --enable-v2=true & + etcd & sleep 1 ./grpc_server_example & diff --git a/README.md b/README.md index 9f6d3d2263e8..e909a3c49c39 100644 --- a/README.md +++ b/README.md @@ -166,10 +166,9 @@ There are several ways to install the Apache Release version of APISIX: apisix start ``` -**Note**: Apache APISIX does not yet support the v3 protocol of etcd, so you need to enable v2 protocol when starting etcd. -We are doing support for etcd v3 protocol. +**Note**: Apache APISIX would not support the v2 protocol of etcd anymore since APISIX v2.0, so you need to enable v3 protocol when starting etcd, if etcd version is below v3.4. ```shell -etcd --enable-v2=true & +export ETCDCTL_API=3 ``` ## For Developer diff --git a/README_CN.md b/README_CN.md index 10c948a7b7e0..f304d7ed907b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -165,10 +165,9 @@ CentOS 7, Ubuntu 16.04, Ubuntu 18.04, Debian 9, Debian 10, macOS, **ARM64** Ubun apisix start ``` -**注意**:Apache APISIX 现在还不支持 etcd 的 v3 协议,所以启动 etcd 时需要开启 v2 协议的支持。 -我们正在做 etcd v3 协议的支持。 +**注意**:Apache APISIX 从 v2.0 开始不再支持 etcd v2 协议,如果 etcd 版本低于 v3.4,启动 etcd 时需要开启 v3 协议的支持。 ```shell -etcd --enable-v2=true & +export ETCDCTL_API=3 ``` ## 针对开发者 diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua index 6e616b08f906..b651ea994581 100644 --- a/apisix/core/config_etcd.lua +++ b/apisix/core/config_etcd.lua @@ -17,6 +17,7 @@ local config_local = require("apisix.core.config_local") local log = require("apisix.core.log") local json = require("apisix.core.json") +local etcd_apisix = require("apisix.core.etcd") local etcd = require("resty.etcd") local new_tab = require("table.new") local clone_tab = require("table.clone") @@ -33,6 +34,7 @@ local sub_str = string.sub local tostring = tostring local tonumber = tonumber local pcall = pcall +local error = error local created_obj = {} @@ -42,6 +44,7 @@ local _M = { clear_local_cache = config_local.clear_cache, } + local mt = { __index = _M, __tostring = function(self) @@ -55,7 +58,7 @@ local function getkey(etcd_cli, key) return nil, "not inited" end - local res, err = etcd_cli:get(key) + local res, err = etcd_cli:readdir(key) if not res then -- log.error("failed to get key from etcd: ", err) return nil, err @@ -65,6 +68,11 @@ local function getkey(etcd_cli, key) return nil, "failed to get key from etcd" end + res, err = etcd_apisix.get_format(res, key) + if not res then + return nil, err + end + return res end @@ -74,7 +82,7 @@ local function readdir(etcd_cli, key) return nil, nil, "not inited" end - local res, err = etcd_cli:readdir(key, true) + local res, err = etcd_cli:readdir(key) if not res then -- log.error("failed to get key from etcd: ", err) return nil, nil, err @@ -84,6 +92,11 @@ local function readdir(etcd_cli, key) return nil, "failed to read etcd dir" end + res, err = etcd_apisix.get_format(res, key .. "/") + if not res then + return nil, err + end + return res end @@ -92,17 +105,30 @@ local function waitdir(etcd_cli, key, modified_index, timeout) return nil, nil, "not inited" end - local res, err = etcd_cli:waitdir(key, modified_index, timeout) + local opts = {} + opts.start_revision = modified_index + opts.timeout = timeout + local res_func, func_err = etcd_cli:watchdir(key, opts) + if not res_func then + return nil, func_err + end + + -- in etcd v3, the 1st res of watch is watch info, useless to us. + -- try twice to skip create info + local res, err = res_func() + if not res or not res.result or not res.result.events then + res, err = res_func() + end + if not res then -- log.error("failed to get key from etcd: ", err) return nil, err end - if type(res.body) ~= "table" then + if type(res.result) ~= "table" then return nil, "failed to read etcd dir" end - - return res + return etcd_apisix.watch_format(res) end @@ -118,10 +144,6 @@ function _M.upgrade_version(self, new_ver) end local pre_index = self.prev_index - if not pre_index then - self.prev_index = new_ver - return - end if new_ver <= pre_index then return @@ -182,7 +204,7 @@ local function sync_data(self) if type(item.value) ~= "table" then data_valid = false log.error("invalid item data of [", self.key .. "/" .. key, - "], val: ", tostring(item.value), + "], val: ", item.value, ", it shoud be a object") end @@ -268,86 +290,90 @@ local function sync_data(self) return false, err end - local key = short_key(self, res.key) - if res.value and type(res.value) ~= "table" then - self:upgrade_version(res.modifiedIndex) - return false, "invalid item data of [" .. self.key .. "/" .. key - .. "], val: " .. tostring(res.value) - .. ", it shoud be a object" - end - - if res.value and self.item_schema then - local ok, err = check_schema(self.item_schema, res.value) - if not ok then + local res_copy = res + for _, res in ipairs(res_copy) do + local key = short_key(self, res.key) + if res.value and type(res.value) ~= "table" then self:upgrade_version(res.modifiedIndex) - - return false, "failed to check item data of [" - .. self.key .. "] err:" .. err + return false, "invalid item data of [" .. self.key .. "/" .. key + .. "], val: " .. res.value + .. ", it shoud be a object" end - end - self:upgrade_version(res.modifiedIndex) + if res.value and self.item_schema then + local ok, err = check_schema(self.item_schema, res.value) + if not ok then + self:upgrade_version(res.modifiedIndex) - if res.dir then - if res.value then - return false, "todo: support for parsing `dir` response " - .. "structures. " .. json.encode(res) + return false, "failed to check item data of [" + .. self.key .. "] err:" .. err + end end - return false - end - if self.filter then - self.filter(res) - end + self:upgrade_version(res.modifiedIndex) - local pre_index = self.values_hash[key] - if pre_index then - local pre_val = self.values[pre_index] - if pre_val and pre_val.clean_handlers then - for _, clean_handler in ipairs(pre_val.clean_handlers) do - clean_handler(pre_val) + if res.dir then + if res.value then + return false, "todo: support for parsing `dir` response " + .. "structures. " .. json.encode(res) end - pre_val.clean_handlers = nil + return false end - if res.value then - res.value.id = key - self.values[pre_index] = res - res.clean_handlers = {} - - else - self.sync_times = self.sync_times + 1 - self.values[pre_index] = false + if self.filter then + self.filter(res) end - elseif res.value then - res.clean_handlers = {} - insert_tab(self.values, res) - self.values_hash[key] = #self.values - res.value.id = key - end - - -- avoid space waste - -- todo: need to cover this path, it is important. - if self.sync_times > 100 then - local count = 0 - for i = 1, #self.values do - local val = self.values[i] - self.values[i] = nil - if val then - count = count + 1 - self.values[count] = val + local pre_index = self.values_hash[key] + if pre_index then + local pre_val = self.values[pre_index] + if pre_val and pre_val.clean_handlers then + for _, clean_handler in ipairs(pre_val.clean_handlers) do + clean_handler(pre_val) + end + pre_val.clean_handlers = nil + end + + if res.value then + res.value.id = key + self.values[pre_index] = res + res.clean_handlers = {} + + else + self.sync_times = self.sync_times + 1 + self.values[pre_index] = false end + + elseif res.value then + res.clean_handlers = {} + insert_tab(self.values, res) + self.values_hash[key] = #self.values + res.value.id = key end - for i = 1, count do - key = short_key(self, self.values[i].key) - self.values_hash[key] = i + -- avoid space waste + -- todo: need to cover this path, it is important. + if self.sync_times > 100 then + local count = 0 + for i = 1, #self.values do + local val = self.values[i] + self.values[i] = nil + if val then + count = count + 1 + self.values[count] = val + end + end + + for i = 1, count do + key = short_key(self, self.values[i].key) + self.values_hash[key] = i + end + self.sync_times = 0 end - self.sync_times = 0 + + self.conf_version = self.conf_version + 1 end - self.conf_version = self.conf_version + 1 return self.values end @@ -380,6 +406,12 @@ local function _automatic_fetch(premature, self) return end + local etcd_cli, _, err = etcd.new(self.etcd_conf) + if not etcd_cli then + error("failed to start a etcd instance: " .. err) + end + self.etcd_cli = etcd_cli + local i = 0 while not exiting() and self.running and i <= 32 do i = i + 1 @@ -430,12 +462,7 @@ function _M.new(key, opts) etcd_conf.http_host = etcd_conf.host etcd_conf.host = nil etcd_conf.prefix = nil - - local etcd_cli - etcd_cli, err = etcd.new(etcd_conf) - if not etcd_cli then - return nil, err - end + etcd_conf.protocol = "v3" local automatic = opts and opts.automatic local item_schema = opts and opts.item_schema @@ -443,7 +470,8 @@ function _M.new(key, opts) local timeout = opts and opts.timeout local obj = setmetatable({ - etcd_cli = etcd_cli, + etcd_cli = nil, + etcd_conf = etcd_conf, key = key and prefix .. key, automatic = automatic, item_schema = item_schema, @@ -453,7 +481,7 @@ function _M.new(key, opts) values = nil, need_reload = true, routes_hash = nil, - prev_index = nil, + prev_index = 0, last_err = nil, last_err_time = nil, timeout = timeout, @@ -466,6 +494,13 @@ function _M.new(key, opts) end ngx_timer_at(0, _automatic_fetch, obj) + + else + local etcd_cli, _, err = etcd.new(etcd_conf) + if not etcd_cli then + return nil, "failed to start a etcd instance: " .. err + end + obj.etcd_cli = etcd_cli end if key then @@ -505,6 +540,7 @@ local function read_etcd_version(etcd_cli) return body end + function _M.server_version(self) if not self.running then return nil, "stoped" diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua index 818c99b95d1c..1c781f8e812a 100644 --- a/apisix/core/etcd.lua +++ b/apisix/core/etcd.lua @@ -15,10 +15,13 @@ -- limitations under the License. -- local fetch_local_conf = require("apisix.core.config_local").local_conf -local etcd = require("resty.etcd") -local clone_tab = require("table.clone") +local etcd = require("resty.etcd") +local clone_tab = require("table.clone") +local ipairs = ipairs +local string = string +local tonumber = tonumber -local _M = {version = 0.1} +local _M = {} local function new() @@ -32,6 +35,7 @@ local function new() etcd_conf.http_host = etcd_conf.host etcd_conf.host = nil etcd_conf.prefix = nil + etcd_conf.protocol = "v3" local etcd_cli etcd_cli, err = etcd.new(etcd_conf) @@ -44,24 +48,134 @@ end _M.new = new +local function kvs_to_node(kvs) + local node = {} + node.key = kvs.key + node.value = kvs.value + node.createdIndex = tonumber(kvs.create_revision) + node.modifiedIndex = tonumber(kvs.mod_revision) + return node +end + +local function kvs_to_nodes(res) + res.body.node.dir = true + res.body.node.nodes = {} + for i=2, #res.body.kvs do + res.body.node.nodes[i-1] = kvs_to_node(res.body.kvs[i]) + end + return res +end + + +local function not_found(res) + res.body.message = "Key not found" + res.reason = "Not found" + res.status = 404 + return res +end + + +function _M.get_format(res, realkey) + if res.body.error == "etcdserver: user name is empty" then + return nil, "insufficient credentials code: 401" + end + + res.headers["X-Etcd-Index"] = res.body.header.revision + + if not res.body.kvs then + return not_found(res) + end + res.body.action = "get" + + -- In etcd v2, the direct key asked for is `node`, others which under this dir are `nodes` + -- While in v3, this structure is flatten and all keys related the key asked for are `kvs` + res.body.node = kvs_to_node(res.body.kvs[1]) + if not res.body.kvs[1].value then + -- remove last "/" when necesary + if string.sub(res.body.node.key, -1, -1) == "/" then + res.body.node.key = string.sub(res.body.node.key, 1, #res.body.node.key-1) + end + res = kvs_to_nodes(res) + end + + res.body.kvs = nil + return res +end + + +function _M.watch_format(v3res) + local v2res = {} + v2res.headers = { + ["X-Etcd-Index"] = v3res.result.header.revision + } + v2res.body = { + node = {} + } + for i, event in ipairs(v3res.result.events) do + v2res.body.node[i] = kvs_to_node(event.kv) + if event.type == "DELETE" then + v2res.body.action = "delete" + end + end + + return v2res +end + + function _M.get(key) local etcd_cli, prefix, err = new() if not etcd_cli then return nil, err end - return etcd_cli:get(prefix .. key) + -- in etcd v2, get could implicitly turn into readdir + -- while in v3, we need to do it explicitly + local res, err = etcd_cli:readdir(prefix .. key) + if not res then + return nil, err + end + + return _M.get_format(res, prefix .. key) end -function _M.set(key, value, ttl) +local function set(key, value, ttl) local etcd_cli, prefix, err = new() if not etcd_cli then return nil, err end - return etcd_cli:set(prefix .. key, value, ttl) + -- lease substitute ttl in v3 + local res, err + if ttl then + local data, grant_err = etcd_cli:grant(tonumber(ttl)) + if not data then + return nil, grant_err + end + res, err = etcd_cli:set(prefix .. key, value, {prev_kv = true, lease = data.body.ID}) + else + res, err = etcd_cli:set(prefix .. key, value, {prev_kv = true}) + end + if not res then + return nil, err + end + + res.headers["X-Etcd-Index"] = res.body.header.revision + + -- etcd v3 set would not return kv info + res.body.action = "set" + res.body.node = {} + res.body.node.key = prefix .. key + res.body.node.value = value + res.status = 201 + if res.body.prev_kv then + res.status = 200 + res.body.prev_kv = nil + end + + return res, nil end +_M.set = set function _M.push(key, value, ttl) @@ -70,7 +184,22 @@ function _M.push(key, value, ttl) return nil, err end - return etcd_cli:push(prefix .. key, value, ttl) + local res, err = etcd_cli:readdir(prefix .. key) + if not res then + return nil, err + end + + -- manually add suffix + local index = res.body.header.revision + index = string.format("%020d", index) + + res, err = set(key .. "/" .. index, value, ttl) + if not res then + return nil, err + end + + res.body.action = "create" + return res, nil end @@ -80,11 +209,28 @@ function _M.delete(key) return nil, err end - return etcd_cli:delete(prefix .. key) + local res, err = etcd_cli:delete(prefix .. key) + + if not res then + return nil, err + end + + res.headers["X-Etcd-Index"] = res.body.header.revision + + if not res.body.deleted then + return not_found(res), nil + end + + -- etcd v3 set would not return kv info + res.body.action = "delete" + res.body.node = {} + res.body.key = prefix .. key + + return res, nil end -function _M.server_version(key) +function _M.server_version() local etcd_cli, err = new() if not etcd_cli then return nil, err diff --git a/bin/apisix b/bin/apisix index d609c21a4edf..96e6b6aae851 100755 --- a/bin/apisix +++ b/bin/apisix @@ -605,6 +605,14 @@ local function merge_conf(base, new_tab) return base end +local function str_split(str, sep) + local t = {} + for s in str:gmatch("([^"..sep.."]+)") do + table.insert(t, s) + end + return t +end + local function read_yaml_conf() local profile = require("apisix.core.profile") @@ -682,7 +690,7 @@ local function split(self, sep) return fields end -local function check_or_version(cur_ver_s, need_ver_s) +local function check_version(cur_ver_s, need_ver_s) local cur_vers = split(cur_ver_s, [[.]]) local need_vers = split(need_ver_s, [[.]]) local len = math.max(#cur_vers, #need_vers) @@ -842,7 +850,7 @@ local function init() end local need_ver = "1.15.8" - if not check_or_version(op_ver, need_ver) then + if not check_version(op_ver, need_ver) then io.stderr:write("openresty version must >=", need_ver, " current ", op_ver, "\n") return end @@ -879,35 +887,35 @@ local function init_etcd(show_output) local host_count = #(yaml_conf.etcd.host) - -- check whether the user has enabled etcd v2 protocol + local etcd_ok = false for index, host in ipairs(yaml_conf.etcd.host) do - uri = host .. "/v2/keys" - local cmd = "curl -i -m ".. timeout * 2 .. " -o /dev/null -s -w %{http_code} " .. uri + -- check if etcd version above 3.4 + cmd = "curl " .. host .. "/version 2>&1" local res = excute_cmd(cmd) - if res == "404" then - io.stderr:write(string.format("failed: please make sure that you have enabled the v2 protocol of etcd on %s.\n", host)) + local op_ver = str_split(res, "\"")[4] + local need_ver = "3.4.0" + if not check_version(op_ver, need_ver) then + io.stderr:write("etcd version must >=", need_ver, " current ", op_ver, "\n") return end - end - - local etcd_ok = false - for index, host in ipairs(yaml_conf.etcd.host) do local is_success = true - uri = host .. "/v2/keys" .. (etcd_conf.prefix or "") for _, dir_name in ipairs({"/routes", "/upstreams", "/services", "/plugins", "/consumers", "/node_status", "/ssl", "/global_rules", "/stream_routes", "/proto"}) do - local cmd = "curl " .. uri .. dir_name - .. "?prev_exist=false -X PUT -d dir=true " - .. "--connect-timeout " .. timeout + local key = (etcd_conf.prefix or "") .. dir_name .. "/" + + local base64_encode = require("base64").encode + local uri = host .. "/v3/kv/put" + local post_json = '{"value":"' .. base64_encode("init_dir") .. '", "key":"' .. base64_encode(key) .. '"}' + cmd = "curl " .. uri .. " -X POST -d '" .. post_json + .. "' --connect-timeout " .. timeout .. " --max-time " .. timeout * 2 .. " --retry 1 2>&1" local res = excute_cmd(cmd) - if not res:find("index", 1, true) - and not res:find("createdIndex", 1, true) then + if (etcd_version == "v3" and not res:find("OK", 1, true)) then is_success = false if (index == host_count) then error(cmd .. "\n" .. res) diff --git a/doc/architecture-design.md b/doc/architecture-design.md index 2dba7666bd77..068da16c8dcd 100644 --- a/doc/architecture-design.md +++ b/doc/architecture-design.md @@ -598,7 +598,7 @@ HTTP/1.1 403 [Plugin](#Plugin) just can be binded to [Service](#Service) or [Route](#Route), if we want a [Plugin](#Plugin) work on all requests, how to do it? We can register a global [Plugin](#Plugin) with `GlobalRule`: -```shell +​```shell curl -X PUT \ https://{apisix_listen_address}/apisix/admin/global_rules/1 \ -H 'Content-Type: application/json' \ diff --git a/doc/install-dependencies.md b/doc/install-dependencies.md index c811bedce37d..99ec723d19a6 100644 --- a/doc/install-dependencies.md +++ b/doc/install-dependencies.md @@ -28,12 +28,12 @@ Note ==== -- Apache APISIX currently only supports the v2 protocol storage to etcd, but the latest version of etcd (starting with 3.4) has turned off the v2 protocol by default. +- Apache APISIX would not support the v2 protocol storage to etcd anymore. If etcd version is below 3.4, the default protocol is still v2 and you need to turn on v3 protocol mannually. -You need to add `--enable-v2=true` to the startup parameter to enable the v2 protocol. The development of the v3 protocol supporting etcd has begun and will soon be available. +You need to add `ETCDCTL_API=3` to the environmental variables to enable the v3 protocol. ```shell -etcd --enable-v2=true & +export ETCDCTL_API=3 ``` - If you want use Tengine instead of OpenResty, please take a look at this installation step script [Install Tengine at Ubuntu](../.travis/linux_tengine_runner.sh). @@ -70,7 +70,7 @@ sudo yum-config-manager --add-repo https://openresty.org/package/fedora/openrest sudo yum install -y etcd openresty curl git gcc luarocks lua-devel # start etcd server -sudo etcd --enable-v2=true & +sudo etcd & ``` Ubuntu 16.04 & 18.04 @@ -127,6 +127,6 @@ Mac OSX # install OpenResty, etcd and some compilation tools brew install openresty/brew/openresty etcd luarocks curl git -# start etcd server with v2 protocol -etcd --enable-v2=true & +# start etcd server +etcd & ``` diff --git a/doc/zh-cn/install-dependencies.md b/doc/zh-cn/install-dependencies.md index f3c12f9c4866..967c429ef4fb 100644 --- a/doc/zh-cn/install-dependencies.md +++ b/doc/zh-cn/install-dependencies.md @@ -27,10 +27,10 @@ 注意 ==== -- Apache APISIX 目前只支持 `v2` 版本的 etcd,但是最新版的 etcd (从 3.4 起)已经默认关闭了 `v2` 版本的功能。所以你需要添加启动参数 `--enable-v2=true` 来开启 `v2` 的功能,目前对 `v3` etcd 的开发工作已经启动,不久后便可投入使用。 +- Apache APISIX 不再支持 `v2` 版本的 etcd。在 etcd 版本低于 3.4 时,默认 API 协议仍为 v2,因此需要添加 `ETCDCTL_API=3` 至环境变量以启动 v3 协议。 ```shell -etcd --enable-v2=true & +export ETCDCTL_API=3 ``` - 如果你要想使用 Tengine 替代 OpenResty,请参考 [Install Tengine at Ubuntu](../../.travis/linux_tengine_runner.sh)。 @@ -59,15 +59,15 @@ Fedora 31 & 32 ============== ```shell -# add OpenResty source +# 添加 OpenResty 源 sudo yum install yum-utils sudo yum-config-manager --add-repo https://openresty.org/package/fedora/openresty.repo -# install OpenResty, etcd and some compilation tools +# 安装 OpenResty, etcd 和 编译工具 sudo yum install -y etcd openresty curl git gcc luarocks lua-devel -# start etcd server -sudo etcd --enable-v2=true & +# 开启 etcd server +sudo etcd & ``` Ubuntu 16.04 & 18.04 @@ -124,6 +124,6 @@ Mac OSX # 安装 OpenResty, etcd 和 编译工具 brew install openresty/brew/openresty etcd luarocks curl git -# 开启 etcd server 并启用 v2 的功能 -etcd --enable-v2=true & +# 开启 etcd server +etcd & ``` diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index 3028097006b9..000988a28eaa 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -32,7 +32,7 @@ description = { dependencies = { "lua-resty-template = 1.9", - "lua-resty-etcd = 1.0", + "lua-resty-etcd = 1.1", "lua-resty-balancer = 0.02rc5", "lua-resty-ngxvar = 0.5", "lua-resty-jit-uuid = 0.0.7", @@ -52,6 +52,7 @@ dependencies = { "lua-resty-kafka = 0.07", "lua-resty-logger-socket = 2.0-0", "skywalking-nginx-lua-plugin = 1.0-0", + "base64 = 1.5-2" } build = { diff --git a/t/admin/routes.t b/t/admin/routes.t index 9d98b25d67ab..d9905c33168c 100644 --- a/t/admin/routes.t +++ b/t/admin/routes.t @@ -1557,7 +1557,8 @@ location /t { ngx.say("code: ", code) ngx.say(body) - ngx.sleep(2) + -- etcd v3 would still get the value at 2s, don't know why yet + ngx.sleep(2.5) -- get again code, body, res = t('/apisix/admin/routes/1', ngx.HTTP_GET) @@ -1610,7 +1611,7 @@ location /t { end ngx.say("[push] succ: ", body) - ngx.sleep(2) + ngx.sleep(2.5) local id = string.sub(res.node.key, #"/apisix/routes/" + 1) code, body = t('/apisix/admin/routes/' .. id, ngx.HTTP_GET) diff --git a/t/core/etcd-auth-fail.t b/t/core/etcd-auth-fail.t index dfeaffee178f..5b3d70482c51 100644 --- a/t/core/etcd-auth-fail.t +++ b/t/core/etcd-auth-fail.t @@ -26,16 +26,19 @@ no_root_location(); log_level("info"); # Authentication is enabled at etcd and credentials are set -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY user add root:5tHkHhYkjr6cQY'); -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY auth enable'); -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY role revoke --path "/*" -rw guest'); +system('etcdctl --endpoints="http://127.0.0.1:2379" user add root:5tHkHhYkjr6cQY'); +system('etcdctl --endpoints="http://127.0.0.1:2379" role add root'); +system('etcdctl --endpoints="http://127.0.0.1:2379" user grant-role root root'); +system('etcdctl --endpoints="http://127.0.0.1:2379" role list'); +system('etcdctl --endpoints="http://127.0.0.1:2379" user user list'); +system('etcdctl --endpoints="http://127.0.0.1:2379" auth enable'); run_tests; -# Authentication is disabled at etcd & guest access is granted -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY auth disable'); -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY role grant --path "/*" -rw guest'); - +# Authentication is disabled at etcd +system('etcdctl --endpoints="http://127.0.0.1:2379" --user root:5tHkHhYkjr6cQY auth disable'); +system('etcdctl --endpoints="http://127.0.0.1:2379" user delete root'); +system('etcdctl --endpoints="http://127.0.0.1:2379" role delete root'); __DATA__ @@ -52,5 +55,6 @@ __DATA__ } --- request GET /t ---- response_body -insufficient credentials code: 401 +--- error_code: 500 +--- error_log eval +qr /insufficient credentials code: 401/ diff --git a/t/core/etcd-auth.t b/t/core/etcd-auth.t index 3051a68ffbde..794013aaa864 100644 --- a/t/core/etcd-auth.t +++ b/t/core/etcd-auth.t @@ -26,15 +26,20 @@ no_root_location(); log_level("info"); # Authentication is enabled at etcd and credentials are set -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY user add root:5tHkHhYkjr6cQY'); -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY auth enable'); -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY role revoke --path "/*" -rw guest'); +system('etcdctl --endpoints="http://127.0.0.1:2379" user add root:5tHkHhYkjr6cQY'); +system('etcdctl --endpoints="http://127.0.0.1:2379" role add root'); +system('etcdctl --endpoints="http://127.0.0.1:2379" user grant-role root root'); +system('etcdctl --endpoints="http://127.0.0.1:2379" role list'); +system('etcdctl --endpoints="http://127.0.0.1:2379" user user list'); +system('etcdctl --endpoints="http://127.0.0.1:2379" auth enable'); run_tests; -# Authentication is disabled at etcd & guest access is granted -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY auth disable'); -system('etcdctl --endpoints="http://127.0.0.1:2379" -u root:5tHkHhYkjr6cQY role grant --path "/*" -rw guest'); +# Authentication is disabled at etcd +system('etcdctl --endpoints="http://127.0.0.1:2379" --user root:5tHkHhYkjr6cQY auth disable'); +system('etcdctl --endpoints="http://127.0.0.1:2379" user delete root'); +system('etcdctl --endpoints="http://127.0.0.1:2379" role delete root'); + __DATA__ diff --git a/t/core/etcd-sync.t b/t/core/etcd-sync.t index ef0ea57cdf5b..9dd1250749bf 100644 --- a/t/core/etcd-sync.t +++ b/t/core/etcd-sync.t @@ -47,6 +47,8 @@ __DATA__ if new_idx > idx then ngx.say("prev_index updated") + else + ngx.say("prev_index not update") end } } diff --git a/t/core/etcd.t b/t/core/etcd.t new file mode 100644 index 000000000000..197f7e93d2bc --- /dev/null +++ b/t/core/etcd.t @@ -0,0 +1,335 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("info"); + +run_tests; + +__DATA__ + +=== TEST 1: delete test data if exists +--- config + location /delete { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_DELETE) + ngx.status = code + ngx.say(body) + } + } +--- request +GET /delete +--- no_error_log +[error] +--- ignore_response + + + +=== TEST 2: (add + update + delete) *2 (same uri) +--- config + location /add { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "host": "foo.com", + "uri": "/hello" + }]], + nil + ) + ngx.status = code + ngx.say(body) + } + } + location /update { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "upstream": { + "nodes": { + "127.0.0.1:1980": 2 + }, + "type": "roundrobin" + }, + "host": "foo.com", + "uri": "/hello" + }]], + nil + ) + ngx.status = code + ngx.say(body) + } + } + location /delete { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_DELETE) + ngx.status = code + ngx.say(body) + } + } +--- pipelined_requests eval +["GET /add", "GET /hello", "GET /update", "GET /hello", "GET /delete", "GET /hello", +"GET /add", "GET /hello", "GET /update", "GET /hello", "GET /delete", "GET /hello"] +--- more_headers +Host: foo.com +--- error_code eval +[201, 200, 200, 200, 200, 404, 201, 200, 200, 200, 200, 404] +--- response_body eval +["passed\n", "hello world\n", "passed\n", "hello world\n", "passed\n", "{\"error_msg\":\"failed to match any routes\"}\n", +"passed\n", "hello world\n", "passed\n", "hello world\n", "passed\n", "{\"error_msg\":\"failed to match any routes\"}\n"] +--- no_error_log +[error] +--- timeout: 5 + + + +=== TEST 3: add + update + delete + add + update + delete (different uris) +--- config + location /add { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "host": "foo.com", + "uri": "/hello" + }]], + nil + ) + ngx.status = code + ngx.say(body) + } + } + location /update { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + + "upstream": { + "nodes": { + "127.0.0.1:1980": 2 + }, + "type": "roundrobin" + }, + "host": "foo.com", + "uri": "/status" + }]], + nil + ) + ngx.status = code + ngx.say(body) + } + } + location /delete { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_DELETE) + ngx.status = code + ngx.say(body) + } + } + location /add2 { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "host": "foo.com", + "uri": "/hello_" + }]], + nil + ) + ngx.sleep(1) + ngx.status = code + ngx.say(body) + } + } + location /update2 { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + + "upstream": { + "nodes": { + "127.0.0.1:1980": 2 + }, + "type": "roundrobin" + }, + "host": "foo.com", + "uri": "/hello1" + }]], + nil + ) + ngx.status = code + ngx.say(body) + } + } + location /delete2 { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_DELETE) + ngx.status = code + ngx.say(body) + } + } +--- pipelined_requests eval +["GET /add", "GET /hello", "GET /update", "GET /hello", "GET /status", "GET /delete", "GET /status", +"GET /add2", "GET /hello_", "GET /update2", "GET /hello_", "GET /hello1", "GET /delete", "GET /hello1"] +--- more_headers +Host: foo.com +--- error_code eval +[201, 200, 200, 404, 200, 200, 404, 201, 200, 200, 404, 200, 200, 404] +--- response_body eval +["passed\n", "hello world\n", "passed\n", "{\"error_msg\":\"failed to match any routes\"}\n", "ok\n", "passed\n", "{\"error_msg\":\"failed to match any routes\"}\n", +"passed\n", "hello world\n", "passed\n", "{\"error_msg\":\"failed to match any routes\"}\n", "hello1 world\n", "passed\n", "{\"error_msg\":\"failed to match any routes\"}\n"] +--- no_error_log +[error] +--- timeout: 5 + + + +=== TEST 4: add*50 + update*50 + delete*50 +--- config + location /add { + content_by_lua_block { + local t = require("lib.test_admin").test + local path = "" + local code, body + for i = 1, 25 do + path = '/apisix/admin/routes/' .. tostring(i) + code, body = t(path, + ngx.HTTP_PUT, + string.format('{"upstream": {"nodes": {"127.0.0.1:1980": 1},"type": "roundrobin"},"host": "foo.com","uri": "/print_uri_%s"}', tostring(i)), + nil + ) + end + ngx.sleep(2) + ngx.status = code + ngx.say(body) + } + } + location /add2 { + content_by_lua_block { + local t = require("lib.test_admin").test + local path = "" + local code, body + for i = 26, 50 do + path = '/apisix/admin/routes/' .. tostring(i) + code, body = t(path, + ngx.HTTP_PUT, + string.format('{"upstream": {"nodes": {"127.0.0.1:1980": 1},"type": "roundrobin"},"host": "foo.com","uri": "/print_uri_%s"}', tostring(i)), + nil + ) + end + ngx.sleep(2) + ngx.status = code + ngx.say(body) + } + } + location /update { + content_by_lua_block { + local t = require("lib.test_admin").test + local path = "" + local code, body + for i = 1, 25 do + path = '/apisix/admin/routes/' .. tostring(i) + code, body = t(path, + ngx.HTTP_PUT, + string.format('{"upstream": {"nodes": {"127.0.0.1:1980": 1},"type": "roundrobin"},"host": "foo.com","uri": "/print_uri_%s"}', tostring(i)), + nil + ) + end + ngx.sleep(2) + ngx.status = code + ngx.say(body) + } + } + location /update2 { + content_by_lua_block { + local t = require("lib.test_admin").test + local path = "" + local code, body + for i = 26, 50 do + path = '/apisix/admin/routes/' .. tostring(i) + code, body = t(path, + ngx.HTTP_PUT, + string.format('{"upstream": {"nodes": {"127.0.0.1:1980": 1},"type": "roundrobin"},"host": "foo.com","uri": "/print_uri_%s"}', tostring(i)), + nil + ) + end + ngx.sleep(2) + ngx.status = code + ngx.say(body) + } + } + location /delete { + content_by_lua_block { + local t = require("lib.test_admin").test + local path = "" + local code, body + for i = 1, 50 do + path = '/apisix/admin/routes/' .. tostring(i) + code, body = t(path, ngx.HTTP_DELETE) + end + ngx.status = code + ngx.say(body) + } + } +--- pipelined_requests eval +["GET /add", "GET /print_uri_20", "GET /add2", "GET /print_uri_36", "GET /update", "GET /print_uri_12", "GET /delete", "GET /print_uri_12"] +--- more_headers +Host: foo.com +--- error_code eval +[201, 200, 201, 200, 200, 200, 200, 404] +--- response_body eval +["passed\n", "/print_uri_20\n", "passed\n", "/print_uri_36\n", "passed\n", "/print_uri_12\n", "passed\n", "{\"error_msg\":\"failed to match any routes\"}\n"] +--- no_error_log +[error] +--- timeout: 20 diff --git a/t/lib/server.lua b/t/lib/server.lua index b9c8db41cdcd..68ba81a99d70 100644 --- a/t/lib/server.lua +++ b/t/lib/server.lua @@ -250,6 +250,13 @@ function _M.websocket_handshake() end _M.websocket_handshake_route = _M.websocket_handshake +local function print_uri() + ngx.say(ngx.var.uri) +end +for i = 1, 100 do + _M["print_uri_" .. i] = print_uri +end + function _M.go() local action = string.sub(ngx.var.uri, 2) action = string.gsub(action, "[/\\.]", "_") diff --git a/utils/install-etcd.sh b/utils/install-etcd.sh new file mode 100755 index 000000000000..83b3d153758d --- /dev/null +++ b/utils/install-etcd.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +wget https://github.com/etcd-io/etcd/releases/download/v3.4.0/etcd-v3.4.0-linux-amd64.tar.gz +tar xf etcd-v3.4.0-linux-amd64.tar.gz +sudo cp etcd-v3.4.0-linux-amd64/etcd /usr/local/bin/ +sudo cp etcd-v3.4.0-linux-amd64/etcdctl /usr/local/bin/ +rm -rf etcd-v3.4.0-linux-amd64