From aaa2642cec6d382324f11074c0fbedcf821c88a6 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Mon, 23 Aug 2021 22:46:45 +0800 Subject: [PATCH 01/73] feat: add k8s discovery module Signed-off-by: adugeek --- apisix/discovery/k8s.lua | 472 ++++++++++++++++++++++++++ ci/install-ext-services-via-docker.sh | 62 ++++ t/APISIX.pm | 2 + t/discovery/k8s.t | 167 +++++++++ 4 files changed, 703 insertions(+) create mode 100644 apisix/discovery/k8s.lua create mode 100644 t/discovery/k8s.t diff --git a/apisix/discovery/k8s.lua b/apisix/discovery/k8s.lua new file mode 100644 index 000000000000..0fbb70296937 --- /dev/null +++ b/apisix/discovery/k8s.lua @@ -0,0 +1,472 @@ +local ngx = ngx +local ipairs = ipairs +local string = string +local tonumber = tonumber +local tostring = tostring +local math = math +local os = os +local process = require("ngx.process") +local core = require("apisix.core") +local util = require("apisix.cli.util") +local http = require("resty.http") +local endpoints_shared = ngx.shared.discovery + +local apiserver_host = "" +local apiserver_port = "" +local apiserver_token = "" + +local default_weight = 50 + +local endpoint_lrucache = core.lrucache.new({ + ttl = 300, + count = 1024 +}) + +local endpoint_cache = {} +local empty_table = {} +local pending_resources + +local function sort_by_ip(a, b) + return a.ip < b.ip +end + +local function on_endpoint_added(endpoint) + local subsets = endpoint.subsets + if subsets == nil or #subsets == 0 then + return + end + + local subset = subsets[1] + + local addresses = subset.addresses + if addresses == nil or #addresses == 0 then + return + end + + local ports = subset.ports + if ports == nil or #ports == 0 then + return + end + + core.table.sort(addresses, sort_by_ip) + core.table.clear(endpoint_cache) + for _, port in ipairs(ports) do + local nodes = core.table.new(#addresses, 0) + for i, address in ipairs(addresses) do + nodes[i] = { + host = address.ip, + port = port.port, + weight = default_weight + } + end + if port.name then + endpoint_cache[port.name] = nodes + else + endpoint_cache[tostring(port.port)] = nodes + end + end + + local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name + local endpoint_content = core.json.encode(endpoint_cache, true) + local endpoint_version = ngx.crc32_long(endpoint_content) + + local _, err + _, err = endpoints_shared:safe_set(endpoint_key .. "#version", endpoint_version) + if err then + core.log.emerg("set endpoint version into discovery DICT failed ,", err) + return + end + endpoints_shared:safe_set(endpoint_key, endpoint_content) + if err then + core.log.emerg("set endpoint into discovery DICT failed ,", err) + endpoints_shared:delete(endpoint_key .. "#version") + end +end + +local function on_endpoint_deleted(endpoint) + local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name + endpoints_shared:delete(endpoint_key .. "#version") + endpoints_shared:delete(endpoint_key) +end + +local function on_endpoint_modified(endpoint) + if endpoint.subsets == nil or #endpoint.subsets == 0 then + return on_endpoint_deleted(endpoint) + end + local subset = endpoint.subsets[1] + + if subset.addresses == nil or #subset.addresses == 0 then + return on_endpoint_deleted(endpoint) + end + local addresses = subset.addresses + + if subset.ports == nil or #subset.ports == 0 then + return on_endpoint_deleted(endpoint) + end + local ports = subset.ports + + core.table.sort(addresses, sort_by_ip) + core.table.clear(endpoint_cache) + for _, port in ipairs(ports) do + local nodes = core.table.new(#addresses, 0) + for i, address in ipairs(addresses) do + nodes[i] = { + host = address.ip, + port = port.port, + weight = default_weight + } + end + if port.name then + endpoint_cache[port.name] = nodes + else + endpoint_cache[tostring(port.port)] = nodes + end + end + + local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name + local endpoint_content = core.json.encode(endpoint_cache, true) + local endpoint_version = ngx.crc32_long(endpoint_content) + + local _, err + _, err = endpoints_shared:safe_set(endpoint_key .. "#version", endpoint_version) + if err then + core.log.emerg("set endpoint version into discovery DICT failed ,", err) + return + end + endpoints_shared:safe_set(endpoint_key, endpoint_content) + if err then + core.log.emerg("set endpoint into discovery DICT failed ,", err) + endpoints_shared:delete(endpoint_key .. "#version") + end +end + +local function list_resource(httpc, resource, continue) + httpc:set_timeouts(2000, 2000, 3000) + local res, err = httpc:request({ + path = resource:list_path(), + query = resource:list_query(continue), + headers = { + ["Host"] = string.format("%s:%s", apiserver_host, apiserver_port), + ["Authorization"] = string.format("Bearer %s", apiserver_token), + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + }) + + if not res then + return false, "RequestError", err or "" + end + + if res.status ~= 200 then + return false, res.reason, res:read_body() or "" + end + + local body, err = res:read_body() + if err then + return false, "ReadBodyError", err + end + + local data, _ = core.json.decode(body) + if not data or data.kind ~= resource.listKind then + return false, "UnexpectedBody", body + end + + resource.newest_resource_version = data.metadata.resourceVersion + + for _, item in ipairs(data.items or empty_table) do + resource:event_dispatch("ADDED", item) + end + + if data.metadata.continue ~= nil and data.metadata.continue ~= "" then + list_resource(httpc, resource, data.metadata.continue) + end + + return true, "Success", "" +end + +local function watch_resource(httpc, resource) + math.randomseed(process.get_master_pid()) + local watch_seconds = 1800 + math.random(60, 1200) + local allowance_seconds = 120 + httpc:set_timeouts(2000, 3000, (watch_seconds + allowance_seconds) * 1000) + local res, err = httpc:request({ + path = resource:watch_path(), + query = resource:watch_query(watch_seconds), + headers = { + ["Host"] = string.format("%s:%s", apiserver_host, apiserver_port), + ["Authorization"] = string.format("Bearer %s", apiserver_token), + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + }) + + if err then + return false, "RequestError", err + end + + if res.status ~= 200 then + return false, res.reason, res:read_body() or "" + end + + local remainder_body = "" + local body = "" + local reader = res.body_reader + local gmatch_iterator; + local captures; + local captured_size = 0 + while true do + + body, err = reader() + if err then + return false, "ReadBodyError", err + end + + if not body then + break + end + + if #remainder_body ~= 0 then + body = remainder_body .. body + end + + gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jiao") + if not gmatch_iterator then + return false, "GmatchError", err + end + + while true do + captures, err = gmatch_iterator() + if err then + return false, "GmatchError", err + end + if not captures then + break + end + captured_size = captured_size + #captures[0] + local v, _ = core.json.decode(captures[0]) + if not v or not v.object or v.object.kind ~= resource.kind then + return false, "UnexpectedBody", captures[0] + end + + resource.newest_resource_version = v.object.metadata.resource_version + if v.type ~= "BOOKMARK" then + resource:event_dispatch(v.type, v.object) + end + end + + if captured_size == #body then + remainder_body = "" + elseif captured_size == 0 then + remainder_body = body + else + remainder_body = string.sub(body, captured_size + 1) + end + end + watch_resource(httpc, resource) +end + +local function fetch_resource(resource) + while true do + local ok = false + local reason, message = "", "" + local retry_interval = 0 + repeat + local httpc = http.new() + resource.watch_state = "connecting" + core.log.info("begin to connect ", apiserver_host, ":", apiserver_port) + ok, message = httpc:connect({ + scheme = "https", + host = apiserver_host, + port = tonumber(apiserver_port), + ssl_verify = false + }) + if not ok then + resource.watch_state = "connecting" + core.log.error("connect apiserver failed , apiserver_host: ", apiserver_host, "apiserver_port", + apiserver_port, "message : ", message) + retry_interval = 100 + break + end + + core.log.info("begin to list ", resource.plural) + resource.watch_state = "listing" + resource:pre_list_callback() + ok, reason, message = list_resource(httpc, resource, nil) + if not ok then + resource.watch_state = "list failed" + core.log.error("list failed , resource: ", resource.plural, " reason: ", reason, "message : ", message) + retry_interval = 100 + break + end + resource.watch_state = "list finished" + resource:post_list_callback() + + core.log.info("begin to watch ", resource.plural) + resource.watch_state = "watching" + ok, reason, message = watch_resource(httpc, resource) + if not ok then + resource.watch_state = "watch failed" + core.log.error("watch failed, resource: ", resource.plural, " reason: ", reason, "message : ", message) + retry_interval = 0 + break + end + resource.watch_state = "watch finished" + retry_interval = 0 + until true + if retry_interval ~= 0 then + ngx.sleep(retry_interval) + end + end +end + +local function fetch() + local threads = core.table.new(#pending_resources, 0) + for i, resource in ipairs(pending_resources) do + threads[i] = ngx.thread.spawn(fetch_resource, resource) + end + for _, thread in ipairs(threads) do + ngx.thread.wait(thread) + end +end + +local function create_endpoint_lrucache(endpoint_key, endpoint_port) + local endpoint_content, _, _ = endpoints_shared:get_stale(endpoint_key) + if not endpoint_content then + core.log.emerg("get empty endpoint content from discovery DICT,this should not happen ", endpoint_key) + return nil + end + + local endpoint, _ = core.json.decode(endpoint_content) + if not endpoint then + core.log.emerg("decode endpoint content failed, this should not happen, content : ", endpoint_content) + end + + return endpoint[endpoint_port] +end + +local _M = { + version = 0.01 +} + +function _M.nodes(service_name) + local pattern = "^(.*):(.*)$" + local match, _ = ngx.re.match(service_name, pattern, "jiao") + if not match then + core.log.info("get unexpected upstream service_name:　", service_name) + return nil + end + + local endpoint_key = match[1] + local endpoint_port = match[2] + local endpoint_version, _, _ = endpoints_shared:get_stale(endpoint_key .. "#version") + if not endpoint_version then + core.log.info("get empty endpoint version from discovery DICT ", endpoint_key) + return nil + end + return endpoint_lrucache(service_name, endpoint_version, create_endpoint_lrucache, endpoint_key, endpoint_port) +end + +local function fill_pending_resources() + pending_resources = core.table.new(1, 0) + + pending_resources[1] = { + + group = "", + version = "v1", + kind = "Endpoints", + listKind = "EndpointsList", + plural = "endpoints", + + newest_resource_version = "", + + label_selector = function() + return "" + end, + + list_path = function(self) + return "/api/v1/endpoints" + end, + + list_query = function(self, continue) + if continue == nil or continue == "" then + return "limit=32" + else + return "limit=32&continue=" .. continue + end + end, + + watch_path = function(self) + return "/api/v1/endpoints" + end, + + watch_query = function(self, timeout) + return string.format("watch=true&allowWatchBookmarks=true&timeoutSeconds=%d&resourceVersion=%s", timeout, + self.newest_resource_version) + end, + + pre_list_callback = function(self) + self.newest_resource_version = "0" + endpoints_shared:flush_all() + end, + + post_list_callback = function(self) + endpoints_shared:flush_expired() + end, + + added_callback = function(self, object) + on_endpoint_added(object) + end, + + modified_callback = function(self, object) + on_endpoint_modified(object) + end, + + deleted_callback = function(self, object) + on_endpoint_deleted(object) + end, + + event_dispatch = function(self, event, object) + if event == "DELETED" or object.deletionTimestamp ~= nil then + self:deleted_callback(object) + return + end + + if event == "ADDED" then + self:added_callback(object) + elseif event == "MODIFIED" then + self:modified_callback(object) + end + end, + } +end + +function _M.init_worker() + if process.type() ~= "privileged agent" then + return + end + + apiserver_host = os.getenv("KUBERNETES_SERVICE_HOST") + --apiserver_host = "127.0.0.1" + if not apiserver_host or apiserver_host == "" then + error("get empty KUBERNETES_SERVICE_HOST value") + end + + apiserver_port = os.getenv("KUBERNETES_SERVICE_PORT") + --apiserver_port = "6443" + if not apiserver_port or apiserver_port == "" then + error("get empty KUBERNETES_SERVICE_PORT value") + end + + local err + apiserver_token, err = util.read_file("/var/run/secrets/kubernetes.io/serviceaccount/token") + if not apiserver_token or apiserver_token == "" then + error("get empty token value " .. (err or "")) + return + end + + fill_pending_resources() + + ngx.timer.at(0, fetch) +end + +return _M diff --git a/ci/install-ext-services-via-docker.sh b/ci/install-ext-services-via-docker.sh index 8f2ff28c65e9..a78997e32f2f 100755 --- a/ci/install-ext-services-via-docker.sh +++ b/ci/install-ext-services-via-docker.sh @@ -84,4 +84,66 @@ until [[ $(curl -s "127.0.0.1:8858/nacos/v1/ns/service/list?groupName=test_grou sleep 1; done +# create kubernetes cluster using kind +echo -e " +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerAddress: "127.0.0.1" + apiServerPort: 6443 +" > kind.yaml + +curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.11.1/kind-$(uname)-amd64" +chmod +x ./kind +./kind delete cluster --name apisix-test +./kind create cluster --name apisix-test --config ./kind.yaml + +echo -e " +kind: ServiceAccount +apiVersion: v1 +metadata: + name: apisix-test + namespace: default +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: apisix-test +rules: + - apiGroups: [ \"\" ] + resources: [ endpoints ] + verbs: [ get,list,watch ] +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: apisix-test +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: apisix-test +subjects: + - kind: ServiceAccount + name: apisix-test + namespace: default +" > apisix-test-rbac.yaml + +curl -Lo ./kubectl "https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl" +chmod +x ./kubectl +./kubectl apply -f ./apisix-test-rbac.yaml + +curl -Lo ./jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 +chmod +x ./jq + +K8S_SERVICEACCOUNT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o json "$1" |./jq -r .data.token | base64 --decode")}') +K8S_SERVICEACCOUNT_TOKEN_DIR="/var/run/secrets/kubernetes.io/serviceaccount" +K8S_SERVICEACCOUNT_TOKEN_FILE="/var/run/secrets/kubernetes.io/serviceaccount/token" + +mkdir -p ${K8S_SERVICEACCOUNT_TOKEN_DIR} +echo -n $K8S_SERVICEACCOUNT_TOKEN_CONTENT > ${K8S_SERVICEACCOUNT_TOKEN_FILE} + +export KUBERNETES_SERVICE_HOST="127.0.0.1" +export KUBERNETES_SERVICE_PORT="6443" + cd .. diff --git a/t/APISIX.pm b/t/APISIX.pm index 5b960cc87f8b..93df67ceaff3 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -239,6 +239,8 @@ env ENABLE_ETCD_AUTH; env APISIX_PROFILE; env PATH; # for searching external plugin runner's binary env TEST_NGINX_HTML_DIR; +env KUBERNETES_SERVICE_HOST; +env KUBERNETES_SERVICE_PORT; _EOC_ # set default `timeout` to 5sec diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t new file mode 100644 index 000000000000..def11753d20a --- /dev/null +++ b/t/discovery/k8s.t @@ -0,0 +1,167 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +no_root_location(); +no_shuffle(); +workers(4); + +our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: {} +nginx_config: + envs: + - KUBERNETES_SERVICE_HOST + - KUBERNETES_SERVICE_PORT + +_EOC_ + +run_tests(); + +__DATA__ + +=== TEST 1: error service_name - bad namespace +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: default/kube-dns:dns-tcp + discovery_type: k8s + type: roundrobin + +#END +--- request +GET /hello +--- error_code: 503 + + + +=== TEST 2: error service_name - bad service +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: kube-systm/notexit:dns-tcp + discovery_type: k8s + type: roundrobin + +#END +--- request +GET /hello +--- error_code: 503 + + + +=== TEST 3: error service_name - bad port +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: kube-systm/kube-dns:notexit + discovery_type: k8s + type: roundrobin + +#END +--- request +GET /hello +--- error_code: 503 + + + +=== TEST 4: get kube-system/kube-dns:dns-tcp from k8s - configured in routes +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + timeout: + connect: 0.5 + send : 1 + read : 1 + service_name : kube-system/kube-dns:dns-tcp + type: roundrobin + discovery_type: k8s +#END +--- request +GET /hello +--- error_code: 504 + + + +=== TEST 5: get kube-system/kube-dns:dns-tcp from k8s - configured in services +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + service_id: 1 +services: + - + id: 1 + upstream: + service_name : kube-system/kube-dns:dns-tcp + type: roundrobin + discovery_type: k8s + timeout: + connect: 0.5 + send : 1 + read : 1 +#END +--- request +GET /hello +--- error_code: 504 + + + +=== TEST 6: get kube-system/kube-dns:dns-tcp info from k8s - configured in upstreams +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + service_id: 1 +services: + - + id: 1 + upstream_id : 1 +upstreams: + - + id: 1 + service_name : kube-system/kube-dns:dns-tcp + type: roundrobin + discovery_type: k8s + timeout: + connect: 0.5 + send : 1 + read : 1 +#END +--- request +GET /hello +--- error_code: 504 From 5a351d19f014017ad0e17b1c2850b3c60ff5ce5f Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Sun, 5 Sep 2021 20:09:53 +0800 Subject: [PATCH 02/73] feat: add k8s discovery module Signed-off-by: adugeek --- apisix/discovery/k8s.lua | 362 +++++++++++++++++--------- ci/install-ext-services-via-docker.sh | 22 +- t/APISIX.pm | 4 + t/discovery/k8s.t | 196 +++++++++++++- 4 files changed, 448 insertions(+), 136 deletions(-) diff --git a/apisix/discovery/k8s.lua b/apisix/discovery/k8s.lua index 0fbb70296937..5f4b85d337a6 100644 --- a/apisix/discovery/k8s.lua +++ b/apisix/discovery/k8s.lua @@ -1,5 +1,23 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + local ngx = ngx local ipairs = ipairs +local pairs = pairs local string = string local tonumber = tonumber local tostring = tostring @@ -8,14 +26,15 @@ local os = os local process = require("ngx.process") local core = require("apisix.core") local util = require("apisix.cli.util") +local local_conf = require("apisix.core.config_local").local_conf() local http = require("resty.http") local endpoints_shared = ngx.shared.discovery +local apiserver_schema = "" local apiserver_host = "" -local apiserver_port = "" +local apiserver_port = 0 local apiserver_token = "" - -local default_weight = 50 +local default_weight = 0 local endpoint_lrucache = core.lrucache.new({ ttl = 300, @@ -30,59 +49,6 @@ local function sort_by_ip(a, b) return a.ip < b.ip end -local function on_endpoint_added(endpoint) - local subsets = endpoint.subsets - if subsets == nil or #subsets == 0 then - return - end - - local subset = subsets[1] - - local addresses = subset.addresses - if addresses == nil or #addresses == 0 then - return - end - - local ports = subset.ports - if ports == nil or #ports == 0 then - return - end - - core.table.sort(addresses, sort_by_ip) - core.table.clear(endpoint_cache) - for _, port in ipairs(ports) do - local nodes = core.table.new(#addresses, 0) - for i, address in ipairs(addresses) do - nodes[i] = { - host = address.ip, - port = port.port, - weight = default_weight - } - end - if port.name then - endpoint_cache[port.name] = nodes - else - endpoint_cache[tostring(port.port)] = nodes - end - end - - local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name - local endpoint_content = core.json.encode(endpoint_cache, true) - local endpoint_version = ngx.crc32_long(endpoint_content) - - local _, err - _, err = endpoints_shared:safe_set(endpoint_key .. "#version", endpoint_version) - if err then - core.log.emerg("set endpoint version into discovery DICT failed ,", err) - return - end - endpoints_shared:safe_set(endpoint_key, endpoint_content) - if err then - core.log.emerg("set endpoint into discovery DICT failed ,", err) - endpoints_shared:delete(endpoint_key .. "#version") - end -end - local function on_endpoint_deleted(endpoint) local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name endpoints_shared:delete(endpoint_key .. "#version") @@ -93,33 +59,40 @@ local function on_endpoint_modified(endpoint) if endpoint.subsets == nil or #endpoint.subsets == 0 then return on_endpoint_deleted(endpoint) end - local subset = endpoint.subsets[1] - if subset.addresses == nil or #subset.addresses == 0 then - return on_endpoint_deleted(endpoint) + local subsets = endpoint.subsets + for _, subset in ipairs(subsets) do + if subset.addresses ~= nil then + local addresses = subset.addresses + + for _, port in ipairs(subset.ports or empty_table) do + local port_name = "" + if port.name then + port_name = port.name + else + port_name = tostring(port.port) + end + + local nodes = endpoint_cache[port_name] + if nodes == nil then + nodes = core.table.new(0, #addresses * #subsets) + endpoint_cache[port_name] = nodes + end + + for _, address in ipairs(subset.addresses) do + core.table.insert(nodes, { + host = address.ip, + port = port.port, + weight = default_weight + }) + end + end + end end - local addresses = subset.addresses - if subset.ports == nil or #subset.ports == 0 then - return on_endpoint_deleted(endpoint) - end - local ports = subset.ports - - core.table.sort(addresses, sort_by_ip) - core.table.clear(endpoint_cache) - for _, port in ipairs(ports) do - local nodes = core.table.new(#addresses, 0) - for i, address in ipairs(addresses) do - nodes[i] = { - host = address.ip, - port = port.port, - weight = default_weight - } - end - if port.name then - endpoint_cache[port.name] = nodes - else - endpoint_cache[tostring(port.port)] = nodes + for _, ports in pairs(endpoint_cache) do + for _, nodes in pairs(ports) do + core.table.sort(nodes, sort_by_ip) end end @@ -130,12 +103,12 @@ local function on_endpoint_modified(endpoint) local _, err _, err = endpoints_shared:safe_set(endpoint_key .. "#version", endpoint_version) if err then - core.log.emerg("set endpoint version into discovery DICT failed ,", err) + core.log.emerg("set endpoint version into discovery DICT failed, ", err) return end endpoints_shared:safe_set(endpoint_key, endpoint_content) if err then - core.log.emerg("set endpoint into discovery DICT failed ,", err) + core.log.emerg("set endpoint into discovery DICT failed, ", err) endpoints_shared:delete(endpoint_key .. "#version") end end @@ -146,7 +119,7 @@ local function list_resource(httpc, resource, continue) path = resource:list_path(), query = resource:list_query(continue), headers = { - ["Host"] = string.format("%s:%s", apiserver_host, apiserver_port), + ["Host"] = string.format("%s:%d", apiserver_host, apiserver_port), ["Authorization"] = string.format("Bearer %s", apiserver_token), ["Accept"] = "application/json", ["Connection"] = "keep-alive" @@ -174,7 +147,7 @@ local function list_resource(httpc, resource, continue) resource.newest_resource_version = data.metadata.resourceVersion for _, item in ipairs(data.items or empty_table) do - resource:event_dispatch("ADDED", item) + resource:event_dispatch("ADDED", item, "list") end if data.metadata.continue ~= nil and data.metadata.continue ~= "" then @@ -193,7 +166,7 @@ local function watch_resource(httpc, resource) path = resource:watch_path(), query = resource:watch_query(watch_seconds), headers = { - ["Host"] = string.format("%s:%s", apiserver_host, apiserver_port), + ["Host"] = string.format("%s:%d", apiserver_host, apiserver_port), ["Authorization"] = string.format("Bearer %s", apiserver_token), ["Accept"] = "application/json", ["Connection"] = "keep-alive" @@ -250,7 +223,7 @@ local function watch_resource(httpc, resource) resource.newest_resource_version = v.object.metadata.resource_version if v.type ~= "BOOKMARK" then - resource:event_dispatch(v.type, v.object) + resource:event_dispatch(v.type, v.object, "watch") end end @@ -266,6 +239,7 @@ local function watch_resource(httpc, resource) end local function fetch_resource(resource) + local begin_time = ngx.time() while true do local ok = false local reason, message = "", "" @@ -275,15 +249,15 @@ local function fetch_resource(resource) resource.watch_state = "connecting" core.log.info("begin to connect ", apiserver_host, ":", apiserver_port) ok, message = httpc:connect({ - scheme = "https", + scheme = apiserver_schema, host = apiserver_host, - port = tonumber(apiserver_port), + port = apiserver_port, ssl_verify = false }) if not ok then resource.watch_state = "connecting" - core.log.error("connect apiserver failed , apiserver_host: ", apiserver_host, "apiserver_port", - apiserver_port, "message : ", message) + core.log.error("connect apiserver failed , apiserver_host: ", apiserver_host, + ", apiserver_port", apiserver_port, ", message : ", message) retry_interval = 100 break end @@ -294,7 +268,8 @@ local function fetch_resource(resource) ok, reason, message = list_resource(httpc, resource, nil) if not ok then resource.watch_state = "list failed" - core.log.error("list failed , resource: ", resource.plural, " reason: ", reason, "message : ", message) + core.log.error("list failed, resource: ", resource.plural, + ", reason: ", reason, ", message : ", message) retry_interval = 100 break end @@ -306,27 +281,28 @@ local function fetch_resource(resource) ok, reason, message = watch_resource(httpc, resource) if not ok then resource.watch_state = "watch failed" - core.log.error("watch failed, resource: ", resource.plural, " reason: ", reason, "message : ", message) + core.log.error("watch failed, resource: ", resource.plural, + ", reason: ", reason, ", message : ", message) retry_interval = 0 break end resource.watch_state = "watch finished" retry_interval = 0 until true + + -- every 3 hours,we should quit and use another timer + local now_time = ngx.time() + if now_time - begin_time >= 10800 then + break + end if retry_interval ~= 0 then ngx.sleep(retry_interval) end end -end - -local function fetch() - local threads = core.table.new(#pending_resources, 0) - for i, resource in ipairs(pending_resources) do - threads[i] = ngx.thread.spawn(fetch_resource, resource) - end - for _, thread in ipairs(threads) do - ngx.thread.wait(thread) + local timer_runner = function() + fetch_resource(resource) end + ngx.timer.at(0, timer_runner) end local function create_endpoint_lrucache(endpoint_key, endpoint_port) @@ -344,6 +320,147 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port) return endpoint[endpoint_port] end +local schema = { + type = "object", + properties = { + service = { + type = "object", + properties = { + schema = { + type = "string", + enum = { "http", "https" }, + default = "https", + }, + host = { + type = "string", + default = "${KUBERNETES_SERVICE_HOST}", + oneOf = { + { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, + { pattern = [[^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$]] }, + } + }, + port = { + type = "string", + default = "${KUBERNETES_SERVICE_PORT}", + oneOf = { + { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, + { pattern = [[^(([1-9]\d{0,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))$]] }, + }, + }, + }, + default = { + schema = "https", + host = "${KUBERNETES_SERVICE_HOST}", + port = "${KUBERNETES_SERVICE_PORT}", + } + }, + client = { + type = "object", + properties = { + token = { + type = "string", + oneOf = { + { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, + { pattern = [[^[A-Za-z0-9+\/_=-]{0,4096}$]] }, + }, + }, + token_file = { + type = "string", + pattern = [[^[^\:*?"<>|]*$]], + minLength = 1, + maxLength = 500, + } + }, + oneOf = { + { required = { "token" } }, + { required = { "token_file" } }, + }, + default = { + token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + }, + default_weight = { + type = "integer", + default = 50, + minimum = 0, + }, + }, + default = { + service = { + schema = "https", + host = "${KUBERNETES_SERVICE_HOST}", + port = "${KUBERNETES_SERVICE_PORT}", + }, + client = { + token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + }, + default_weight = 50 + } +} + +local function load_value(key) + if #key > 3 then + local a, b = string.byte(key, 1, 2) + local c = string.byte(key, #key, #key) + -- '$', '{', '}' == 36,123,125 + if a == 36 and b == 123 and c == 125 then + local env = string.sub(key, 3, #key - 1) + local val = os.getenv(env) + if not val or val == "" then + return false, nil, "get empty " .. key .. " value" + end + return true, val, nil + end + end + return true, key, nil +end + +local function read_conf(conf) + apiserver_schema = conf.service.schema + + local ok, value, error + ok, value, error = load_value(conf.service.host) + if not ok then + return false, error + end + apiserver_host = value + + ok, value, error = load_value(conf.service.port) + if not ok then + return false, error + end + apiserver_port = tonumber(value) + if not apiserver_port or apiserver_port <= 0 or apiserver_port > 65535 then + return false, "get invalid port value: " .. apiserver_port + end + + -- we should not check if the apiserver_token is empty here + if conf.client.token then + ok, value, error = load_value(conf.client.token) + if not ok then + return false, error + end + apiserver_token = value + elseif conf.client.token_file and conf.client.token_file ~= "" then + ok, value, error = load_value(conf.client.token_file) + if not ok then + return false, error + end + local apiserver_token_file = value + + apiserver_token, error = util.read_file(apiserver_token_file) + if not apiserver_token then + return false, error + end + else + return false, "invalid k8s discovery configuration: should set one of [client.token,client.token_file] but none" + end + + default_weight = conf.default_weight or 50 + + return true, nil +end + local _M = { version = 0.01 } @@ -400,8 +517,8 @@ local function fill_pending_resources() end, watch_query = function(self, timeout) - return string.format("watch=true&allowWatchBookmarks=true&timeoutSeconds=%d&resourceVersion=%s", timeout, - self.newest_resource_version) + return string.format("watch=true&allowWatchBookmarks=true&timeoutSeconds=%d&resourceVersion=%s", + timeout, self.newest_resource_version) end, pre_list_callback = function(self) @@ -413,8 +530,8 @@ local function fill_pending_resources() endpoints_shared:flush_expired() end, - added_callback = function(self, object) - on_endpoint_added(object) + added_callback = function(self, object, drive) + on_endpoint_modified(object) end, modified_callback = function(self, object) @@ -425,14 +542,14 @@ local function fill_pending_resources() on_endpoint_deleted(object) end, - event_dispatch = function(self, event, object) + event_dispatch = function(self, event, object, drive) if event == "DELETED" or object.deletionTimestamp ~= nil then self:deleted_callback(object) return end if event == "ADDED" then - self:added_callback(object) + self:added_callback(object, drive) elseif event == "MODIFIED" then self:modified_callback(object) end @@ -445,28 +562,33 @@ function _M.init_worker() return end - apiserver_host = os.getenv("KUBERNETES_SERVICE_HOST") - --apiserver_host = "127.0.0.1" - if not apiserver_host or apiserver_host == "" then - error("get empty KUBERNETES_SERVICE_HOST value") + if not local_conf.discovery.k8s then + error("does not set k8s discovery configuration") + return end - apiserver_port = os.getenv("KUBERNETES_SERVICE_PORT") - --apiserver_port = "6443" - if not apiserver_port or apiserver_port == "" then - error("get empty KUBERNETES_SERVICE_PORT value") + core.log.info("k8s discovery configuration: ", core.json.encode(local_conf.discovery.k8s, true)) + + local ok, err = core.schema.check(schema, local_conf.discovery.k8s) + if not ok then + error("invalid k8s discovery configuration: " .. err) + return end - local err - apiserver_token, err = util.read_file("/var/run/secrets/kubernetes.io/serviceaccount/token") - if not apiserver_token or apiserver_token == "" then - error("get empty token value " .. (err or "")) + ok, err = read_conf(local_conf.discovery.k8s) + if not ok then + error(err) return end fill_pending_resources() - ngx.timer.at(0, fetch) + for _, resource in ipairs(pending_resources) do + local timer_runner = function() + fetch_resource(resource) + end + ngx.timer.at(0, timer_runner) + end end return _M diff --git a/ci/install-ext-services-via-docker.sh b/ci/install-ext-services-via-docker.sh index a78997e32f2f..c406fa77913a 100755 --- a/ci/install-ext-services-via-docker.sh +++ b/ci/install-ext-services-via-docker.sh @@ -132,18 +132,30 @@ subjects: curl -Lo ./kubectl "https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl" chmod +x ./kubectl ./kubectl apply -f ./apisix-test-rbac.yaml +./kubectl proxy -p 6445 & +until [[ $(curl 127.0.0.1:6445/api/v1/pods?fieldSelector=status.phase%21%3DRunning |./jq .items) == "[]" ]]; do + echo 'wait k8s start...' + sleep 1; +done curl -Lo ./jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 chmod +x ./jq -K8S_SERVICEACCOUNT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o json "$1" |./jq -r .data.token | base64 --decode")}') -K8S_SERVICEACCOUNT_TOKEN_DIR="/var/run/secrets/kubernetes.io/serviceaccount" -K8S_SERVICEACCOUNT_TOKEN_FILE="/var/run/secrets/kubernetes.io/serviceaccount/token" +KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o json "$1" |./jq -r .data.token | base64 --decode")}') + +# if we do not have permission to create folders under the /var/run path, we will use the /tmp as an alternative +KUBERNETES_CLIENT_TOKEN_DIR="/var/run/secrets/kubernetes.io/serviceaccount" +if ! mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} ;then + KUBERNETES_CLIENT_TOKEN_DIR=/tmp${KUBERNETES_CLIENT_TOKEN_DIR} + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} +fi -mkdir -p ${K8S_SERVICEACCOUNT_TOKEN_DIR} -echo -n $K8S_SERVICEACCOUNT_TOKEN_CONTENT > ${K8S_SERVICEACCOUNT_TOKEN_FILE} +KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}"/token" +echo -n $KUBERNETES_CLIENT_TOKEN_CONTENT > ${KUBERNETES_CLIENT_TOKEN_FILE} export KUBERNETES_SERVICE_HOST="127.0.0.1" export KUBERNETES_SERVICE_PORT="6443" +export KUBERNETES_CLIENT_TOKEN=${KUBERNETES_CLIENT_TOKEN_CONTENT} +export KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_FILE} cd .. diff --git a/t/APISIX.pm b/t/APISIX.pm index 93df67ceaff3..cd159e922c0c 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -241,6 +241,10 @@ env PATH; # for searching external plugin runner's binary env TEST_NGINX_HTML_DIR; env KUBERNETES_SERVICE_HOST; env KUBERNETES_SERVICE_PORT; +env KUBERNETES_CLIENT_TOKEN; +env KUBERNETES_CLIENT_TOKEN_FILE; +ls + _EOC_ # set default `timeout` to 5sec diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t index def11753d20a..fca19867589c 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/k8s.t @@ -28,19 +28,196 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: {} + k8s: + client: + token: \$\{KUBERNETES_CLIENT_TOKEN\} nginx_config: envs: - KUBERNETES_SERVICE_HOST - KUBERNETES_SERVICE_PORT - + - KUBERNETES_CLIENT_TOKEN _EOC_ run_tests(); __DATA__ -=== TEST 1: error service_name - bad namespace +=== TEST 1: use default parameters +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token: ${KUBERNETES_CLIENT_TOKEN} +nginx_config: + envs: + - KUBERNETES_SERVICE_HOST + - KUBERNETES_SERVICE_PORT + - KUBERNETES_CLIENT_TOKEN +--- apisix_yaml +routes: + - uri: /hello + upstream: + timeout: + connect: 0.5 + send : 1 + read : 1 + service_name : kube-system/kube-dns:dns-tcp + type: roundrobin + discovery_type: k8s +#END +--- request +GET /hello +--- error_code: 504 + + + +=== TEST 2: use specify parameters +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + service: + host: "127.0.0.1" + port: "6443" + client: + token: "${KUBERNETES_CLIENT_TOKEN}" +nginx_config: + envs: + - KUBERNETES_CLIENT_TOKEN +--- apisix_yaml +routes: + - + uri: /hello + upstream: + timeout: + connect: 0.5 + send : 1 + read : 1 + service_name : kube-system/kube-dns:dns-tcp + type: roundrobin + discovery_type: k8s +#END +--- request +GET /hello +--- error_code: 504 + + + + +=== TEST 3: use specify environment parameters +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + service: + host: ${KUBERNETES_SERVICE_HOST} + port: ${KUBERNETES_SERVICE_PORT} + client: + token: "${KUBERNETES_CLIENT_TOKEN}" +nginx_config: + envs: + - KUBERNETES_SERVICE_HOST + - KUBERNETES_SERVICE_PORT + - KUBERNETES_CLIENT_TOKEN +--- apisix_yaml +routes: + - + uri: /hello + upstream: + timeout: + connect: 0.5 + send : 1 + read : 1 + service_name : kube-system/kube-dns:dns-tcp + type: roundrobin + discovery_type: k8s +#END +--- request +GET /hello +--- error_code: 504 + + + + +=== TEST 4: use token_file +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} +nginx_config: + envs: + - KUBERNETES_SERVICE_HOST + - KUBERNETES_SERVICE_PORT + - KUBERNETES_CLIENT_TOKEN_FILE +--- apisix_yaml +routes: + - + uri: /hello + upstream: + timeout: + connect: 0.5 + send : 1 + read : 1 + service_name : kube-system/kube-dns:dns-tcp + type: roundrobin + discovery_type: k8s +#END +--- request +GET /hello +--- error_code: 504 + + + + +=== TEST 5: use http +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + service: + schema: http + host: "127.0.0.1" + port: "6445" + client: + token: "" +--- apisix_yaml +routes: + - + uri: /hello + upstream: + timeout: + connect: 0.5 + send : 1 + read : 1 + service_name : kube-system/kube-dns:dns-tcp + type: roundrobin + discovery_type: k8s +#END +--- request +GET /hello +--- error_code: 504 + + + +=== TEST 6: error service_name - bad namespace --- yaml_config eval: $::yaml_config --- apisix_yaml routes: @@ -50,7 +227,6 @@ routes: service_name: default/kube-dns:dns-tcp discovery_type: k8s type: roundrobin - #END --- request GET /hello @@ -58,7 +234,7 @@ GET /hello -=== TEST 2: error service_name - bad service +=== TEST 7: error service_name - bad service --- yaml_config eval: $::yaml_config --- apisix_yaml routes: @@ -68,7 +244,6 @@ routes: service_name: kube-systm/notexit:dns-tcp discovery_type: k8s type: roundrobin - #END --- request GET /hello @@ -76,7 +251,7 @@ GET /hello -=== TEST 3: error service_name - bad port +=== TEST 8: error service_name - bad port --- yaml_config eval: $::yaml_config --- apisix_yaml routes: @@ -86,7 +261,6 @@ routes: service_name: kube-systm/kube-dns:notexit discovery_type: k8s type: roundrobin - #END --- request GET /hello @@ -94,7 +268,7 @@ GET /hello -=== TEST 4: get kube-system/kube-dns:dns-tcp from k8s - configured in routes +=== TEST 9: get kube-system/kube-dns:dns-tcp from k8s - configured in routes --- yaml_config eval: $::yaml_config --- apisix_yaml routes: @@ -115,7 +289,7 @@ GET /hello -=== TEST 5: get kube-system/kube-dns:dns-tcp from k8s - configured in services +=== TEST 10: get kube-system/kube-dns:dns-tcp from k8s - configured in services --- yaml_config eval: $::yaml_config --- apisix_yaml routes: @@ -140,7 +314,7 @@ GET /hello -=== TEST 6: get kube-system/kube-dns:dns-tcp info from k8s - configured in upstreams +=== TEST 11: get kube-system/kube-dns:dns-tcp info from k8s - configured in upstreams --- yaml_config eval: $::yaml_config --- apisix_yaml routes: From 77dd742a368ff797ebd7af33ac8e04bbd4086829 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Mon, 6 Sep 2021 09:46:35 +0800 Subject: [PATCH 03/73] feat: add k8s discovery module Signed-off-by: adugeek --- ci/install-ext-services-via-docker.sh | 7 ++++--- t/discovery/k8s.t | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/install-ext-services-via-docker.sh b/ci/install-ext-services-via-docker.sh index c406fa77913a..78cf3f6f63ee 100755 --- a/ci/install-ext-services-via-docker.sh +++ b/ci/install-ext-services-via-docker.sh @@ -133,14 +133,15 @@ curl -Lo ./kubectl "https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl" chmod +x ./kubectl ./kubectl apply -f ./apisix-test-rbac.yaml ./kubectl proxy -p 6445 & + +curl -Lo ./jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 +chmod +x ./jq + until [[ $(curl 127.0.0.1:6445/api/v1/pods?fieldSelector=status.phase%21%3DRunning |./jq .items) == "[]" ]]; do echo 'wait k8s start...' sleep 1; done -curl -Lo ./jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -chmod +x ./jq - KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o json "$1" |./jq -r .data.token | base64 --decode")}') # if we do not have permission to create folders under the /var/run path, we will use the /tmp as an alternative diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t index fca19867589c..b488e1802e3d 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/k8s.t @@ -83,7 +83,7 @@ apisix: enable_admin: false discovery: k8s: - service: + service: host: "127.0.0.1" port: "6443" client: @@ -119,7 +119,7 @@ apisix: enable_admin: false discovery: k8s: - service: + service: host: ${KUBERNETES_SERVICE_HOST} port: ${KUBERNETES_SERVICE_PORT} client: @@ -192,7 +192,7 @@ apisix: enable_admin: false discovery: k8s: - service: + service: schema: http host: "127.0.0.1" port: "6445" @@ -326,7 +326,7 @@ services: id: 1 upstream_id : 1 upstreams: - - + - id: 1 service_name : kube-system/kube-dns:dns-tcp type: roundrobin From 45e1bcd01f2634d6328adfbedaed017286696f23 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Mon, 6 Sep 2021 10:57:58 +0800 Subject: [PATCH 04/73] feat: add k8s discovery module Signed-off-by: adugeek --- t/APISIX.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/t/APISIX.pm b/t/APISIX.pm index cd159e922c0c..82742a8133b5 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -243,7 +243,6 @@ env KUBERNETES_SERVICE_HOST; env KUBERNETES_SERVICE_PORT; env KUBERNETES_CLIENT_TOKEN; env KUBERNETES_CLIENT_TOKEN_FILE; -ls _EOC_ From 8042718c7100628c5e508156b2f93c672d752200 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Mon, 6 Sep 2021 23:46:40 +0800 Subject: [PATCH 05/73] feat: add k8s discovery module Signed-off-by: adugeek --- apisix/discovery/k8s.lua | 59 +++++---- t/discovery/k8s.t | 266 ++++++++++++--------------------------- 2 files changed, 110 insertions(+), 215 deletions(-) diff --git a/apisix/discovery/k8s.lua b/apisix/discovery/k8s.lua index 5f4b85d337a6..076e36254761 100644 --- a/apisix/discovery/k8s.lua +++ b/apisix/discovery/k8s.lua @@ -66,7 +66,7 @@ local function on_endpoint_modified(endpoint) local addresses = subset.addresses for _, port in ipairs(subset.ports or empty_table) do - local port_name = "" + local port_name if port.name then port_name = port.name else @@ -119,8 +119,8 @@ local function list_resource(httpc, resource, continue) path = resource:list_path(), query = resource:list_query(continue), headers = { - ["Host"] = string.format("%s:%d", apiserver_host, apiserver_port), - ["Authorization"] = string.format("Bearer %s", apiserver_token), + ["Host"] = apiserver_host .. ":" .. apiserver_port, + ["Authorization"] = "Bearer " .. apiserver_token, ["Accept"] = "application/json", ["Connection"] = "keep-alive" } @@ -133,7 +133,6 @@ local function list_resource(httpc, resource, continue) if res.status ~= 200 then return false, res.reason, res:read_body() or "" end - local body, err = res:read_body() if err then return false, "ReadBodyError", err @@ -166,8 +165,8 @@ local function watch_resource(httpc, resource) path = resource:watch_path(), query = resource:watch_query(watch_seconds), headers = { - ["Host"] = string.format("%s:%d", apiserver_host, apiserver_port), - ["Authorization"] = string.format("Bearer %s", apiserver_token), + ["Host"] = apiserver_host .. ":" .. apiserver_port, + ["Authorization"] = "Bearer " .. apiserver_token, ["Accept"] = "application/json", ["Connection"] = "keep-alive" } @@ -182,10 +181,10 @@ local function watch_resource(httpc, resource) end local remainder_body = "" - local body = "" + local body local reader = res.body_reader - local gmatch_iterator; - local captures; + local gmatch_iterator + local captures local captured_size = 0 while true do @@ -241,9 +240,9 @@ end local function fetch_resource(resource) local begin_time = ngx.time() while true do - local ok = false - local reason, message = "", "" - local retry_interval = 0 + local ok + local reason, message + local retry_interval repeat local httpc = http.new() resource.watch_state = "connecting" @@ -308,18 +307,30 @@ end local function create_endpoint_lrucache(endpoint_key, endpoint_port) local endpoint_content, _, _ = endpoints_shared:get_stale(endpoint_key) if not endpoint_content then - core.log.emerg("get empty endpoint content from discovery DICT,this should not happen ", endpoint_key) + core.log.emerg("get empty endpoint content from discovery DICT,this should not happen ", + endpoint_key) return nil end local endpoint, _ = core.json.decode(endpoint_content) if not endpoint then - core.log.emerg("decode endpoint content failed, this should not happen, content : ", endpoint_content) + core.log.emerg("decode endpoint content failed, this should not happen, content : ", + endpoint_content) end return endpoint[endpoint_port] end +local host_patterns = { + { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, + { pattern = [[^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$]] }, +} + +local port_patterns = { + { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, + { pattern = [[^(([1-9]\d{0,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))$]] }, +} + local schema = { type = "object", properties = { @@ -334,18 +345,12 @@ local schema = { host = { type = "string", default = "${KUBERNETES_SERVICE_HOST}", - oneOf = { - { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, - { pattern = [[^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$]] }, - } + oneOf = host_patterns, }, port = { type = "string", default = "${KUBERNETES_SERVICE_PORT}", - oneOf = { - { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, - { pattern = [[^(([1-9]\d{0,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))$]] }, - }, + oneOf = port_patterns, }, }, default = { @@ -453,7 +458,8 @@ local function read_conf(conf) return false, error end else - return false, "invalid k8s discovery configuration: should set one of [client.token,client.token_file] but none" + return false, "invalid k8s discovery configuration:" .. + "should set one of [client.token,client.token_file] but none" end default_weight = conf.default_weight or 50 @@ -480,7 +486,8 @@ function _M.nodes(service_name) core.log.info("get empty endpoint version from discovery DICT ", endpoint_key) return nil end - return endpoint_lrucache(service_name, endpoint_version, create_endpoint_lrucache, endpoint_key, endpoint_port) + return endpoint_lrucache(service_name, endpoint_version, + create_endpoint_lrucache, endpoint_key, endpoint_port) end local function fill_pending_resources() @@ -517,8 +524,8 @@ local function fill_pending_resources() end, watch_query = function(self, timeout) - return string.format("watch=true&allowWatchBookmarks=true&timeoutSeconds=%d&resourceVersion=%s", - timeout, self.newest_resource_version) + return "watch=true&allowWatchBookmarks=true&timeoutSeconds=" .. timeout .. + "&resourceVersion=" .. self.newest_resource_version end, pre_list_callback = function(self) diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t index b488e1802e3d..c277d7585b4e 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/k8s.t @@ -22,7 +22,10 @@ no_root_location(); no_shuffle(); workers(4); -our $yaml_config = <<_EOC_; +add_block_preprocessor(sub { + my ($block) = @_; + + my $yaml_config = $block->yaml_config // <<_EOC_; apisix: node_listen: 1984 config_center: yaml @@ -37,6 +40,40 @@ nginx_config: - KUBERNETES_SERVICE_PORT - KUBERNETES_CLIENT_TOKEN _EOC_ + + $block->set_value("yaml_config", $yaml_config); + + my $apisix_yaml = $block->apisix_yaml // <<_EOC_; +routes: [] +#END +_EOC_ + + $block->set_value("apisix_yaml", $apisix_yaml); + + + my $config = $block->config // <<_EOC_; + location /t { + content_by_lua_block { + local d = require("apisix.discovery.k8s") + ngx.sleep(1) + local s = ngx.var.arg_s + local nodes = d.nodes(s) + + ngx.status = 200 + local body="" + + if nodes == nil or #nodes == 0 then + body="empty" + else + body="passed" + end + ngx.say(body) + } + } +_EOC_ + + $block->set_value("config", $config); +}); run_tests(); @@ -57,21 +94,10 @@ nginx_config: - KUBERNETES_SERVICE_HOST - KUBERNETES_SERVICE_PORT - KUBERNETES_CLIENT_TOKEN ---- apisix_yaml -routes: - - uri: /hello - upstream: - timeout: - connect: 0.5 - send : 1 - read : 1 - service_name : kube-system/kube-dns:dns-tcp - type: roundrobin - discovery_type: k8s -#END --- request -GET /hello ---- error_code: 504 +GET /t?s=default/kubernetes:https +--- response_body +passed @@ -91,23 +117,12 @@ discovery: nginx_config: envs: - KUBERNETES_CLIENT_TOKEN ---- apisix_yaml -routes: - - - uri: /hello - upstream: - timeout: - connect: 0.5 - send : 1 - read : 1 - service_name : kube-system/kube-dns:dns-tcp - type: roundrobin - discovery_type: k8s -#END --- request -GET /hello ---- error_code: 504 - +GET /t?s=default/kubernetes:https +--- response_body +passed +--- no_error_log +[error] @@ -123,29 +138,18 @@ discovery: host: ${KUBERNETES_SERVICE_HOST} port: ${KUBERNETES_SERVICE_PORT} client: - token: "${KUBERNETES_CLIENT_TOKEN}" + token: ${KUBERNETES_CLIENT_TOKEN} nginx_config: envs: - KUBERNETES_SERVICE_HOST - KUBERNETES_SERVICE_PORT - KUBERNETES_CLIENT_TOKEN ---- apisix_yaml -routes: - - - uri: /hello - upstream: - timeout: - connect: 0.5 - send : 1 - read : 1 - service_name : kube-system/kube-dns:dns-tcp - type: roundrobin - discovery_type: k8s -#END --- request -GET /hello ---- error_code: 504 - +GET /t?s=default/kubernetes:https +--- response_body +passed +--- no_error_log +[error] @@ -164,23 +168,12 @@ nginx_config: - KUBERNETES_SERVICE_HOST - KUBERNETES_SERVICE_PORT - KUBERNETES_CLIENT_TOKEN_FILE ---- apisix_yaml -routes: - - - uri: /hello - upstream: - timeout: - connect: 0.5 - send : 1 - read : 1 - service_name : kube-system/kube-dns:dns-tcp - type: roundrobin - discovery_type: k8s -#END --- request -GET /hello ---- error_code: 504 - +GET /t?s=default/kubernetes:https +--- response_body +passed +--- no_error_log +[error] @@ -198,144 +191,39 @@ discovery: port: "6445" client: token: "" ---- apisix_yaml -routes: - - - uri: /hello - upstream: - timeout: - connect: 0.5 - send : 1 - read : 1 - service_name : kube-system/kube-dns:dns-tcp - type: roundrobin - discovery_type: k8s -#END --- request -GET /hello ---- error_code: 504 +GET /t?s=default/kubernetes:https +--- response_body +passed +--- no_error_log +[error] === TEST 6: error service_name - bad namespace ---- yaml_config eval: $::yaml_config ---- apisix_yaml -routes: - - - uri: /hello - upstream: - service_name: default/kube-dns:dns-tcp - discovery_type: k8s - type: roundrobin -#END --- request -GET /hello ---- error_code: 503 +GET /t?s=notexist/kubernetes:https +--- response_body +empty +--- no_error_log +[error] === TEST 7: error service_name - bad service ---- yaml_config eval: $::yaml_config ---- apisix_yaml -routes: - - - uri: /hello - upstream: - service_name: kube-systm/notexit:dns-tcp - discovery_type: k8s - type: roundrobin -#END --- request -GET /hello ---- error_code: 503 +GET /t?s=default/notexist:https +--- response_body +empty +--- no_error_log +[error] === TEST 8: error service_name - bad port ---- yaml_config eval: $::yaml_config ---- apisix_yaml -routes: - - - uri: /hello - upstream: - service_name: kube-systm/kube-dns:notexit - discovery_type: k8s - type: roundrobin -#END ---- request -GET /hello ---- error_code: 503 - - - -=== TEST 9: get kube-system/kube-dns:dns-tcp from k8s - configured in routes ---- yaml_config eval: $::yaml_config ---- apisix_yaml -routes: - - - uri: /hello - upstream: - timeout: - connect: 0.5 - send : 1 - read : 1 - service_name : kube-system/kube-dns:dns-tcp - type: roundrobin - discovery_type: k8s -#END ---- request -GET /hello ---- error_code: 504 - - - -=== TEST 10: get kube-system/kube-dns:dns-tcp from k8s - configured in services ---- yaml_config eval: $::yaml_config ---- apisix_yaml -routes: - - - uri: /hello - service_id: 1 -services: - - - id: 1 - upstream: - service_name : kube-system/kube-dns:dns-tcp - type: roundrobin - discovery_type: k8s - timeout: - connect: 0.5 - send : 1 - read : 1 -#END ---- request -GET /hello ---- error_code: 504 - - - -=== TEST 11: get kube-system/kube-dns:dns-tcp info from k8s - configured in upstreams ---- yaml_config eval: $::yaml_config ---- apisix_yaml -routes: - - - uri: /hello - service_id: 1 -services: - - - id: 1 - upstream_id : 1 -upstreams: - - - id: 1 - service_name : kube-system/kube-dns:dns-tcp - type: roundrobin - discovery_type: k8s - timeout: - connect: 0.5 - send : 1 - read : 1 -#END --- request -GET /hello ---- error_code: 504 +GET /t?s=default/kubernetes:notexist +--- response_body +empty +--- no_error_log +[error] From 2694c7fecf6384191a9327b6f0866e2ecd50ecf2 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Tue, 7 Sep 2021 15:41:27 +0800 Subject: [PATCH 06/73] feat: add k8s discovery module Signed-off-by: adugeek --- ci/install-ext-services-via-docker.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/install-ext-services-via-docker.sh b/ci/install-ext-services-via-docker.sh index 78cf3f6f63ee..1398142b848c 100755 --- a/ci/install-ext-services-via-docker.sh +++ b/ci/install-ext-services-via-docker.sh @@ -159,4 +159,9 @@ export KUBERNETES_SERVICE_PORT="6443" export KUBERNETES_CLIENT_TOKEN=${KUBERNETES_CLIENT_TOKEN_CONTENT} export KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_FILE} +echo 'set env KUBERNETES_SERVICE_HOST='${KUBERNETES_SERVICE_HOST} +echo 'set env KUBERNETES_SERVICE_PORT='${KUBERNETES_SERVICE_PORT} +echo 'set env KUBERNETES_CLIENT_TOKEN='${KUBERNETES_CLIENT_TOKEN} +echo 'set env KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} + cd .. From 046b65cc769d7c6d3c4d3de46082b94e7d7bef16 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Wed, 8 Sep 2021 08:32:08 +0800 Subject: [PATCH 07/73] feat: add k8s discovery module Signed-off-by: adugeek --- apisix/discovery/k8s.lua | 32 ++++++++++++++++++-------------- t/discovery/k8s.t | 6 +++--- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/apisix/discovery/k8s.lua b/apisix/discovery/k8s.lua index 076e36254761..7fe406032a67 100644 --- a/apisix/discovery/k8s.lua +++ b/apisix/discovery/k8s.lua @@ -23,6 +23,7 @@ local tonumber = tonumber local tostring = tostring local math = math local os = os +local error = error local process = require("ngx.process") local core = require("apisix.core") local util = require("apisix.cli.util") @@ -307,7 +308,7 @@ end local function create_endpoint_lrucache(endpoint_key, endpoint_port) local endpoint_content, _, _ = endpoints_shared:get_stale(endpoint_key) if not endpoint_content then - core.log.emerg("get empty endpoint content from discovery DICT,this should not happen ", + core.log.emerg("get empty endpoint content from discovery DIC, this should not happen ", endpoint_key) return nil end @@ -411,8 +412,8 @@ local function load_value(key) if a == 36 and b == 123 and c == 125 then local env = string.sub(key, 3, #key - 1) local val = os.getenv(env) - if not val or val == "" then - return false, nil, "get empty " .. key .. " value" + if not val then + return false, nil, "not found environment variable " .. env end return true, val, nil end @@ -423,16 +424,19 @@ end local function read_conf(conf) apiserver_schema = conf.service.schema - local ok, value, error - ok, value, error = load_value(conf.service.host) + local ok, value, message + ok, value, message = load_value(conf.service.host) if not ok then - return false, error + return false, message end apiserver_host = value + if apiserver_host == "" then + return false, "get empty host value" + end - ok, value, error = load_value(conf.service.port) + ok, value, message = load_value(conf.service.port) if not ok then - return false, error + return false, message end apiserver_port = tonumber(value) if not apiserver_port or apiserver_port <= 0 or apiserver_port > 65535 then @@ -441,21 +445,21 @@ local function read_conf(conf) -- we should not check if the apiserver_token is empty here if conf.client.token then - ok, value, error = load_value(conf.client.token) + ok, value, message = load_value(conf.client.token) if not ok then - return false, error + return false, message end apiserver_token = value elseif conf.client.token_file and conf.client.token_file ~= "" then - ok, value, error = load_value(conf.client.token_file) + ok, value, message = load_value(conf.client.token_file) if not ok then - return false, error + return false, message end local apiserver_token_file = value - apiserver_token, error = util.read_file(apiserver_token_file) + apiserver_token, message = util.read_file(apiserver_token_file) if not apiserver_token then - return false, error + return false, message end else return false, "invalid k8s discovery configuration:" .. diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t index c277d7585b4e..b7aea53c0022 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/k8s.t @@ -60,16 +60,16 @@ _EOC_ local nodes = d.nodes(s) ngx.status = 200 - local body="" + local body if nodes == nil or #nodes == 0 then - body="empty" + body="empty" else body="passed" end ngx.say(body) } - } + } _EOC_ $block->set_value("config", $config); From 99aef06d60534bb6a6d694ff7859cc6ca1d78e1d Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Wed, 8 Sep 2021 15:27:51 +0800 Subject: [PATCH 08/73] feat: add k8s discovery module Signed-off-by: adugeek --- ci/install-ext-services-via-docker.sh | 26 +++++----- t/APISIX.pm | 5 -- t/discovery/k8s.t | 69 +++++++++++++++++++-------- 3 files changed, 61 insertions(+), 39 deletions(-) diff --git a/ci/install-ext-services-via-docker.sh b/ci/install-ext-services-via-docker.sh index 1398142b848c..efa1b176a476 100755 --- a/ci/install-ext-services-via-docker.sh +++ b/ci/install-ext-services-via-docker.sh @@ -146,22 +146,22 @@ KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk # if we do not have permission to create folders under the /var/run path, we will use the /tmp as an alternative KUBERNETES_CLIENT_TOKEN_DIR="/var/run/secrets/kubernetes.io/serviceaccount" +KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token + if ! mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} ;then - KUBERNETES_CLIENT_TOKEN_DIR=/tmp${KUBERNETES_CLIENT_TOKEN_DIR} - mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} + KUBERNETES_CLIENT_TOKEN_DIR=/tmp${KUBERNETES_CLIENT_TOKEN_DIR} + KUBERNETES_CLIENT_TOKEN_FILE=/tmp/${KUBERNETES_CLIENT_TOKEN_FILE} + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} fi -KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}"/token" -echo -n $KUBERNETES_CLIENT_TOKEN_CONTENT > ${KUBERNETES_CLIENT_TOKEN_FILE} - -export KUBERNETES_SERVICE_HOST="127.0.0.1" -export KUBERNETES_SERVICE_PORT="6443" -export KUBERNETES_CLIENT_TOKEN=${KUBERNETES_CLIENT_TOKEN_CONTENT} -export KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_FILE} +if ! echo -n $KUBERNETES_CLIENT_TOKEN_CONTENT > ${KUBERNETES_CLIENT_TOKEN_FILE} ;then + echo 'Save Kubernetes token file error' + exit -1 +fi -echo 'set env KUBERNETES_SERVICE_HOST='${KUBERNETES_SERVICE_HOST} -echo 'set env KUBERNETES_SERVICE_PORT='${KUBERNETES_SERVICE_PORT} -echo 'set env KUBERNETES_CLIENT_TOKEN='${KUBERNETES_CLIENT_TOKEN} -echo 'set env KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} +echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' +echo 'KUBERNETES_SERVICE_PORT=6443' +echo 'KUBERNETES_CLIENT_TOKEN='${KUBERNETES_CLIENT_TOKEN_CONTENT} +echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} cd .. diff --git a/t/APISIX.pm b/t/APISIX.pm index 82742a8133b5..5b960cc87f8b 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -239,11 +239,6 @@ env ENABLE_ETCD_AUTH; env APISIX_PROFILE; env PATH; # for searching external plugin runner's binary env TEST_NGINX_HTML_DIR; -env KUBERNETES_SERVICE_HOST; -env KUBERNETES_SERVICE_PORT; -env KUBERNETES_CLIENT_TOKEN; -env KUBERNETES_CLIENT_TOKEN_FILE; - _EOC_ # set default `timeout` to 5sec diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t index b7aea53c0022..beadc38de19c 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/k8s.t @@ -14,6 +14,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +BEGIN { + $ENV{KUBERNETES_SERVICE_HOST} = "127.0.0.1"; + $ENV{KUBERNETES_SERVICE_PORT} = "6443"; + + my $token_var_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"; + my $token_from_var = eval { `cat ${token_var_file} 2>/dev/null` }; + if ($token_from_var){ + $ENV{KUBERNETES_TOKEN_IN_VAR}="true"; + + $ENV{KUBERNETES_CLIENT_TOKEN}=$token_from_var; + $ENV{KUBERNETES_CLIENT_TOKEN_FILE}=$token_var_file; + }else { + my $token_tmp_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; + my $token_from_tmp = eval { `cat ${token_tmp_file} 2>/dev/null` }; + if ($token_from_tmp) { + $ENV{KUBERNETES_TOKEN_IN_TMP}="true"; + $ENV{KUBERNETES_CLIENT_TOKEN}=$token_from_tmp; + $ENV{KUBERNETES_CLIENT_TOKEN_FILE}=$token_tmp_file; + } + } +} + use t::APISIX 'no_plan'; repeat_each(1); @@ -25,22 +48,32 @@ workers(4); add_block_preprocessor(sub { my ($block) = @_; + my $token_in_var = eval { `echo -n \$KUBERNETES_TOKEN_IN_VAR 2>/dev/null` }; + my $token_in_tmp = eval { `echo -n \$KUBERNETES_TOKEN_IN_TMP 2>/dev/null` }; + my $yaml_config = $block->yaml_config // <<_EOC_; apisix: node_listen: 1984 config_center: yaml enable_admin: false +_EOC_ + + if ($token_in_var eq "true") { + $yaml_config .= <<_EOC_; +discovery: + k8s: {} +_EOC_ + } + + if ($token_in_tmp eq "true") { + $yaml_config .= <<_EOC_; discovery: k8s: client: - token: \$\{KUBERNETES_CLIENT_TOKEN\} -nginx_config: - envs: - - KUBERNETES_SERVICE_HOST - - KUBERNETES_SERVICE_PORT - - KUBERNETES_CLIENT_TOKEN + token_file: /tmp/var/run/secrets/kubernetes.io/serviceaccount/token _EOC_ - + } + $block->set_value("yaml_config", $yaml_config); my $apisix_yaml = $block->apisix_yaml // <<_EOC_; @@ -50,6 +83,14 @@ _EOC_ $block->set_value("apisix_yaml", $apisix_yaml); + my $main_config = $block->main_config // <<_EOC_; +env KUBERNETES_SERVICE_HOST; +env KUBERNETES_SERVICE_PORT; +env KUBERNETES_CLIENT_TOKEN; +env KUBERNETES_CLIENT_TOKEN_FILE; +_EOC_ + + $block->set_value("main_config", $main_config); my $config = $block->config // <<_EOC_; location /t { @@ -80,20 +121,6 @@ run_tests(); __DATA__ === TEST 1: use default parameters ---- yaml_config -apisix: - node_listen: 1984 - config_center: yaml - enable_admin: false -discovery: - k8s: - client: - token: ${KUBERNETES_CLIENT_TOKEN} -nginx_config: - envs: - - KUBERNETES_SERVICE_HOST - - KUBERNETES_SERVICE_PORT - - KUBERNETES_CLIENT_TOKEN --- request GET /t?s=default/kubernetes:https --- response_body From 276705a24b14de7f56305462e9305d1e5d48d025 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Wed, 8 Sep 2021 17:17:09 +0800 Subject: [PATCH 09/73] feat: add k8s discovery module Signed-off-by: adugeek --- ci/install-ext-services-via-docker.sh | 2 +- t/discovery/k8s.t | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/install-ext-services-via-docker.sh b/ci/install-ext-services-via-docker.sh index efa1b176a476..8210460459b5 100755 --- a/ci/install-ext-services-via-docker.sh +++ b/ci/install-ext-services-via-docker.sh @@ -150,7 +150,7 @@ KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token if ! mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} ;then KUBERNETES_CLIENT_TOKEN_DIR=/tmp${KUBERNETES_CLIENT_TOKEN_DIR} - KUBERNETES_CLIENT_TOKEN_FILE=/tmp/${KUBERNETES_CLIENT_TOKEN_FILE} + KUBERNETES_CLIENT_TOKEN_FILE=/tmp${KUBERNETES_CLIENT_TOKEN_FILE} mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} fi diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t index beadc38de19c..4d0b6e1911f8 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/k8s.t @@ -18,12 +18,11 @@ BEGIN { $ENV{KUBERNETES_SERVICE_HOST} = "127.0.0.1"; $ENV{KUBERNETES_SERVICE_PORT} = "6443"; - + my $token_var_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"; my $token_from_var = eval { `cat ${token_var_file} 2>/dev/null` }; if ($token_from_var){ $ENV{KUBERNETES_TOKEN_IN_VAR}="true"; - $ENV{KUBERNETES_CLIENT_TOKEN}=$token_from_var; $ENV{KUBERNETES_CLIENT_TOKEN_FILE}=$token_var_file; }else { @@ -87,7 +86,7 @@ _EOC_ env KUBERNETES_SERVICE_HOST; env KUBERNETES_SERVICE_PORT; env KUBERNETES_CLIENT_TOKEN; -env KUBERNETES_CLIENT_TOKEN_FILE; +env KUBERNETES_CLIENT_TOKEN_FILE; _EOC_ $block->set_value("main_config", $main_config); @@ -99,7 +98,7 @@ _EOC_ ngx.sleep(1) local s = ngx.var.arg_s local nodes = d.nodes(s) - + ngx.status = 200 local body From 9cbe29a8ae90fa0a47ec6011839d42820b580ccb Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Sun, 19 Sep 2021 13:30:00 +0800 Subject: [PATCH 10/73] feat: add k8s discovery module Signed-off-by: adugeek --- apisix/discovery/k8s.lua | 129 ++++++++- t/discovery/k8s.t | 595 +++++++++++++++++++++++++++++++++------ 2 files changed, 626 insertions(+), 98 deletions(-) diff --git a/apisix/discovery/k8s.lua b/apisix/discovery/k8s.lua index 7fe406032a67..d2cfd75398d2 100644 --- a/apisix/discovery/k8s.lua +++ b/apisix/discovery/k8s.lua @@ -35,6 +35,8 @@ local apiserver_schema = "" local apiserver_host = "" local apiserver_port = 0 local apiserver_token = "" +local namespace_selector_string = "" +local namespace_selector_function local default_weight = 0 local endpoint_lrucache = core.lrucache.new({ @@ -50,17 +52,30 @@ local function sort_by_ip(a, b) return a.ip < b.ip end -local function on_endpoint_deleted(endpoint) +local function on_endpoint_deleted(endpoint, selector_check) + if selector_check then + if namespace_selector_function and + not namespace_selector_function(endpoint.metadata.namespace) then + return + end + end + local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name endpoints_shared:delete(endpoint_key .. "#version") endpoints_shared:delete(endpoint_key) end local function on_endpoint_modified(endpoint) + if namespace_selector_function and + not namespace_selector_function(endpoint.metadata.namespace) then + return + end + if endpoint.subsets == nil or #endpoint.subsets == 0 then return on_endpoint_deleted(endpoint) end + core.table.clear(endpoint_cache) local subsets = endpoint.subsets for _, subset in ipairs(subsets) do if subset.addresses ~= nil then @@ -127,6 +142,8 @@ local function list_resource(httpc, resource, continue) } }) + core.log.debug("--raw=" .. resource:list_path() .. "?" .. resource:list_query(continue)) + if not res then return false, "RequestError", err or "" end @@ -173,6 +190,8 @@ local function watch_resource(httpc, resource) } }) + core.log.debug("--raw=" .. resource:watch_path() .. "?" .. resource:watch_query(watch_seconds)) + if err then return false, "RequestError", err end @@ -202,7 +221,7 @@ local function watch_resource(httpc, resource) body = remainder_body .. body end - gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jiao") + gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") if not gmatch_iterator then return false, "GmatchError", err end @@ -332,6 +351,10 @@ local port_patterns = { { pattern = [[^(([1-9]\d{0,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))$]] }, } +local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]] + +local namespace_regex_pattern = [[^[\x21-\x7e]*$]] + local schema = { type = "object", properties = { @@ -390,6 +413,42 @@ local schema = { default = 50, minimum = 0, }, + namespace_selector = { + type = "object", + properties = { + equal = { + type = "string", + pattern = namespace_pattern, + }, + not_equal = { + type = "string", + pattern = namespace_pattern, + }, + match = { + type = "array", + items = { + type = "string", + pattern = namespace_regex_pattern + }, + minItems = 1 + }, + not_match = { + type = "array", + items = { + type = "string", + pattern = namespace_regex_pattern + }, + minItems = 1 + }, + }, + oneOf = { + { required = { } }, + { required = { "equal" } }, + { required = { "not_equal" } }, + { required = { "match" } }, + { required = { "not_match" } } + }, + }, }, default = { service = { @@ -404,7 +463,7 @@ local schema = { } } -local function load_value(key) +local function load_api_context(key) if #key > 3 then local a, b = string.byte(key, 1, 2) local c = string.byte(key, #key, #key) @@ -421,11 +480,57 @@ local function load_value(key) return true, key, nil end +local function build_namespace_selector(conf) + namespace_selector_string = "" + namespace_selector_function = nil + + if conf.namespace_selector == nil then + return + end + + local ns = conf.namespace_selector + if ns.equal then + namespace_selector_string = "&fieldSelector=metadata.namespace%3D" .. ns.equal + elseif ns.not_equal then + namespace_selector_string = "&fieldSelector=metadata.namespace%21%3D" .. ns.not_equal + elseif ns.match then + namespace_selector_function = function(namespace) + local match = conf.namespace_selector.match + local m, err + for _, v in ipairs(match) do + m, err = ngx.re.match(namespace, v, "j") + if m and m[0] == namespace then + return true + end + if err then + core.log.error("ngx.re.match failed: ", err) + end + end + return false + end + elseif ns.not_match then + namespace_selector_function = function(namespace) + local not_match = conf.namespace_selector.not_match + local m, err + for _, v in ipairs(not_match) do + m, err = ngx.re.match(namespace, v, "j") + if m and m[0] == namespace then + return false + end + if err then + return false + end + end + return true + end + end +end + local function read_conf(conf) apiserver_schema = conf.service.schema local ok, value, message - ok, value, message = load_value(conf.service.host) + ok, value, message = load_api_context(conf.service.host) if not ok then return false, message end @@ -434,7 +539,7 @@ local function read_conf(conf) return false, "get empty host value" end - ok, value, message = load_value(conf.service.port) + ok, value, message = load_api_context(conf.service.port) if not ok then return false, message end @@ -445,13 +550,13 @@ local function read_conf(conf) -- we should not check if the apiserver_token is empty here if conf.client.token then - ok, value, message = load_value(conf.client.token) + ok, value, message = load_api_context(conf.client.token) if not ok then return false, message end apiserver_token = value elseif conf.client.token_file and conf.client.token_file ~= "" then - ok, value, message = load_value(conf.client.token_file) + ok, value, message = load_api_context(conf.client.token_file) if not ok then return false, message end @@ -468,6 +573,8 @@ local function read_conf(conf) default_weight = conf.default_weight or 50 + build_namespace_selector(conf) + return true, nil end @@ -477,7 +584,7 @@ local _M = { function _M.nodes(service_name) local pattern = "^(.*):(.*)$" - local match, _ = ngx.re.match(service_name, pattern, "jiao") + local match, _ = ngx.re.match(service_name, pattern, "jo") if not match then core.log.info("get unexpected upstream service_name:　", service_name) return nil @@ -517,9 +624,9 @@ local function fill_pending_resources() list_query = function(self, continue) if continue == nil or continue == "" then - return "limit=32" + return "limit=32" .. namespace_selector_string else - return "limit=32&continue=" .. continue + return "limit=32" .. namespace_selector_string .. "&continue=" .. continue end end, @@ -529,7 +636,7 @@ local function fill_pending_resources() watch_query = function(self, timeout) return "watch=true&allowWatchBookmarks=true&timeoutSeconds=" .. timeout .. - "&resourceVersion=" .. self.newest_resource_version + "&resourceVersion=" .. self.newest_resource_version .. namespace_selector_string end, pre_list_callback = function(self) diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t index 4d0b6e1911f8..6120f7d848b5 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/k8s.t @@ -22,24 +22,69 @@ BEGIN { my $token_var_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"; my $token_from_var = eval { `cat ${token_var_file} 2>/dev/null` }; if ($token_from_var){ - $ENV{KUBERNETES_TOKEN_IN_VAR}="true"; + + our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: {} +_EOC_ + $ENV{KUBERNETES_CLIENT_TOKEN}=$token_from_var; $ENV{KUBERNETES_CLIENT_TOKEN_FILE}=$token_var_file; }else { my $token_tmp_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; my $token_from_tmp = eval { `cat ${token_tmp_file} 2>/dev/null` }; if ($token_from_tmp) { - $ENV{KUBERNETES_TOKEN_IN_TMP}="true"; + + our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: /tmp/var/run/secrets/kubernetes.io/serviceaccount/token +_EOC_ $ENV{KUBERNETES_CLIENT_TOKEN}=$token_from_tmp; $ENV{KUBERNETES_CLIENT_TOKEN_FILE}=$token_tmp_file; } } + + our $scale_ns_c = <<_EOC_; +[ + { + "op": "replace_endpoints", + "name": "ep", + "namespace": "ns-c", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + } + ] + } +] +_EOC_ + } use t::APISIX 'no_plan'; repeat_each(1); -log_level('info'); +log_level('debug'); no_root_location(); no_shuffle(); workers(4); @@ -47,34 +92,6 @@ workers(4); add_block_preprocessor(sub { my ($block) = @_; - my $token_in_var = eval { `echo -n \$KUBERNETES_TOKEN_IN_VAR 2>/dev/null` }; - my $token_in_tmp = eval { `echo -n \$KUBERNETES_TOKEN_IN_TMP 2>/dev/null` }; - - my $yaml_config = $block->yaml_config // <<_EOC_; -apisix: - node_listen: 1984 - config_center: yaml - enable_admin: false -_EOC_ - - if ($token_in_var eq "true") { - $yaml_config .= <<_EOC_; -discovery: - k8s: {} -_EOC_ - } - - if ($token_in_tmp eq "true") { - $yaml_config .= <<_EOC_; -discovery: - k8s: - client: - token_file: /tmp/var/run/secrets/kubernetes.io/serviceaccount/token -_EOC_ - } - - $block->set_value("yaml_config", $yaml_config); - my $apisix_yaml = $block->apisix_yaml // <<_EOC_; routes: [] #END @@ -92,24 +109,107 @@ _EOC_ $block->set_value("main_config", $main_config); my $config = $block->config // <<_EOC_; - location /t { + location /queries { content_by_lua_block { + local core = require("apisix.core") local d = require("apisix.discovery.k8s") - ngx.sleep(1) - local s = ngx.var.arg_s - local nodes = d.nodes(s) - ngx.status = 200 - local body + ngx.sleep(1) - if nodes == nil or #nodes == 0 then - body="empty" - else - body="passed" + ngx.req.read_body() + local request_body = ngx.req.get_body_data() + local queries = core.json.decode(request_body) + local response_body = "{" + for _,query in ipairs(queries) do + local nodes = d.nodes(query) + if nodes==nil or #nodes==0 then + response_body=response_body.." "..0 + else + response_body=response_body.." "..#nodes + end end - ngx.say(body) + ngx.say(response_body.." }") } } + + location /operators { + content_by_lua_block { + local http = require("resty.http") + local core = require("apisix.core") + local ipairs = ipairs + + ngx.req.read_body() + local request_body = ngx.req.get_body_data() + local operators = core.json.decode(request_body) + + core.log.info("get body ", request_body) + core.log.info("get operators ", #operators) + for _, op in ipairs(operators) do + local method, path, body + local headers = { + ["Host"] = "127.0.0.1:6445" + } + + if op.op == "create_namespace" then + method = "POST" + path = "/api/v1/namespaces" + local t = { metadata = { name = op.name } } + body = core.json.encode(t, true) + headers["Content-Type"] = "application/json" + end + + if op.op == "create_endpoints" then + method = "POST" + path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints" + local t = { + metadata = { name = op.name, namespace = op.namespace }, + subsets = op.subsets, + } + body = core.json.encode(t, true) + headers["Content-Type"] = "application/json" + end + + if op.op == "replace_endpoints" then + method = "PATCH" + path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints/" .. op.name + if #op.subsets == 0 then + body = '[{"path":"/subsets","op":"replace","value":[]}]' + else + local t = { { op = "replace", path = "/subsets", value = op.subsets } } + body = core.json.encode(t, true) + end + headers["Content-Type"] = "application/json-patch+json" + end + + local httpc = http.new() + core.log.info("begin to connect ", "127.0.0.1:6445") + local ok, message = httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = 6445, + }) + if not ok then + core.log.error("connect 127.0.0.1:6445 failed, message : ", message) + ngx.say("FAILED") + end + local res, err = httpc:request({ + method = method, + path = path, + headers = headers, + body = body, + }) + if err ~= nil then + core.log.err("operator k8s cluster error: ", err) + return 500 + end + if res.status ~= 200 and res.status ~= 201 and res.status ~= 409 then + return res.status + end + end + ngx.say("DONE") + } + } + _EOC_ $block->set_value("config", $config); @@ -119,15 +219,162 @@ run_tests(); __DATA__ -=== TEST 1: use default parameters +=== TEST 1: create namespace and endpoints +--- yaml_config eval: $::yaml_config --- request -GET /t?s=default/kubernetes:https ---- response_body -passed +POST /operators +[ + { + "op": "create_namespace", + "name": "ns-a" + }, + { + "op": "create_endpoints", + "namespace": "ns-a", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "name": "p2", + "port": 5002 + } + ] + } + ] + }, + { + "op": "create_namespace", + "name": "ns-b" + }, + { + "op": "create_endpoints", + "namespace": "ns-b", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "name": "p2", + "port": 5002 + } + ] + } + ] + }, + { + "op": "create_namespace", + "name": "ns-c" + }, + { + "op": "create_endpoints", + "namespace": "ns-c", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "port": 5002 + } + ] + } + ] + } +] +--- more_headers +Content-type: application/json +--- error_code: 200 +--- no_error_log +[error] + + + +=== TEST 2: use default parameters +--- yaml_config eval: $::yaml_config +--- request +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } +--- no_error_log +[error] -=== TEST 2: use specify parameters +=== TEST 3: use specify parameters --- yaml_config apisix: node_listen: 1984 @@ -140,19 +387,19 @@ discovery: port: "6443" client: token: "${KUBERNETES_CLIENT_TOKEN}" -nginx_config: - envs: - - KUBERNETES_CLIENT_TOKEN --- request -GET /t?s=default/kubernetes:https ---- response_body -passed +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } --- no_error_log [error] -=== TEST 3: use specify environment parameters +=== TEST 4: use specify environment parameters --- yaml_config apisix: node_listen: 1984 @@ -165,21 +412,19 @@ discovery: port: ${KUBERNETES_SERVICE_PORT} client: token: ${KUBERNETES_CLIENT_TOKEN} -nginx_config: - envs: - - KUBERNETES_SERVICE_HOST - - KUBERNETES_SERVICE_PORT - - KUBERNETES_CLIENT_TOKEN --- request -GET /t?s=default/kubernetes:https ---- response_body -passed +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } --- no_error_log [error] -=== TEST 4: use token_file +=== TEST 5: use token_file --- yaml_config apisix: node_listen: 1984 @@ -189,21 +434,19 @@ discovery: k8s: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} -nginx_config: - envs: - - KUBERNETES_SERVICE_HOST - - KUBERNETES_SERVICE_PORT - - KUBERNETES_CLIENT_TOKEN_FILE --- request -GET /t?s=default/kubernetes:https ---- response_body -passed +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } --- no_error_log [error] -=== TEST 5: use http +=== TEST 6: use http --- yaml_config apisix: node_listen: 1984 @@ -218,38 +461,216 @@ discovery: client: token: "" --- request -GET /t?s=default/kubernetes:https ---- response_body -passed +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } --- no_error_log [error] -=== TEST 6: error service_name - bad namespace +=== TEST 7: use namespace selector equal +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} + namespace_selector: + equal: ns-a --- request -GET /t?s=notexist/kubernetes:https ---- response_body -empty +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 0 0 0 0 } --- no_error_log [error] -=== TEST 7: error service_name - bad service +=== TEST 8: use namespace selector not_equal +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} + namespace_selector: + not_equal: ns-a --- request -GET /t?s=default/notexist:https ---- response_body -empty +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 0 0 2 2 2 2 } --- no_error_log [error] -=== TEST 8: error service_name - bad port +=== TEST 9: use namespace selector match +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} + namespace_selector: + match: [ns-a,ns-b] +--- request +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 0 0 } +--- no_error_log +[error] + + + +=== TEST 10: use namespace selector match with regex +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} + namespace_selector: + match: ["ns-[ab]"] +--- request +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 0 0 } +--- no_error_log +[error] + + + +=== TEST 11: use namespace selector not_match +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} + namespace_selector: + not_match: ["ns-a"] --- request -GET /t?s=default/kubernetes:notexist ---- response_body -empty +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 0 0 2 2 2 2 } +--- no_error_log +[error] + + + +=== TEST 12: use namespace selector not_match with regex +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} + namespace_selector: + not_match: ["ns-[ab]"] +--- request +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 0 0 0 0 2 2 } +--- no_error_log +[error] + + + +=== TEST 13: use namespace selector not_match with regex +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + k8s: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} + namespace_selector: + not_match: ["ns-[ab]"] +--- request +GET /queries +["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 0 0 0 0 2 2 } +--- no_error_log +[error] + + + +=== TEST 14: scale endpoints +--- yaml_config eval: $::yaml_config +--- request eval +[ +"GET /queries +[\"ns-a/ep:p1\",\"ns-a/ep:p2\"]", + +"POST /operators +[{\"op\":\"replace_endpoints\",\"name\":\"ep\",\"namespace\":\"ns-a\",\"subsets\":[]}]", + +"GET /queries +[\"ns-a/ep:p1\",\"ns-a/ep:p2\"]", + +"GET /queries +[\"ns-c/ep:5001\",\"ns-c/ep:5002\",\"ns-c/ep:p1\"]", + +"POST /operators +$::scale_ns_c", + +"GET /queries +[\"ns-c/ep:5001\",\"ns-c/ep:5002\",\"ns-c/ep:p1\"]", + +] +--- response_body eval +[ + "{ 2 2 }\n", + "DONE\n", + "{ 0 0 }\n", + "{ 2 2 0 }\n", + "DONE\n", + "{ 0 0 1 }\n", +] --- no_error_log [error] From a1d9b8e0a5ba6191da3aaecbecc4441f3da8e6b6 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Sun, 19 Sep 2021 15:55:18 +0800 Subject: [PATCH 11/73] feat: add k8s discovery module Signed-off-by: adugeek --- t/discovery/k8s.t | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/t/discovery/k8s.t b/t/discovery/k8s.t index 6120f7d848b5..ec1263900e99 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/k8s.t @@ -16,11 +16,8 @@ # BEGIN { - $ENV{KUBERNETES_SERVICE_HOST} = "127.0.0.1"; - $ENV{KUBERNETES_SERVICE_PORT} = "6443"; - my $token_var_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"; - my $token_from_var = eval { `cat ${token_var_file} 2>/dev/null` }; + my $token_from_var = eval { `cat $token_var_file 2>/dev/null` }; if ($token_from_var){ our $yaml_config = <<_EOC_; @@ -31,15 +28,16 @@ apisix: discovery: k8s: {} _EOC_ + our $token_file = $token_var_file; + our $token_value = $token_from_var; + + } - $ENV{KUBERNETES_CLIENT_TOKEN}=$token_from_var; - $ENV{KUBERNETES_CLIENT_TOKEN_FILE}=$token_var_file; - }else { - my $token_tmp_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; - my $token_from_tmp = eval { `cat ${token_tmp_file} 2>/dev/null` }; - if ($token_from_tmp) { + my $token_tmp_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; + my $token_from_tmp = eval { `cat $token_tmp_file 2>/dev/null` }; + if ($token_from_tmp) { - our $yaml_config = <<_EOC_; + our $yaml_config = <<_EOC_; apisix: node_listen: 1984 config_center: yaml @@ -49,9 +47,8 @@ discovery: client: token_file: /tmp/var/run/secrets/kubernetes.io/serviceaccount/token _EOC_ - $ENV{KUBERNETES_CLIENT_TOKEN}=$token_from_tmp; - $ENV{KUBERNETES_CLIENT_TOKEN_FILE}=$token_tmp_file; - } + our $token_file = $token_tmp_file; + our $token_value = $token_from_tmp; } our $scale_ns_c = <<_EOC_; @@ -100,10 +97,10 @@ _EOC_ $block->set_value("apisix_yaml", $apisix_yaml); my $main_config = $block->main_config // <<_EOC_; -env KUBERNETES_SERVICE_HOST; -env KUBERNETES_SERVICE_PORT; -env KUBERNETES_CLIENT_TOKEN; -env KUBERNETES_CLIENT_TOKEN_FILE; +env KUBERNETES_SERVICE_HOST=127.0.0.1; +env KUBERNETES_SERVICE_PORT=6443; +env KUBERNETES_CLIENT_TOKEN=$::token_value; +env KUBERNETES_CLIENT_TOKEN_FILE=$::token_file; _EOC_ $block->set_value("main_config", $main_config); From 35ad63947f2a62f834493971f3ccdf0349b6834d Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Sun, 19 Sep 2021 19:50:23 +0800 Subject: [PATCH 12/73] feat: add k8s discovery module Signed-off-by: adugeek --- .github/workflows/centos7-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index cb387470267e..7ec2c02543fa 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -84,7 +84,7 @@ jobs: - name: Run centos7 docker and mapping apisix into container run: | - docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" docker.io/centos:7 /bin/bash + docker run -itd -v /home/runner/work/apisix/apisix:/apisix -v /tmp:/tmp --name centos7Instance --net="host" docker.io/centos:7 /bin/bash # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" - name: Run other docker containers for test From f66573455d8a98114aa953e9d8500082dd148e30 Mon Sep 17 00:00:00 2001 From: zhixiogndu Date: Mon, 20 Sep 2021 16:29:50 +0800 Subject: [PATCH 13/73] feat: add k8s discovery module Signed-off-by: adugeek --- apisix/discovery/k8s.lua | 19 +++++++---- ci/install-ext-services-via-docker.sh | 49 ++++++++++++++------------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/apisix/discovery/k8s.lua b/apisix/discovery/k8s.lua index d2cfd75398d2..60779b31ebaf 100644 --- a/apisix/discovery/k8s.lua +++ b/apisix/discovery/k8s.lua @@ -31,6 +31,11 @@ local local_conf = require("apisix.core.config_local").local_conf() local http = require("resty.http") local endpoints_shared = ngx.shared.discovery +local AddedEventType = "ADDED" +local ModifiedEventType = "MODIFIED" +local DeletedEventType = "DELETED" +local BookmarkEventType = "BOOKMARK" + local apiserver_schema = "" local apiserver_host = "" local apiserver_port = 0 @@ -72,7 +77,7 @@ local function on_endpoint_modified(endpoint) end if endpoint.subsets == nil or #endpoint.subsets == 0 then - return on_endpoint_deleted(endpoint) + return on_endpoint_deleted(endpoint, false) end core.table.clear(endpoint_cache) @@ -164,7 +169,7 @@ local function list_resource(httpc, resource, continue) resource.newest_resource_version = data.metadata.resourceVersion for _, item in ipairs(data.items or empty_table) do - resource:event_dispatch("ADDED", item, "list") + resource:event_dispatch(AddedEventType, item, "list") end if data.metadata.continue ~= nil and data.metadata.continue ~= "" then @@ -241,7 +246,7 @@ local function watch_resource(httpc, resource) end resource.newest_resource_version = v.object.metadata.resource_version - if v.type ~= "BOOKMARK" then + if v.type ~= BookmarkEventType then resource:event_dispatch(v.type, v.object, "watch") end end @@ -657,18 +662,18 @@ local function fill_pending_resources() end, deleted_callback = function(self, object) - on_endpoint_deleted(object) + on_endpoint_deleted(object, true) end, event_dispatch = function(self, event, object, drive) - if event == "DELETED" or object.deletionTimestamp ~= nil then + if event == DeletedEventType or object.deletionTimestamp ~= nil then self:deleted_callback(object) return end - if event == "ADDED" then + if event == AddedEventType then self:added_callback(object, drive) - elseif event == "MODIFIED" then + elseif event == ModifiedEventType then self:modified_callback(object) end end, diff --git a/ci/install-ext-services-via-docker.sh b/ci/install-ext-services-via-docker.sh index eeacf36db107..7c46b47980d9 100755 --- a/ci/install-ext-services-via-docker.sh +++ b/ci/install-ext-services-via-docker.sh @@ -88,19 +88,29 @@ until [[ $(curl -s "127.0.0.1:8858/nacos/v1/ns/service/list?groupName=test_grou done # create kubernetes cluster using kind +curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.11.1/kind-$(uname)-amd64" +curl -Lo ./kubectl "https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl" +chmod +x ./kind +chmod +x ./kubectl + echo -e " kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: - apiServerAddress: "127.0.0.1" + apiServerAddress: 127.0.0.1 apiServerPort: 6443 -" > kind.yaml +" >kind.yaml -curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.11.1/kind-$(uname)-amd64" -chmod +x ./kind ./kind delete cluster --name apisix-test ./kind create cluster --name apisix-test --config ./kind.yaml +echo "wait k8s start..." +sleep 10 +until [[ $(./kubectl get pods -A --field-selector 'status.phase!=Running' 2>&1) =~ "No resources found" ]]; do + echo 'still wait k8s start...' + sleep 1 +done + echo -e " kind: ServiceAccount apiVersion: v1 @@ -108,6 +118,7 @@ metadata: name: apisix-test namespace: default --- + kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -130,41 +141,31 @@ subjects: - kind: ServiceAccount name: apisix-test namespace: default -" > apisix-test-rbac.yaml +" >apisix-test-rbac.yaml -curl -Lo ./kubectl "https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl" -chmod +x ./kubectl ./kubectl apply -f ./apisix-test-rbac.yaml ./kubectl proxy -p 6445 & -curl -Lo ./jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -chmod +x ./jq - -until [[ $(curl 127.0.0.1:6445/api/v1/pods?fieldSelector=status.phase%21%3DRunning |./jq .items) == "[]" ]]; do - echo 'wait k8s start...' - sleep 1; -done - -KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o json "$1" |./jq -r .data.token | base64 --decode")}') +KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') # if we do not have permission to create folders under the /var/run path, we will use the /tmp as an alternative KUBERNETES_CLIENT_TOKEN_DIR="/var/run/secrets/kubernetes.io/serviceaccount" KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token -if ! mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} ;then - KUBERNETES_CLIENT_TOKEN_DIR=/tmp${KUBERNETES_CLIENT_TOKEN_DIR} - KUBERNETES_CLIENT_TOKEN_FILE=/tmp${KUBERNETES_CLIENT_TOKEN_FILE} - mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} +if ! mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR}; then + KUBERNETES_CLIENT_TOKEN_DIR=/tmp${KUBERNETES_CLIENT_TOKEN_DIR} + KUBERNETES_CLIENT_TOKEN_FILE=/tmp${KUBERNETES_CLIENT_TOKEN_FILE} + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} fi -if ! echo -n $KUBERNETES_CLIENT_TOKEN_CONTENT > ${KUBERNETES_CLIENT_TOKEN_FILE} ;then - echo 'Save Kubernetes token file error' - exit -1 +if ! echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" >${KUBERNETES_CLIENT_TOKEN_FILE}; then + echo 'save kubernetes token file error' + exit -1 fi echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' echo 'KUBERNETES_SERVICE_PORT=6443' -echo 'KUBERNETES_CLIENT_TOKEN='${KUBERNETES_CLIENT_TOKEN_CONTENT} +echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} cd .. From 55ebf1cb1434624281d6029b1517df67b8a26713 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 10 Jan 2022 18:06:55 +0800 Subject: [PATCH 14/73] Merge remote-tracking branch 'upstream/master' # Conflicts: # .github/workflows/centos7-ci.yml # ci/install-ext-services-via-docker.sh --- apisix/discovery/{k8s.lua => k8s/init.lua} | 155 ++------------------- apisix/discovery/k8s/schema.lua | 137 ++++++++++++++++++ ci/linux-ci-init-service.sh | 36 +++++ 3 files changed, 184 insertions(+), 144 deletions(-) rename apisix/discovery/{k8s.lua => k8s/init.lua} (78%) create mode 100644 apisix/discovery/k8s/schema.lua create mode 100644 ci/linux-ci-init-service.sh diff --git a/apisix/discovery/k8s.lua b/apisix/discovery/k8s/init.lua similarity index 78% rename from apisix/discovery/k8s.lua rename to apisix/discovery/k8s/init.lua index 4c1f2f7a1a82..4366a5f6a029 100644 --- a/apisix/discovery/k8s.lua +++ b/apisix/discovery/k8s/init.lua @@ -151,39 +151,39 @@ local function create_resource(group, version, kind, plural, namespace) _t.path = _t.path .. "/" .. plural function _t.list_query(self) - local query = "limit=" .. self.limit + local uri = "limit=" .. self.limit if self.continue ~= nil and self.continue ~= "" then - query = query .. "&continue=" .. self.continue + uri = uri .. "&continue=" .. self.continue end if self.label_selector and self.label_selector ~= "" then - query = query .. "&labelSelector=" .. self.label_selector + uri = uri .. "&labelSelector=" .. self.label_selector end if self.field_selector and self.field_selector ~= "" then - query = query .. "&filedSelector=" .. self.field_selector + uri = uri .. "&filedSelector=" .. self.field_selector end - return query + return uri end function _t.watch_query(self) - local query = "watch=true&allowWatchBookmarks=true&timeoutSeconds=" .. self.overtime + local uri = "watch=true&allowWatchBookmarks=true&timeoutSeconds=" .. self.overtime if self.version ~= nil and self.version ~= "" then - query = query .. "&resourceVersion=" .. self.version + uri = uri .. "&resourceVersion=" .. self.version end if self.label_selector and self.label_selector ~= "" then - query = query .. "&labelSelector=" .. self.label_selector + uri = uri .. "&labelSelector=" .. self.label_selector end if self.field_selector and self.field_selector ~= "" then - query = query .. "&filedSelector=" .. self.field_selector + uri = uri .. "&filedSelector=" .. self.field_selector end - return query + return uri end return _t @@ -389,126 +389,6 @@ local function fetch_resource(resource) end end -local host_patterns = { - { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, - { pattern = [[^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$]] }, -} - -local port_patterns = { - { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, - { pattern = [[^(([1-9]\d{0,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))$]] }, -} - -local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]] -local namespace_regex_pattern = [[^[\x21-\x7e]*$]] -local schema = { - type = "object", - properties = { - service = { - type = "object", - properties = { - schema = { - type = "string", - enum = { "http", "https" }, - default = "https", - }, - host = { - type = "string", - default = "${KUBERNETES_SERVICE_HOST}", - oneOf = host_patterns, - }, - port = { - type = "string", - default = "${KUBERNETES_SERVICE_PORT}", - oneOf = port_patterns, - }, - }, - default = { - schema = "https", - host = "${KUBERNETES_SERVICE_HOST}", - port = "${KUBERNETES_SERVICE_PORT}", - } - }, - client = { - type = "object", - properties = { - token = { - type = "string", - oneOf = { - { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, - { pattern = [[^[A-Za-z0-9+\/._=-]{0,4096}$]] }, - }, - }, - token_file = { - type = "string", - pattern = [[^[^\:*?"<>|]*$]], - minLength = 1, - maxLength = 500, - } - }, - oneOf = { - { required = { "token" } }, - { required = { "token_file" } }, - }, - default = { - token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - } - }, - default_weight = { - type = "integer", - default = 50, - minimum = 0, - }, - namespace_selector = { - type = "object", - properties = { - equal = { - type = "string", - pattern = namespace_pattern, - }, - not_equal = { - type = "string", - pattern = namespace_pattern, - }, - match = { - type = "array", - items = { - type = "string", - pattern = namespace_regex_pattern - }, - minItems = 1 - }, - not_match = { - type = "array", - items = { - type = "string", - pattern = namespace_regex_pattern - }, - minItems = 1 - }, - }, - oneOf = { - { required = { } }, - { required = { "equal" } }, - { required = { "not_equal" } }, - { required = { "match" } }, - { required = { "not_match" } } - }, - }, - }, - default = { - service = { - schema = "https", - host = "${KUBERNETES_SERVICE_HOST}", - port = "${KUBERNETES_SERVICE_PORT}", - }, - client = { - token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - }, - default_weight = 50 - } -} - local function set_namespace_selector(conf, resource) local ns = conf.namespace_selector if ns == nil then @@ -670,20 +550,7 @@ function _M.init_worker() return end - if not local_conf.discovery.k8s then - error("does not set k8s discovery configuration") - return - end - - core.log.info("k8s discovery configuration: ", core.json.encode(local_conf.discovery.k8s, true)) - - local ok, err = core.schema.check(schema, local_conf.discovery.k8s) - if not ok then - error("invalid k8s discovery configuration: " .. err) - return - end - - ok, err = read_conf(local_conf.discovery.k8s) + local ok, err = read_conf(local_conf.discovery.k8s) if not ok then error(err) return diff --git a/apisix/discovery/k8s/schema.lua b/apisix/discovery/k8s/schema.lua new file mode 100644 index 000000000000..8bc32910eaef --- /dev/null +++ b/apisix/discovery/k8s/schema.lua @@ -0,0 +1,137 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local host_patterns = { + { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, + { pattern = [[^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$]] }, +} + +local port_patterns = { + { pattern = [[^\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, + { pattern = [[^(([1-9]\d{0,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))$]] }, +} + +local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]] +local namespace_regex_pattern = [[^[\x21-\x7e]*$]] + +return { + type = "object", + properties = { + service = { + type = "object", + properties = { + schema = { + type = "string", + enum = { "http", "https" }, + default = "https", + }, + host = { + type = "string", + default = "${KUBERNETES_SERVICE_HOST}", + oneOf = host_patterns, + }, + port = { + type = "string", + default = "${KUBERNETES_SERVICE_PORT}", + oneOf = port_patterns, + }, + }, + default = { + schema = "https", + host = "${KUBERNETES_SERVICE_HOST}", + port = "${KUBERNETES_SERVICE_PORT}", + } + }, + client = { + type = "object", + properties = { + token = { + type = "string", + oneOf = { + { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] }, + { pattern = [[^[A-Za-z0-9+\/._=-]{0,4096}$]] }, + }, + }, + token_file = { + type = "string", + pattern = [[^[^\:*?"<>|]*$]], + minLength = 1, + maxLength = 500, + } + }, + oneOf = { + { required = { "token" } }, + { required = { "token_file" } }, + }, + default = { + token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + }, + default_weight = { + type = "integer", + default = 50, + minimum = 0, + }, + namespace_selector = { + type = "object", + properties = { + equal = { + type = "string", + pattern = namespace_pattern, + }, + not_equal = { + type = "string", + pattern = namespace_pattern, + }, + match = { + type = "array", + items = { + type = "string", + pattern = namespace_regex_pattern + }, + minItems = 1 + }, + not_match = { + type = "array", + items = { + type = "string", + pattern = namespace_regex_pattern + }, + minItems = 1 + }, + }, + oneOf = { + { required = { } }, + { required = { "equal" } }, + { required = { "not_equal" } }, + { required = { "match" } }, + { required = { "not_match" } } + }, + }, + }, + default = { + service = { + schema = "https", + host = "${KUBERNETES_SERVICE_HOST}", + port = "${KUBERNETES_SERVICE_PORT}", + }, + client = { + token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + }, + default_weight = 50 + } +} diff --git a/ci/linux-ci-init-service.sh b/ci/linux-ci-init-service.sh new file mode 100644 index 000000000000..41fc435edd5f --- /dev/null +++ b/ci/linux-ci-init-service.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +docker exec -i apache-apisix_kafka-server1_1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 1 --topic test2 +docker exec -i apache-apisix_kafka-server1_1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 3 --topic test3 +docker exec -i apache-apisix_kafka-server2_1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server2:2181 --replication-factor 1 --partitions 1 --topic test4 + +# prepare openwhisk env +docker pull openwhisk/action-nodejs-v14:nightly +docker run --rm -d --name openwhisk -p 3233:3233 -p 3232:3232 -v /var/run/docker.sock:/var/run/docker.sock openwhisk/standalone:nightly +docker exec -i openwhisk waitready +docker exec -i openwhisk bash -c "wsk action update test <(echo 'function main(args){return {\"hello\":args.name || \"test\"}}') --kind nodejs:14" + +docker exec -i rmqnamesrv rm /home/rocketmq/rocketmq-4.6.0/conf/tools.yml +docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic -n rocketmq_namesrv:9876 -t test -c DefaultCluster +docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic -n rocketmq_namesrv:9876 -t test2 -c DefaultCluster +docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic -n rocketmq_namesrv:9876 -t test3 -c DefaultCluster +docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic -n rocketmq_namesrv:9876 -t test4 -c DefaultCluster + +# prepare vault kv engine +docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault secrets enable -path=kv -version=1 kv" \ No newline at end of file From 3b6eac2cc8a113507c02e3a101eae80d91cfa79c Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 10 Jan 2022 18:09:13 +0800 Subject: [PATCH 15/73] Merge remote-tracking branch 'upstream/master' # Conflicts: # .github/workflows/centos7-ci.yml # ci/install-ext-services-via-docker.sh --- ci/linux-ci-init-service.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/linux-ci-init-service.sh b/ci/linux-ci-init-service.sh index 41fc435edd5f..6a7ffbbfb9b2 100644 --- a/ci/linux-ci-init-service.sh +++ b/ci/linux-ci-init-service.sh @@ -33,4 +33,4 @@ docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic -n rocketmq_namesrv:9876 -t test4 -c DefaultCluster # prepare vault kv engine -docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault secrets enable -path=kv -version=1 kv" \ No newline at end of file +docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault secrets enable -path=kv -version=1 kv" From ef8306eb127a262f24ef5a8fcba5f39f94702bfc Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 10 Jan 2022 18:14:11 +0800 Subject: [PATCH 16/73] Merge remote-tracking branch 'upstream/master' # Conflicts: # .github/workflows/centos7-ci.yml # ci/install-ext-services-via-docker.sh --- .github/workflows/centos7-ci.yml | 73 ++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/centos7-ci.yml diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml new file mode 100644 index 000000000000..9042fca23f07 --- /dev/null +++ b/.github/workflows/centos7-ci.yml @@ -0,0 +1,73 @@ +name: CI Centos7 + +on: + push: + branches: [master, 'release/**'] + paths-ignore: + - 'docs/**' + - '**/*.md' + pull_request: + branches: [master, 'release/**'] + paths-ignore: + - 'docs/**' + - '**/*.md' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} + cancel-in-progress: true + +jobs: + test_apisix: + name: run ci on centos7 + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: Check out code + uses: actions/checkout@v2.4.0 + with: + submodules: recursive + + - name: Extract branch name + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + id: branch_env + shell: bash + run: | + echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})" + - name: Linux launch common services + run: | + make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml + - name: Build rpm package + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + run: | + export VERSION=${{ steps.branch_env.outputs.version }} + sudo gem install --no-document fpm + git clone -b v2.6.0 https://github.com/api7/apisix-build-tools.git + # move codes under build tool + mkdir ./apisix-build-tools/apisix + for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done + cd apisix-build-tools + make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix + cd .. + rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) + - name: Run centos7 docker and mapping apisix into container + run: | + docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 --dns-search apache.org docker.io/centos:7 /bin/bash + # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" + - name: Run other docker containers for test + run: | + make ci-env-up + ./ci/linux-ci-init-service.sh + - name: Install dependencies + run: | + docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh install_dependencies" + - name: Install rpm package + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + run: | + docker exec centos7Instance bash -c "cd apisix && rpm -iv --prefix=/apisix ./apisix-build-tools/output/apisix-${{ steps.branch_env.outputs.version }}-0.el7.x86_64.rpm" + # Dependencies are attached with rpm, so revert `make deps` + docker exec centos7Instance bash -c "cd apisix && rm -rf deps" + docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." + - name: Run test cases + run: | + docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh run_case" From 8b39f7398abc30bff22138b8ea1860affeaf67d1 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 10 Jan 2022 18:15:57 +0800 Subject: [PATCH 17/73] Merge remote-tracking branch 'upstream/master' # Conflicts: # .github/workflows/centos7-ci.yml # ci/install-ext-services-via-docker.sh --- .github/workflows/centos7-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 9042fca23f07..dcd870fe047f 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -52,7 +52,7 @@ jobs: rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) - name: Run centos7 docker and mapping apisix into container run: | - docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 --dns-search apache.org docker.io/centos:7 /bin/bash + docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 --dns-search apache.org -v /tmp:/tmp docker.io/centos:7 /bin/bash # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" - name: Run other docker containers for test run: | From 004d99d63729bca47eee45156e47bcbcaeeef908 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 10 Jan 2022 18:20:12 +0800 Subject: [PATCH 18/73] Merge remote-tracking branch 'upstream/master' # Conflicts: # .github/workflows/centos7-ci.yml # ci/install-ext-services-via-docker.sh --- .github/workflows/centos7-ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index dcd870fe047f..70150a925c70 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -34,33 +34,41 @@ jobs: shell: bash run: | echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})" + - name: Linux launch common services run: | make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml + - name: Build rpm package if: ${{ startsWith(github.ref, 'refs/heads/release/') }} run: | export VERSION=${{ steps.branch_env.outputs.version }} sudo gem install --no-document fpm git clone -b v2.6.0 https://github.com/api7/apisix-build-tools.git + # move codes under build tool mkdir ./apisix-build-tools/apisix for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done + cd apisix-build-tools make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix cd .. rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) + - name: Run centos7 docker and mapping apisix into container run: | docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 --dns-search apache.org -v /tmp:/tmp docker.io/centos:7 /bin/bash # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" + - name: Run other docker containers for test run: | make ci-env-up ./ci/linux-ci-init-service.sh + - name: Install dependencies run: | docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh install_dependencies" + - name: Install rpm package if: ${{ startsWith(github.ref, 'refs/heads/release/') }} run: | @@ -68,6 +76,7 @@ jobs: # Dependencies are attached with rpm, so revert `make deps` docker exec centos7Instance bash -c "cd apisix && rm -rf deps" docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." + - name: Run test cases run: | docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh run_case" From 3e8b434576dacdb8a95f30946b28dc9f825ee2a6 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 10 Jan 2022 18:29:13 +0800 Subject: [PATCH 19/73] Merge remote-tracking branch 'upstream/master' # Conflicts: # .github/workflows/centos7-ci.yml # ci/install-ext-services-via-docker.sh --- .github/workflows/centos7-ci.yml | 98 ++++++++++++++++---------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 70150a925c70..3d0234ec7089 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -23,60 +23,60 @@ jobs: timeout-minutes: 90 steps: - - name: Check out code - uses: actions/checkout@v2.4.0 - with: - submodules: recursive + - name: Check out code + uses: actions/checkout@v2.4.0 + with: + submodules: recursive - - name: Extract branch name - if: ${{ startsWith(github.ref, 'refs/heads/release/') }} - id: branch_env - shell: bash - run: | - echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})" + - name: Extract branch name + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + id: branch_env + shell: bash + run: | + echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})" - - name: Linux launch common services - run: | - make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml + - name: Linux launch common services + run: | + make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml - - name: Build rpm package - if: ${{ startsWith(github.ref, 'refs/heads/release/') }} - run: | - export VERSION=${{ steps.branch_env.outputs.version }} - sudo gem install --no-document fpm - git clone -b v2.6.0 https://github.com/api7/apisix-build-tools.git - - # move codes under build tool - mkdir ./apisix-build-tools/apisix - for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done - - cd apisix-build-tools - make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix - cd .. - rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) + - name: Build rpm package + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + run: | + export VERSION=${{ steps.branch_env.outputs.version }} + sudo gem install --no-document fpm + git clone -b v2.6.0 https://github.com/api7/apisix-build-tools.git - - name: Run centos7 docker and mapping apisix into container - run: | - docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 --dns-search apache.org -v /tmp:/tmp docker.io/centos:7 /bin/bash - # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" + # move codes under build tool + mkdir ./apisix-build-tools/apisix + for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done - - name: Run other docker containers for test - run: | - make ci-env-up - ./ci/linux-ci-init-service.sh + cd apisix-build-tools + make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix + cd .. + rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) - - name: Install dependencies - run: | - docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh install_dependencies" + - name: Run centos7 docker and mapping apisix into container + run: | + docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 -v /tmp:/tmp --dns-search apache.org docker.io/centos:7 /bin/bash + # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" - - name: Install rpm package - if: ${{ startsWith(github.ref, 'refs/heads/release/') }} - run: | - docker exec centos7Instance bash -c "cd apisix && rpm -iv --prefix=/apisix ./apisix-build-tools/output/apisix-${{ steps.branch_env.outputs.version }}-0.el7.x86_64.rpm" - # Dependencies are attached with rpm, so revert `make deps` - docker exec centos7Instance bash -c "cd apisix && rm -rf deps" - docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." + - name: Run other docker containers for test + run: | + make ci-env-up + ./ci/linux-ci-init-service.sh - - name: Run test cases - run: | - docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh run_case" + - name: Install dependencies + run: | + docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh install_dependencies" + + - name: Install rpm package + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + run: | + docker exec centos7Instance bash -c "cd apisix && rpm -iv --prefix=/apisix ./apisix-build-tools/output/apisix-${{ steps.branch_env.outputs.version }}-0.el7.x86_64.rpm" + # Dependencies are attached with rpm, so revert `make deps` + docker exec centos7Instance bash -c "cd apisix && rm -rf deps" + docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." + + - name: Run test cases + run: | + docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh run_case" From 786d53ba771d212ea50ea93e2f359d60a621b6cc Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 11 Jan 2022 00:56:47 +0800 Subject: [PATCH 20/73] Merge remote-tracking branch 'upstream/master' # Conflicts: # .github/workflows/centos7-ci.yml # ci/install-ext-services-via-docker.sh --- apisix/discovery/k8s/init.lua | 246 +++++++++++++++++----------------- ci/common.sh | 68 +++++----- ci/linux-ci-init-service.sh | 0 3 files changed, 157 insertions(+), 157 deletions(-) mode change 100644 => 100755 ci/linux-ci-init-service.sh diff --git a/apisix/discovery/k8s/init.lua b/apisix/discovery/k8s/init.lua index 4366a5f6a029..511e9fc66d9a 100644 --- a/apisix/discovery/k8s/init.lua +++ b/apisix/discovery/k8s/init.lua @@ -54,8 +54,8 @@ local endpoint_buffer = {} local empty_table = {} local function sort_cmp(left, right) - if left.ip ~= right.ip then - return left.ip < right.ip + if left.host ~= right.host then + return left.host < right.host end return left.port < right.port end @@ -108,12 +108,12 @@ local function on_endpoint_modified(endpoint) local _, err _, err = endpoints_shared:safe_set(endpoint_key .. "#version", endpoint_version) if err then - core.log.emerg("set endpoint version into discovery DICT failed, ", err) + core.log.error("set endpoint version into discovery DICT failed, ", err) return end endpoints_shared:safe_set(endpoint_key, endpoint_content) if err then - core.log.emerg("set endpoint into discovery DICT failed, ", err) + core.log.error("set endpoint into discovery DICT failed, ", err) endpoints_shared:delete(endpoint_key .. "#version") end end @@ -162,7 +162,7 @@ local function create_resource(group, version, kind, plural, namespace) end if self.field_selector and self.field_selector ~= "" then - uri = uri .. "&filedSelector=" .. self.field_selector + uri = uri .. "&fieldSelector=" .. self.field_selector end return uri @@ -180,7 +180,7 @@ local function create_resource(group, version, kind, plural, namespace) end if self.field_selector and self.field_selector ~= "" then - uri = uri .. "&filedSelector=" .. self.field_selector + uri = uri .. "&fieldSelector=" .. self.field_selector end return uri @@ -237,65 +237,71 @@ local function list_resource(httpc, resource) end local function watch_resource(httpc, resource) - local watch_seconds = 1800 + math.random(9, 999) - resource.overtime = watch_seconds - local http_seconds = watch_seconds + 120 - httpc:set_timeouts(2000, 3000, http_seconds * 1000) - local res, err = httpc:request({ - path = resource.path, - query = resource:watch_query(), - headers = { - ["Host"] = apiserver_host .. ":" .. apiserver_port, - ["Authorization"] = "Bearer " .. apiserver_token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } - }) - - core.log.info("--raw=" .. resource.path .. "?" .. resource:watch_query()) - - if err then - return false, "RequestError", err - end - - if res.status ~= 200 then - return false, res.reason, res:read_body() or "" - end + local max_watch_times = 3 + + for _ = 0, max_watch_times do + local watch_seconds = 1800 + math.random(9, 999) + resource.overtime = watch_seconds + local http_seconds = watch_seconds + 120 + httpc:set_timeouts(2000, 3000, http_seconds * 1000) + + local res, err = httpc:request({ + path = resource.path, + query = resource:watch_query(), + headers = { + ["Host"] = apiserver_host .. ":" .. apiserver_port, + ["Authorization"] = "Bearer " .. apiserver_token, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + }) + + core.log.info("--raw=" .. resource.path .. "?" .. resource:watch_query()) - local remainder_body = "" - local body - local reader = res.body_reader - local gmatch_iterator - local captures - local captured_size = 0 - while true do - - body, err = reader() if err then - return false, "ReadBodyError", err - end - - if not body then - break + return false, "RequestError", err end - if #remainder_body ~= 0 then - body = remainder_body .. body + if res.status ~= 200 then + return false, res.reason, res:read_body() or "" end - gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") - if not gmatch_iterator then - return false, "GmatchError", err - end + local remainder_body = "" + local body + local reader = res.body_reader + local gmatch_iterator + local captures + local captured_size = 0 while true do + body, err = reader() + if err then + return false, "ReadBodyError", err + end + + if not body then + break + end + + if #remainder_body ~= 0 then + body = remainder_body .. body + end + + gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") + if not gmatch_iterator then + return false, "GmatchError", err + end + captures, err = gmatch_iterator() + if err then return false, "GmatchError", err end + if not captures then break end + captured_size = captured_size + #captures[0] local v, _ = core.json.decode(captures[0]) if not v or not v.object or v.object.kind ~= resource.kind then @@ -319,94 +325,85 @@ local function watch_resource(httpc, resource) elseif type == BookmarkEvent then -- do nothing end - end - if captured_size == #body then - remainder_body = "" - elseif captured_size == 0 then - remainder_body = body - else - remainder_body = string.sub(body, captured_size + 1) + if captured_size == #body then + remainder_body = "" + elseif captured_size == 0 then + remainder_body = body + else + remainder_body = string.sub(body, captured_size + 1) + end end end - watch_resource(httpc, resource) end local function fetch_resource(resource) - while true do - local ok - local reason, message - local retry_interval - repeat - local httpc = http.new() - resource.fetch_state = "connecting" - core.log.info("begin to connect ", apiserver_host, ":", apiserver_port) - ok, message = httpc:connect({ - scheme = apiserver_schema, - host = apiserver_host, - port = apiserver_port, - ssl_verify = false - }) - if not ok then - resource.fetch_state = "connecting" - core.log.error("connect apiserver failed , apiserver_host: ", apiserver_host, - ", apiserver_port", apiserver_port, ", message : ", message) - retry_interval = 100 - break - end + local ok + local reason, message + local httpc = http.new() + + resource.fetch_state = "connecting" + core.log.info("begin to connect ", apiserver_host, ":", apiserver_port) + + ok, message = httpc:connect({ + scheme = apiserver_schema, + host = apiserver_host, + port = apiserver_port, + ssl_verify = false + }) - core.log.info("begin to list ", resource.plural) - resource.fetch_state = "listing" - if resource.pre_List ~= nil then - resource:pre_list() - end - ok, reason, message = list_resource(httpc, resource) - if not ok then - resource.fetch_state = "list failed" - core.log.error("list failed, resource: ", resource.plural, - ", reason: ", reason, ", message : ", message) - retry_interval = 100 - break - end - resource.fetch_state = "list finished" - if resource.post_List ~= nil then - resource:post_list() - end + if not ok then + resource.fetch_state = "connecting" + core.log.error("connect apiserver failed , apiserver_host: ", apiserver_host, + ", apiserver_port", apiserver_port, ", message : ", message) + return false + end - core.log.info("begin to watch ", resource.plural) - resource.fetch_state = "watching" - ok, reason, message = watch_resource(httpc, resource) - if not ok then - resource.fetch_state = "watch failed" - core.log.error("watch failed, resource: ", resource.plural, - ", reason: ", reason, ", message : ", message) - retry_interval = 0 - break - end - resource.fetch_state = "watch finished" - retry_interval = 0 - until true + core.log.info("begin to list ", resource.plural) + resource.fetch_state = "listing" + if resource.pre_List ~= nil then + resource:pre_list() end + + ok, reason, message = list_resource(httpc, resource) + if not ok then + resource.fetch_state = "list failed" + core.log.error("list failed, resource: ", resource.plural, + ", reason: ", reason, ", message : ", message) + return false + end + + resource.fetch_state = "list finished" + if resource.post_List ~= nil then + resource:post_list() + end + + core.log.info("begin to watch ", resource.plural) + resource.fetch_state = "watching" + ok, reason, message = watch_resource(httpc, resource) + if not ok then + resource.fetch_state = "watch failed" + core.log.error("watch failed, resource: ", resource.plural, + ", reason: ", reason, ", message : ", message) + return false + end + + resource.fetch_state = "watch finished" + return true end local function set_namespace_selector(conf, resource) local ns = conf.namespace_selector if ns == nil then - resource.namespace_filter = function(self, namespace) - return true - end + resource.namespace_selector = nil elseif ns.equal then resource.field_selector = "metadata.namespace%3D" .. ns.equal - resource.namespace_filter = function(self, namespace) - return true - end + resource.namespace_selector = nil elseif ns.not_equal then resource.field_selector = "metadata.namespace%21%3D" .. ns.not_equal - resource.namespace_filter = function(self, namespace) - return true - end + resource.namespace_selector = nil elseif ns.match then - resource.namespace_filter = function(self, namespace) + resource.namespace_selector = function(self, namespace) local match = conf.namespace_selector.match local m, err for _, v in ipairs(match) do @@ -421,7 +418,7 @@ local function set_namespace_selector(conf, resource) return false end elseif ns.not_match then - resource.namespace_filter = function(self, namespace) + resource.namespace_selector = function(self, namespace) local not_match = conf.namespace_selector.not_match local m, err for _, v in ipairs(not_match) do @@ -508,14 +505,14 @@ end local function create_endpoint_lrucache(endpoint_key, endpoint_port) local endpoint_content, _, _ = endpoints_shared:get_stale(endpoint_key) if not endpoint_content then - core.log.emerg("get empty endpoint content from discovery DIC, this should not happen ", + core.log.error("get empty endpoint content from discovery DIC, this should not happen ", endpoint_key) return nil end local endpoint, _ = core.json.decode(endpoint_content) if not endpoint then - core.log.emerg("decode endpoint content failed, this should not happen, content : ", + core.log.error("decode endpoint content failed, this should not happen, content: ", endpoint_content) end @@ -584,7 +581,10 @@ function _M.init_worker() local timer_runner timer_runner = function() - fetch_resource(resource) + if not fetch_resource(resource) then + local retry_interval = 40 + ngx.sleep(retry_interval) + end ngx.timer.at(0, timer_runner) end ngx.timer.at(0, timer_runner) diff --git a/ci/common.sh b/ci/common.sh index 0037257d8897..1b8c6ab497cb 100644 --- a/ci/common.sh +++ b/ci/common.sh @@ -84,12 +84,12 @@ install_k8s () { chmod +x ./kubectl echo -e " - kind: Cluster - apiVersion: kind.x-k8s.io/v1alpha4 - networking: - apiServerAddress: 127.0.0.1 - apiServerPort: 6443 - " >kind.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerAddress: 127.0.0.1 + apiServerPort: 6443 +" >kind.yaml ./kind delete cluster --name apisix-test ./kind create cluster --name apisix-test --config ./kind.yaml @@ -102,34 +102,34 @@ install_k8s () { done echo -e " - kind: ServiceAccount - apiVersion: v1 - metadata: - name: apisix-test - namespace: default - --- - kind: ClusterRole - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: apisix-test - rules: - - apiGroups: [ \"\" ] - resources: [ endpoints ] - verbs: [ get,list,watch ] - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: apisix-test - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: apisix-test - subjects: - - kind: ServiceAccount - name: apisix-test - namespace: default - " >apisix-test-rbac.yaml +kind: ServiceAccount +apiVersion: v1 +metadata: + name: apisix-test + namespace: default +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: apisix-test +rules: + - apiGroups: [ \"\" ] + resources: [ endpoints ] + verbs: [ get,list,watch ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: apisix-test +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: apisix-test +subjects: + - kind: ServiceAccount + name: apisix-test + namespace: default +" >apisix-test-rbac.yaml ./kubectl apply -f ./apisix-test-rbac.yaml ./kubectl proxy -p 6445 & diff --git a/ci/linux-ci-init-service.sh b/ci/linux-ci-init-service.sh old mode 100644 new mode 100755 From 769ad8f38023e451a926fcdf48154e09839b3a96 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 11 Jan 2022 09:31:08 +0800 Subject: [PATCH 21/73] Merge remote-tracking branch 'upstream/master' # Conflicts: # .github/workflows/centos7-ci.yml # ci/install-ext-services-via-docker.sh --- ci/common.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/common.sh b/ci/common.sh index 1b8c6ab497cb..d67e11bff3bf 100644 --- a/ci/common.sh +++ b/ci/common.sh @@ -78,8 +78,10 @@ GRPC_SERVER_EXAMPLE_VER=20210819 install_k8s () { # create kubernetes cluster using kind - curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.11.1/kind-$(uname)-amd64" - curl -Lo ./kubectl "https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl" + KIND_VERSION="v0.11.1" + KUBECTL_VERSION="v1.22.0" + curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" + curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" chmod +x ./kind chmod +x ./kubectl From fdee535f0458ab4ddd4b7f6f68b3da4969d0b16a Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 11:39:04 +0800 Subject: [PATCH 22/73] feat: add kubernetes discovery module --- apisix/discovery/k8s/init.lua | 593 ------------------ apisix/discovery/kubernetes/init.lua | 318 ++++++++++ .../discovery/{k8s => kubernetes}/schema.lua | 0 apisix/kubernetes.lua | 318 ++++++++++ ci/common.sh | 1 - t/discovery/{k8s.t => kubernetes.t} | 48 +- 6 files changed, 660 insertions(+), 618 deletions(-) delete mode 100644 apisix/discovery/k8s/init.lua create mode 100644 apisix/discovery/kubernetes/init.lua rename apisix/discovery/{k8s => kubernetes}/schema.lua (100%) create mode 100644 apisix/kubernetes.lua rename t/discovery/{k8s.t => kubernetes.t} (95%) diff --git a/apisix/discovery/k8s/init.lua b/apisix/discovery/k8s/init.lua deleted file mode 100644 index 511e9fc66d9a..000000000000 --- a/apisix/discovery/k8s/init.lua +++ /dev/null @@ -1,593 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -local ngx = ngx -local ipairs = ipairs -local pairs = pairs -local string = string -local tonumber = tonumber -local tostring = tostring -local math = math -local os = os -local error = error -local process = require("ngx.process") -local core = require("apisix.core") -local util = require("apisix.cli.util") -local local_conf = require("apisix.core.config_local").local_conf() -local http = require("resty.http") -local endpoints_shared = ngx.shared.discovery - -local AddedEven = "ADDED" -local ModifiedEvent = "MODIFIED" -local DeletedEvent = "DELETED" -local BookmarkEvent = "BOOKMARK" - -local ListDrive = "list" -local WatchDrive = "watch" - -local apiserver_schema = "" -local apiserver_host = "" -local apiserver_port = 0 -local apiserver_token = "" -local default_weight = 0 - -local endpoint_lrucache = core.lrucache.new({ - ttl = 300, - count = 1024 -}) - -local endpoint_buffer = {} -local empty_table = {} - -local function sort_cmp(left, right) - if left.host ~= right.host then - return left.host < right.host - end - return left.port < right.port -end - -local function on_endpoint_modified(endpoint) - core.log.debug(core.json.encode(endpoint, true)) - core.table.clear(endpoint_buffer) - - local subsets = endpoint.subsets - for _, subset in ipairs(subsets or empty_table) do - if subset.addresses ~= nil then - local addresses = subset.addresses - for _, port in ipairs(subset.ports or empty_table) do - local port_name - if port.name then - port_name = port.name - elseif port.targetPort then - port_name = tostring(port.targetPort) - else - port_name = tostring(port.port) - end - - local nodes = endpoint_buffer[port_name] - if nodes == nil then - nodes = core.table.new(0, #subsets * #addresses) - endpoint_buffer[port_name] = nodes - end - - for _, address in ipairs(subset.addresses) do - core.table.insert(nodes, { - host = address.ip, - port = port.port, - weight = default_weight - }) - end - end - end - end - - for _, ports in pairs(endpoint_buffer) do - for _, nodes in pairs(ports) do - core.table.sort(nodes, sort_cmp) - end - end - - local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name - local endpoint_content = core.json.encode(endpoint_buffer, true) - local endpoint_version = ngx.crc32_long(endpoint_content) - - local _, err - _, err = endpoints_shared:safe_set(endpoint_key .. "#version", endpoint_version) - if err then - core.log.error("set endpoint version into discovery DICT failed, ", err) - return - end - endpoints_shared:safe_set(endpoint_key, endpoint_content) - if err then - core.log.error("set endpoint into discovery DICT failed, ", err) - endpoints_shared:delete(endpoint_key .. "#version") - end -end - -local function on_endpoint_deleted(endpoint) - core.log.debug(core.json.encode(endpoint, true)) - local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name - endpoints_shared:delete(endpoint_key .. "#version") - endpoints_shared:delete(endpoint_key) -end - -local function create_resource(group, version, kind, plural, namespace) - local _t = { - kind = kind, - list_kind = kind .. "List", - plural = plural, - path = "", - version = "", - continue = "", - overtime = "1800", - limit = 30, - label_selector = "", - field_selector = "", - } - - if group == "" then - _t.path = _t.path .. "/api/" .. version - else - _t.path = _t.path .. "/apis/" .. group .. "/" .. version - end - - if namespace ~= "" then - _t.path = _t.path .. "/namespace/" .. namespace - end - _t.path = _t.path .. "/" .. plural - - function _t.list_query(self) - local uri = "limit=" .. self.limit - - if self.continue ~= nil and self.continue ~= "" then - uri = uri .. "&continue=" .. self.continue - end - - if self.label_selector and self.label_selector ~= "" then - uri = uri .. "&labelSelector=" .. self.label_selector - end - - if self.field_selector and self.field_selector ~= "" then - uri = uri .. "&fieldSelector=" .. self.field_selector - end - - return uri - end - - function _t.watch_query(self) - local uri = "watch=true&allowWatchBookmarks=true&timeoutSeconds=" .. self.overtime - - if self.version ~= nil and self.version ~= "" then - uri = uri .. "&resourceVersion=" .. self.version - end - - if self.label_selector and self.label_selector ~= "" then - uri = uri .. "&labelSelector=" .. self.label_selector - end - - if self.field_selector and self.field_selector ~= "" then - uri = uri .. "&fieldSelector=" .. self.field_selector - end - - return uri - end - - return _t -end - -local function list_resource(httpc, resource) - local res, err = httpc:request({ - path = resource.path, - query = resource:list_query(), - headers = { - ["Host"] = apiserver_host .. ":" .. apiserver_port, - ["Authorization"] = "Bearer " .. apiserver_token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } - }) - - core.log.info("--raw=" .. resource.path .. "?" .. resource:list_query()) - - if not res then - return false, "RequestError", err or "" - end - - if res.status ~= 200 then - return false, res.reason, res:read_body() or "" - end - local body, err = res:read_body() - if err then - return false, "ReadBodyError", err - end - - local data, _ = core.json.decode(body) - if not data or data.kind ~= resource.list_kind then - return false, "UnexpectedBody", body - end - - resource.version = data.metadata.resourceVersion - - if resource.on_added ~= nil then - for _, item in ipairs(data.items or empty_table) do - resource:on_added(item, ListDrive) - end - end - - resource.continue = data.metadata.continue - if data.metadata.continue ~= nil and data.metadata.continue ~= "" then - list_resource(httpc, resource) - end - - return true, "Success", "" -end - -local function watch_resource(httpc, resource) - local max_watch_times = 3 - - for _ = 0, max_watch_times do - local watch_seconds = 1800 + math.random(9, 999) - resource.overtime = watch_seconds - local http_seconds = watch_seconds + 120 - httpc:set_timeouts(2000, 3000, http_seconds * 1000) - - local res, err = httpc:request({ - path = resource.path, - query = resource:watch_query(), - headers = { - ["Host"] = apiserver_host .. ":" .. apiserver_port, - ["Authorization"] = "Bearer " .. apiserver_token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } - }) - - core.log.info("--raw=" .. resource.path .. "?" .. resource:watch_query()) - - if err then - return false, "RequestError", err - end - - if res.status ~= 200 then - return false, res.reason, res:read_body() or "" - end - - local remainder_body = "" - local body - local reader = res.body_reader - local gmatch_iterator - local captures - local captured_size = 0 - - while true do - body, err = reader() - if err then - return false, "ReadBodyError", err - end - - if not body then - break - end - - if #remainder_body ~= 0 then - body = remainder_body .. body - end - - gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") - if not gmatch_iterator then - return false, "GmatchError", err - end - - captures, err = gmatch_iterator() - - if err then - return false, "GmatchError", err - end - - if not captures then - break - end - - captured_size = captured_size + #captures[0] - local v, _ = core.json.decode(captures[0]) - if not v or not v.object or v.object.kind ~= resource.kind then - return false, "UnexpectedBody", captures[0] - end - - resource.version = v.object.metadata.resourceVersion - local type = v.type - if type == AddedEven then - if resource.on_added ~= nil then - resource:on_added(v.object, WatchDrive) - end - elseif type == DeletedEvent then - if resource.on_deleted ~= nil then - resource:on_deleted(v.object) - end - elseif type == ModifiedEvent then - if resource.on_modified ~= nil then - resource:on_modified(v.object) - end - elseif type == BookmarkEvent then - -- do nothing - end - - if captured_size == #body then - remainder_body = "" - elseif captured_size == 0 then - remainder_body = body - else - remainder_body = string.sub(body, captured_size + 1) - end - end - end -end - -local function fetch_resource(resource) - local ok - local reason, message - local httpc = http.new() - - resource.fetch_state = "connecting" - core.log.info("begin to connect ", apiserver_host, ":", apiserver_port) - - ok, message = httpc:connect({ - scheme = apiserver_schema, - host = apiserver_host, - port = apiserver_port, - ssl_verify = false - }) - - if not ok then - resource.fetch_state = "connecting" - core.log.error("connect apiserver failed , apiserver_host: ", apiserver_host, - ", apiserver_port", apiserver_port, ", message : ", message) - return false - end - - core.log.info("begin to list ", resource.plural) - resource.fetch_state = "listing" - if resource.pre_List ~= nil then - resource:pre_list() - end - - ok, reason, message = list_resource(httpc, resource) - if not ok then - resource.fetch_state = "list failed" - core.log.error("list failed, resource: ", resource.plural, - ", reason: ", reason, ", message : ", message) - return false - end - - resource.fetch_state = "list finished" - if resource.post_List ~= nil then - resource:post_list() - end - - core.log.info("begin to watch ", resource.plural) - resource.fetch_state = "watching" - ok, reason, message = watch_resource(httpc, resource) - if not ok then - resource.fetch_state = "watch failed" - core.log.error("watch failed, resource: ", resource.plural, - ", reason: ", reason, ", message : ", message) - return false - end - - resource.fetch_state = "watch finished" - return true -end - -local function set_namespace_selector(conf, resource) - local ns = conf.namespace_selector - if ns == nil then - resource.namespace_selector = nil - elseif ns.equal then - resource.field_selector = "metadata.namespace%3D" .. ns.equal - resource.namespace_selector = nil - elseif ns.not_equal then - resource.field_selector = "metadata.namespace%21%3D" .. ns.not_equal - resource.namespace_selector = nil - elseif ns.match then - resource.namespace_selector = function(self, namespace) - local match = conf.namespace_selector.match - local m, err - for _, v in ipairs(match) do - m, err = ngx.re.match(namespace, v, "j") - if m and m[0] == namespace then - return true - end - if err then - core.log.error("ngx.re.match failed: ", err) - end - end - return false - end - elseif ns.not_match then - resource.namespace_selector = function(self, namespace) - local not_match = conf.namespace_selector.not_match - local m, err - for _, v in ipairs(not_match) do - m, err = ngx.re.match(namespace, v, "j") - if m and m[0] == namespace then - return false - end - if err then - return false - end - end - return true - end - end -end - -local function read_env(key) - if #key > 3 then - local a, b = string.byte(key, 1, 2) - local c = string.byte(key, #key, #key) - -- '$', '{', '}' == 36,123,125 - if a == 36 and b == 123 and c == 125 then - local env = string.sub(key, 3, #key - 1) - local val = os.getenv(env) - if not val then - return false, nil, "not found environment variable " .. env - end - return true, val, nil - end - end - return true, key, nil -end - -local function read_conf(conf) - apiserver_schema = conf.service.schema - - local ok, value, message - ok, value, message = read_env(conf.service.host) - if not ok then - return false, message - end - apiserver_host = value - if apiserver_host == "" then - return false, "get empty host value" - end - - ok, value, message = read_env(conf.service.port) - if not ok then - return false, message - end - apiserver_port = tonumber(value) - if not apiserver_port or apiserver_port <= 0 or apiserver_port > 65535 then - return false, "get invalid port value: " .. apiserver_port - end - - -- we should not check if the apiserver_token is empty here - if conf.client.token then - ok, value, message = read_env(conf.client.token) - if not ok then - return false, message - end - apiserver_token = value - elseif conf.client.token_file and conf.client.token_file ~= "" then - ok, value, message = read_env(conf.client.token_file) - if not ok then - return false, message - end - local apiserver_token_file = value - - apiserver_token, message = util.read_file(apiserver_token_file) - if not apiserver_token then - return false, message - end - else - return false, "invalid k8s discovery configuration:" .. - "should set one of [client.token,client.token_file] but none" - end - - default_weight = conf.default_weight or 50 - - return true, nil -end - -local function create_endpoint_lrucache(endpoint_key, endpoint_port) - local endpoint_content, _, _ = endpoints_shared:get_stale(endpoint_key) - if not endpoint_content then - core.log.error("get empty endpoint content from discovery DIC, this should not happen ", - endpoint_key) - return nil - end - - local endpoint, _ = core.json.decode(endpoint_content) - if not endpoint then - core.log.error("decode endpoint content failed, this should not happen, content: ", - endpoint_content) - end - - return endpoint[endpoint_port] -end - -local _M = { - version = "0.0.1" -} - -function _M.nodes(service_name) - local pattern = "^(.*):(.*)$" -- namespace/name:port_name - local match, _ = ngx.re.match(service_name, pattern, "jo") - if not match then - core.log.info("get unexpected upstream service_name:　", service_name) - return nil - end - - local endpoint_key = match[1] - local endpoint_port = match[2] - local endpoint_version, _, _ = endpoints_shared:get_stale(endpoint_key .. "#version") - if not endpoint_version then - core.log.info("get empty endpoint version from discovery DICT ", endpoint_key) - return nil - end - return endpoint_lrucache(service_name, endpoint_version, - create_endpoint_lrucache, endpoint_key, endpoint_port) -end - -function _M.init_worker() - if process.type() ~= "privileged agent" then - return - end - - local ok, err = read_conf(local_conf.discovery.k8s) - if not ok then - error(err) - return - end - - local resource = create_resource("", "v1", "Endpoints", "endpoints", "") - - set_namespace_selector(local_conf.discovery.k8s, resource) - - resource.on_added = function(self, object, drive) - if self.namespace_selector ~= nil then - if self:namespace_selector(object.metadata.namespace) then - on_endpoint_modified(object) - end - else - on_endpoint_modified(object) - end - end - - resource.on_modified = resource.on_added - - resource.on_deleted = function(self, object) - if self.namespace_selector ~= nil then - if self:namespace_selector(object.metadata.namespace) then - on_endpoint_deleted(object) - end - else - on_endpoint_deleted(object) - end - end - - local timer_runner - timer_runner = function() - if not fetch_resource(resource) then - local retry_interval = 40 - ngx.sleep(retry_interval) - end - ngx.timer.at(0, timer_runner) - end - ngx.timer.at(0, timer_runner) -end - -return _M diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua new file mode 100644 index 000000000000..af26ef0a0d9d --- /dev/null +++ b/apisix/discovery/kubernetes/init.lua @@ -0,0 +1,318 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local ngx = ngx +local ipairs = ipairs +local pairs = pairs +local string = string +local tonumber = tonumber +local tostring = tostring +local os = os +local error = error +local process = require("ngx.process") +local core = require("apisix.core") +local util = require("apisix.cli.util") +local local_conf = require("apisix.core.config_local").local_conf() +local kubernetes = require("apisix.kubernetes") +local endpoint_dict = ngx.shared.discovery + +local default_weight = 0 + +local endpoint_lrucache = core.lrucache.new({ + ttl = 300, + count = 1024 +}) + +local endpoint_buffer = {} +local empty_table = {} + +local function sort_cmp(left, right) + if left.host ~= right.host then + return left.host < right.host + end + return left.port < right.port +end + +local function on_endpoint_modified(endpoint) + core.log.debug(core.json.encode(endpoint, true)) + core.table.clear(endpoint_buffer) + + local subsets = endpoint.subsets + for _, subset in ipairs(subsets or empty_table) do + if subset.addresses ~= nil then + local addresses = subset.addresses + for _, port in ipairs(subset.ports or empty_table) do + local port_name + if port.name then + port_name = port.name + elseif port.targetPort then + port_name = tostring(port.targetPort) + else + port_name = tostring(port.port) + end + + local nodes = endpoint_buffer[port_name] + if nodes == nil then + nodes = core.table.new(0, #subsets * #addresses) + endpoint_buffer[port_name] = nodes + end + + for _, address in ipairs(subset.addresses) do + core.table.insert(nodes, { + host = address.ip, + port = port.port, + weight = default_weight + }) + end + end + end + end + + for _, ports in pairs(endpoint_buffer) do + for _, nodes in pairs(ports) do + core.table.sort(nodes, sort_cmp) + end + end + + local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name + local endpoint_content = core.json.encode(endpoint_buffer, true) + local endpoint_version = ngx.crc32_long(endpoint_content) + + local _, err + _, err = endpoint_dict:safe_set(endpoint_key .. "#version", endpoint_version) + if err then + core.log.error("set endpoint version into discovery DICT failed, ", err) + return + end + endpoint_dict:safe_set(endpoint_key, endpoint_content) + if err then + core.log.error("set endpoint into discovery DICT failed, ", err) + endpoint_dict:delete(endpoint_key .. "#version") + end +end + +local function on_endpoint_deleted(endpoint) + core.log.debug(core.json.encode(endpoint, true)) + local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name + endpoint_dict:delete(endpoint_key .. "#version") + endpoint_dict:delete(endpoint_key) +end + +local function set_namespace_selector(conf, informer) + local ns = conf.namespace_selector + if ns == nil then + informer.namespace_selector = nil + elseif ns.equal then + informer.field_selector = "metadata.namespace%3D" .. ns.equal + informer.namespace_selector = nil + elseif ns.not_equal then + informer.field_selector = "metadata.namespace%21%3D" .. ns.not_equal + informer.namespace_selector = nil + elseif ns.match then + informer.namespace_selector = function(self, namespace) + local match = conf.namespace_selector.match + local m, err + for _, v in ipairs(match) do + m, err = ngx.re.match(namespace, v, "j") + if m and m[0] == namespace then + return true + end + if err then + core.log.error("ngx.re.match failed: ", err) + end + end + return false + end + elseif ns.not_match then + informer.namespace_selector = function(self, namespace) + local not_match = conf.namespace_selector.not_match + local m, err + for _, v in ipairs(not_match) do + m, err = ngx.re.match(namespace, v, "j") + if m and m[0] == namespace then + return false + end + if err then + return false + end + end + return true + end + end +end + +local function read_env(key) + if #key > 3 then + local a, b = string.byte(key, 1, 2) + local c = string.byte(key, #key, #key) + -- '$', '{', '}' == 36,123,125 + if a == 36 and b == 123 and c == 125 then + local env = string.sub(key, 3, #key - 1) + local val = os.getenv(env) + if not val then + return false, nil, "not found environment variable " .. env + end + return true, val, nil + end + end + return true, key, nil +end + +local function read_conf(conf, apiserver) + + apiserver.schema = conf.service.schema + + local ok, value, message + ok, value, message = read_env(conf.service.host) + if not ok then + return false, message + end + + apiserver.host = value + if apiserver.host == "" then + return false, "get empty host value" + end + + ok, value, message = read_env(conf.service.port) + if not ok then + return false, message + end + + apiserver.port = tonumber(value) + if not apiserver.port or apiserver.port <= 0 or apiserver.port > 65535 then + return false, "get invalid port value: " .. apiserver.port + end + + -- we should not check if the apiserver.token is empty here + if conf.client.token then + ok, value, message = read_env(conf.client.token) + if not ok then + return false, message + end + apiserver.token = value + elseif conf.client.token_file and conf.client.token_file ~= "" then + ok, value, message = read_env(conf.client.token_file) + if not ok then + return false, message + end + local apiserver_token_file = value + + apiserver.token, message = util.read_file(apiserver_token_file) + if not apiserver.token then + return false, message + end + else + return false, "invalid kubernetes discovery configuration:" .. + "should set one of [client.token,client.token_file] but none" + end + + default_weight = conf.default_weight or 50 + + return true, nil +end + +local function create_endpoint_lrucache(endpoint_key, endpoint_port) + local endpoint_content, _, _ = endpoint_dict:get_stale(endpoint_key) + if not endpoint_content then + core.log.error("get empty endpoint content from discovery DIC, this should not happen ", + endpoint_key) + return nil + end + + local endpoint, _ = core.json.decode(endpoint_content) + if not endpoint then + core.log.error("decode endpoint content failed, this should not happen, content: ", + endpoint_content) + end + + return endpoint[endpoint_port] +end + +local _M = { + version = "0.0.1" +} + +function _M.nodes(service_name) + local pattern = "^(.*):(.*)$" -- namespace/name:port_name + local match, _ = ngx.re.match(service_name, pattern, "jo") + if not match then + core.log.info("get unexpected upstream service_name:　", service_name) + return nil + end + + local endpoint_key = match[1] + local endpoint_port = match[2] + local endpoint_version, _, _ = endpoint_dict:get_stale(endpoint_key .. "#version") + if not endpoint_version then + core.log.info("get empty endpoint version from discovery DICT ", endpoint_key) + return nil + end + return endpoint_lrucache(service_name, endpoint_version, + create_endpoint_lrucache, endpoint_key, endpoint_port) +end + +function _M.init_worker() + if process.type() ~= "privileged agent" then + return + end + + local ok, err = read_conf(local_conf.discovery.kubernetes, kubernetes.apiserver) + if not ok then + error(err) + return + end + + local endpoint_informer = kubernetes.informer_factory.new("", "v1", + "Endpoints", "endpoints", "") + + set_namespace_selector(local_conf.discovery.kubernetes, endpoint_informer) + + endpoint_informer.on_added = function(self, object, drive) + if self.namespace_selector ~= nil then + if self:namespace_selector(object.metadata.namespace) then + on_endpoint_modified(object) + end + else + on_endpoint_modified(object) + end + end + + endpoint_informer.on_modified = endpoint_informer.on_added + + endpoint_informer.on_deleted = function(self, object) + if self.namespace_selector ~= nil then + if self:namespace_selector(object.metadata.namespace) then + on_endpoint_deleted(object) + end + else + on_endpoint_deleted(object) + end + end + + local timer_runner + timer_runner = function(premature, informer) + if informer:list_watch() then + local retry_interval = 40 + ngx.sleep(retry_interval) + end + ngx.timer.at(0, timer_runner, informer) + end + + ngx.timer.at(0, timer_runner, endpoint_informer) +end + +return _M diff --git a/apisix/discovery/k8s/schema.lua b/apisix/discovery/kubernetes/schema.lua similarity index 100% rename from apisix/discovery/k8s/schema.lua rename to apisix/discovery/kubernetes/schema.lua diff --git a/apisix/kubernetes.lua b/apisix/kubernetes.lua new file mode 100644 index 000000000000..d75bfe9f1154 --- /dev/null +++ b/apisix/kubernetes.lua @@ -0,0 +1,318 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local ngx = ngx +local ipairs = ipairs +local string = string +local math = math +local core = require("apisix.core") +local http = require("resty.http") + +local _constants = { + AddedEven = "ADDED", + ModifiedEvent = "MODIFIED", + DeletedEvent = "DELETED", + BookmarkEvent = "BOOKMARK", + ListDrive = "list", + WatchDrive = "watch", +} + +local _apiserver = { + schema = "", + host = "", + port = "", + token = "" +} + +local empty_table = {} + +local function list(httpc, informer) + local res, err = httpc:request({ + path = informer.path, + query = informer:list_query(), + headers = { + ["Host"] = _apiserver.host .. ":" .. _apiserver.port, + ["Authorization"] = "Bearer " .. _apiserver.token, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + }) + + core.log.info("--raw=" .. informer.path .. "?" .. informer:list_query()) + + if not res then + return false, "RequestError", err or "" + end + + if res.status ~= 200 then + return false, res.reason, res:read_body() or "" + end + local body, err = res:read_body() + if err then + return false, "ReadBodyError", err + end + + local data, _ = core.json.decode(body) + if not data or data.kind ~= informer.list_kind then + return false, "UnexpectedBody", body + end + + informer.version = data.metadata.resourceVersion + + if informer.on_added ~= nil then + for _, item in ipairs(data.items or empty_table) do + informer:on_added(item, _constants.ListDrive) + end + end + + informer.continue = data.metadata.continue + if informer.continue ~= nil and informer.continue ~= "" then + list(httpc, informer) + end + + return true, "Success", "" +end + +local function watch(httpc, informer) + local max_watch_times = 3 + + for _ = 0, max_watch_times do + local watch_seconds = 1800 + math.random(9, 999) + informer.overtime = watch_seconds + local http_seconds = watch_seconds + 120 + httpc:set_timeouts(2000, 3000, http_seconds * 1000) + + local res, err = httpc:request({ + path = informer.path, + query = informer:watch_query(), + headers = { + ["Host"] = _apiserver.host .. ":" .. _apiserver.port, + ["Authorization"] = "Bearer " .. _apiserver.token, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + }) + + core.log.info("--raw=" .. informer.path .. "?" .. informer:watch_query()) + + if err then + return false, "RequestError", err + end + + if res.status ~= 200 then + return false, res.reason, res:read_body() or "" + end + + local remainder_body = "" + local body + local reader = res.body_reader + local gmatch_iterator + local captures + local captured_size = 0 + + while true do + body, err = reader() + if err then + return false, "ReadBodyError", err + end + + if not body then + break + end + + if #remainder_body ~= 0 then + body = remainder_body .. body + end + + gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") + if not gmatch_iterator then + return false, "GmatchError", err + end + + captures, err = gmatch_iterator() + + if err then + return false, "GmatchError", err + end + + if not captures then + break + end + + captured_size = captured_size + #captures[0] + local v, _ = core.json.decode(captures[0]) + if not v or not v.object or v.object.kind ~= informer.kind then + return false, "UnexpectedBody", captures[0] + end + + informer.version = v.object.metadata.resourceVersion + local type = v.type + if type == _constants.AddedEven then + if informer.on_added ~= nil then + informer:on_added(v.object, _constants.WatchDrive) + end + elseif type == _constants.DeletedEvent then + if informer.on_deleted ~= nil then + informer:on_deleted(v.object) + end + elseif type == _constants.ModifiedEvent then + if informer.on_modified ~= nil then + informer:on_modified(v.object) + end + -- elseif type == _constants.BookmarkEvent then + -- do nothing + end + + if captured_size == #body then + remainder_body = "" + elseif captured_size == 0 then + remainder_body = body + else + remainder_body = string.sub(body, captured_size + 1) + end + end + end +end + +local _informer_factory = { +} + +function _informer_factory.new(group, version, kind, plural, namespace) + local _t = { + kind = kind, + list_kind = kind .. "List", + plural = plural, + path = "", + version = "", + continue = "", + overtime = "1800", + limit = 30, + label_selector = "", + field_selector = "", + } + + if group == "" then + _t.path = _t.path .. "/api/" .. version + else + _t.path = _t.path .. "/apis/" .. group .. "/" .. version + end + + if namespace ~= "" then + _t.path = _t.path .. "/namespace/" .. namespace + end + _t.path = _t.path .. "/" .. plural + + function _t.list_query(self) + local uri = "limit=" .. self.limit + + if self.continue ~= nil and self.continue ~= "" then + uri = uri .. "&continue=" .. self.continue + end + + if self.label_selector and self.label_selector ~= "" then + uri = uri .. "&labelSelector=" .. self.label_selector + end + + if self.field_selector and self.field_selector ~= "" then + uri = uri .. "&fieldSelector=" .. self.field_selector + end + + return uri + end + + function _t.watch_query(self) + local uri = "watch=true&allowWatchBookmarks=true&timeoutSeconds=" .. self.overtime + + if self.version ~= nil and self.version ~= "" then + uri = uri .. "&resourceVersion=" .. self.version + end + + if self.label_selector and self.label_selector ~= "" then + uri = uri .. "&labelSelector=" .. self.label_selector + end + + if self.field_selector and self.field_selector ~= "" then + uri = uri .. "&fieldSelector=" .. self.field_selector + end + + return uri + end + + function _t.list_watch(self) + local ok + local reason, message + local httpc = http.new() + + self.fetch_state = "connecting" + core.log.info("begin to connect ", _apiserver.host, ":", _apiserver.port) + + ok, message = httpc:connect({ + scheme = _apiserver.schema, + host = _apiserver.host, + port = _apiserver.port, + ssl_verify = false + }) + + if not ok then + self.fetch_state = "connecting" + core.log.error("connect apiserver failed , _apiserver.host: ", _apiserver.host, + ", _apiserver.port", _apiserver.port, ", message : ", message) + return false + end + + core.log.info("begin to list ", self.kind) + self.fetch_state = "listing" + if self.pre_List ~= nil then + self:pre_list() + end + + ok, reason, message = list(httpc, self) + if not ok then + self.fetch_state = "list failed" + core.log.error("list failed, kind: ", self.kind, + ", reason: ", reason, ", message : ", message) + return false + end + + self.fetch_state = "list finished" + if self.post_List ~= nil then + self:post_list() + end + + core.log.info("begin to watch ", self.kind) + self.fetch_state = "watching" + ok, reason, message = watch(httpc, self) + if not ok then + self.fetch_state = "watch failed" + core.log.error("watch failed, kind: ", self.kind, + ", reason: ", reason, ", message : ", message) + return false + end + + self.fetch_state = "watch finished" + return true + end + + return _t +end + +return { + version = "0.0.1", + informer_factory = _informer_factory, + apiserver = _apiserver, + __index = _constants +} diff --git a/ci/common.sh b/ci/common.sh index d67e11bff3bf..21e6022a9122 100644 --- a/ci/common.sh +++ b/ci/common.sh @@ -157,5 +157,4 @@ subjects: echo 'KUBERNETES_SERVICE_PORT=6443' echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} - } diff --git a/t/discovery/k8s.t b/t/discovery/kubernetes.t similarity index 95% rename from t/discovery/k8s.t rename to t/discovery/kubernetes.t index ec1263900e99..541cd6b59727 100644 --- a/t/discovery/k8s.t +++ b/t/discovery/kubernetes.t @@ -17,38 +17,38 @@ BEGIN { my $token_var_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"; - my $token_from_var = eval { `cat $token_var_file 2>/dev/null` }; - if ($token_from_var){ + my $token_from_var = eval {`cat $token_var_file 2>/dev/null`}; + if ($token_from_var) { - our $yaml_config = <<_EOC_; + our $yaml_config = <<_EOC_; apisix: node_listen: 1984 config_center: yaml enable_admin: false discovery: - k8s: {} + kubernetes: {} _EOC_ - our $token_file = $token_var_file; - our $token_value = $token_from_var; + our $token_file = $token_var_file; + our $token_value = $token_from_var; } my $token_tmp_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; - my $token_from_tmp = eval { `cat $token_tmp_file 2>/dev/null` }; + my $token_from_tmp = eval {`cat $token_tmp_file 2>/dev/null`}; if ($token_from_tmp) { - our $yaml_config = <<_EOC_; + our $yaml_config = <<_EOC_; apisix: node_listen: 1984 config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: /tmp/var/run/secrets/kubernetes.io/serviceaccount/token _EOC_ - our $token_file = $token_tmp_file; - our $token_value = $token_from_tmp; + our $token_file = $token_tmp_file; + our $token_value = $token_from_tmp; } our $scale_ns_c = <<_EOC_; @@ -105,11 +105,11 @@ _EOC_ $block->set_value("main_config", $main_config); - my $config = $block->config // <<_EOC_; + my $config = $block->config // <<_EOC_; location /queries { content_by_lua_block { local core = require("apisix.core") - local d = require("apisix.discovery.k8s") + local d = require("apisix.discovery.kubernetes") ngx.sleep(1) @@ -378,7 +378,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: service: host: "127.0.0.1" port: "6443" @@ -403,7 +403,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: service: host: ${KUBERNETES_SERVICE_HOST} port: ${KUBERNETES_SERVICE_PORT} @@ -428,7 +428,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} --- request @@ -450,7 +450,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: service: schema: http host: "127.0.0.1" @@ -476,7 +476,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} namespace_selector: @@ -500,7 +500,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} namespace_selector: @@ -524,7 +524,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} namespace_selector: @@ -548,7 +548,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} namespace_selector: @@ -572,7 +572,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} namespace_selector: @@ -596,7 +596,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} namespace_selector: @@ -620,7 +620,7 @@ apisix: config_center: yaml enable_admin: false discovery: - k8s: + kubernetes: client: token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} namespace_selector: From 46c8180fbb5d1a76805fa7f7fd2143ae562789f6 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 14:32:17 +0800 Subject: [PATCH 23/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/init.lua | 10 +++++++++- apisix/kubernetes.lua | 25 +++++++++++++++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index af26ef0a0d9d..22f9f669e0eb 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -98,7 +98,7 @@ local function on_endpoint_modified(endpoint) core.log.error("set endpoint version into discovery DICT failed, ", err) return end - endpoint_dict:safe_set(endpoint_key, endpoint_content) + _, err = endpoint_dict:safe_set(endpoint_key, endpoint_content) if err then core.log.error("set endpoint into discovery DICT failed, ", err) endpoint_dict:delete(endpoint_key .. "#version") @@ -303,6 +303,14 @@ function _M.init_worker() end end + endpoint_informer.pre_list = function(self) + endpoint_dict:flush_all() + end + + endpoint_informer.post_list = function(self) + endpoint_dict:flush_expired() + end + local timer_runner timer_runner = function(premature, informer) if informer:list_watch() then diff --git a/apisix/kubernetes.lua b/apisix/kubernetes.lua index d75bfe9f1154..6af5ff1b010d 100644 --- a/apisix/kubernetes.lua +++ b/apisix/kubernetes.lua @@ -23,12 +23,14 @@ local core = require("apisix.core") local http = require("resty.http") local _constants = { - AddedEven = "ADDED", + ErrorEvent = "ERROR", + AddedEvent = "ADDED", ModifiedEvent = "MODIFIED", DeletedEvent = "DELETED", BookmarkEvent = "BOOKMARK", ListDrive = "list", WatchDrive = "watch", + ErrorGone = 410, } local _apiserver = { @@ -88,8 +90,7 @@ local function list(httpc, informer) end local function watch(httpc, informer) - local max_watch_times = 3 - + local max_watch_times = 5 for _ = 0, max_watch_times do local watch_seconds = 1800 + math.random(9, 999) informer.overtime = watch_seconds @@ -155,13 +156,20 @@ local function watch(httpc, informer) captured_size = captured_size + #captures[0] local v, _ = core.json.decode(captures[0]) - if not v or not v.object or v.object.kind ~= informer.kind then + + if not v or not v.object then return false, "UnexpectedBody", captures[0] end - informer.version = v.object.metadata.resourceVersion local type = v.type - if type == _constants.AddedEven then + if type == _constants.ErrorEvent then + if v.object.code == _constants.ErrorGone then + return true, "Success", nil + end + return false, "UnexpectedBody", captures[0] + end + + if type == _constants.AddedEvent then if informer.on_added ~= nil then informer:on_added(v.object, _constants.WatchDrive) end @@ -186,6 +194,7 @@ local function watch(httpc, informer) end end end + return true, "Success", "" end local _informer_factory = { @@ -200,7 +209,7 @@ function _informer_factory.new(group, version, kind, plural, namespace) version = "", continue = "", overtime = "1800", - limit = 30, + limit = 120, label_selector = "", field_selector = "", } @@ -270,7 +279,7 @@ function _informer_factory.new(group, version, kind, plural, namespace) if not ok then self.fetch_state = "connecting" core.log.error("connect apiserver failed , _apiserver.host: ", _apiserver.host, - ", _apiserver.port", _apiserver.port, ", message : ", message) + ", _apiserver.port: ", _apiserver.port, ", message : ", message) return false end From 3ab6747e70a426d493c63751cd1acd1de31be8ad Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 14:46:13 +0800 Subject: [PATCH 24/73] feat: add kubernetes discovery module --- apisix/kubernetes.lua | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apisix/kubernetes.lua b/apisix/kubernetes.lua index 6af5ff1b010d..3f030e8beb33 100644 --- a/apisix/kubernetes.lua +++ b/apisix/kubernetes.lua @@ -169,17 +169,20 @@ local function watch(httpc, informer) return false, "UnexpectedBody", captures[0] end + local object = v.object + informer.version = object.metadata.resourceVersion + if type == _constants.AddedEvent then if informer.on_added ~= nil then - informer:on_added(v.object, _constants.WatchDrive) + informer:on_added(object, _constants.WatchDrive) end elseif type == _constants.DeletedEvent then if informer.on_deleted ~= nil then - informer:on_deleted(v.object) + informer:on_deleted(object) end elseif type == _constants.ModifiedEvent then if informer.on_modified ~= nil then - informer:on_modified(v.object) + informer:on_modified(object) end -- elseif type == _constants.BookmarkEvent then -- do nothing From 645388921e92ec4115fceab1863cc7e7c89b4d2e Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 16:09:50 +0800 Subject: [PATCH 25/73] feat: add kubernetes discovery module --- .github/workflows/centos7-ci.yml | 31 +++++++++ .github/workflows/cli.yml | 31 +++++++++ ci/common.sh | 83 ----------------------- ci/linux_openresty_common_runner.sh | 2 - t/discovery/kubernetes/account.yaml | 27 ++++++++ t/discovery/kubernetes/kind.yaml | 5 ++ t/discovery/{ => kubernetes}/kubernetes.t | 0 7 files changed, 94 insertions(+), 85 deletions(-) create mode 100644 t/discovery/kubernetes/account.yaml create mode 100644 t/discovery/kubernetes/kind.yaml rename t/discovery/{ => kubernetes}/kubernetes.t (100%) diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 580f90780b71..1c828e2d02d4 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -77,6 +77,37 @@ jobs: docker exec centos7Instance bash -c "cd apisix && rm -rf deps" docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." + - name: using kind setup kubernetes cluster for dicovery.kubernetes test + run: | + KIND_VERSION="v0.11.1" + KUBECTL_VERSION="v1.22.0" + curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" + curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + chmod +x ./kind + chmod +x ./kubectl + + ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml + + ./kubectl wait --for=condition=Ready nodes --all + + ./kubectl apply -f ./t/discovery/kubernetes/account.yaml + + ./kubectl proxy -p 6645 + + KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') + + KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" + + KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token + + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} + echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > {KUBERNETES_CLIENT_TOKEN_FILE} + + echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' + echo 'KUBERNETES_SERVICE_PORT=6443' + echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" + echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} + - name: Run test cases run: | docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh run_case" diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 32f3feed5306..d4e8ae027a87 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -62,5 +62,36 @@ jobs: sudo --preserve-env=OPENRESTY_VERSION \ ./ci/${{ matrix.job_name }}_runner.sh do_install + - name: using kind setup kubernetes cluster for dicovery.kubernetes test + run: | + KIND_VERSION="v0.11.1" + KUBECTL_VERSION="v1.22.0" + curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" + curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + chmod +x ./kind + chmod +x ./kubectl + + ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml + + ./kubectl wait --for=condition=Ready nodes --all + + ./kubectl apply -f ./t/discovery/kubernetes/account.yaml + + ./kubectl proxy -p 6645 + + KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') + + KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" + + KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token + + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} + echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > {KUBERNETES_CLIENT_TOKEN_FILE} + + echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' + echo 'KUBERNETES_SERVICE_PORT=6443' + echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" + echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} + - name: Linux Script run: sudo ./ci/${{ matrix.job_name }}_runner.sh script diff --git a/ci/common.sh b/ci/common.sh index 21e6022a9122..fe015b16d666 100644 --- a/ci/common.sh +++ b/ci/common.sh @@ -75,86 +75,3 @@ install_nodejs () { } GRPC_SERVER_EXAMPLE_VER=20210819 - -install_k8s () { - # create kubernetes cluster using kind - KIND_VERSION="v0.11.1" - KUBECTL_VERSION="v1.22.0" - curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" - curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - chmod +x ./kind - chmod +x ./kubectl - - echo -e " -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: - apiServerAddress: 127.0.0.1 - apiServerPort: 6443 -" >kind.yaml - - ./kind delete cluster --name apisix-test - ./kind create cluster --name apisix-test --config ./kind.yaml - - echo "wait k8s start..." - sleep 10 - until [[ $(./kubectl get pods -A --field-selector 'status.phase!=Running' 2>&1) =~ "No resources found" ]]; do - echo 'still wait k8s start...' - sleep 1 - done - - echo -e " -kind: ServiceAccount -apiVersion: v1 -metadata: - name: apisix-test - namespace: default ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: apisix-test -rules: - - apiGroups: [ \"\" ] - resources: [ endpoints ] - verbs: [ get,list,watch ] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: apisix-test -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: apisix-test -subjects: - - kind: ServiceAccount - name: apisix-test - namespace: default -" >apisix-test-rbac.yaml - - ./kubectl apply -f ./apisix-test-rbac.yaml - ./kubectl proxy -p 6445 & - - KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - - # if we do not have permission to create folders under the /var/run path, we will use the /tmp as an alternative - KUBERNETES_CLIENT_TOKEN_DIR="/var/run/secrets/kubernetes.io/serviceaccount" - KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token - - if ! mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR}; then - KUBERNETES_CLIENT_TOKEN_DIR=/tmp${KUBERNETES_CLIENT_TOKEN_DIR} - KUBERNETES_CLIENT_TOKEN_FILE=/tmp${KUBERNETES_CLIENT_TOKEN_FILE} - mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} - fi - - if ! echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" >${KUBERNETES_CLIENT_TOKEN_FILE}; then - echo 'save kubernetes token file error' - exit 1 - fi - - echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' - echo 'KUBERNETES_SERVICE_PORT=6443' - echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" - echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} -} diff --git a/ci/linux_openresty_common_runner.sh b/ci/linux_openresty_common_runner.sh index 9e9fbaf56e74..91bd9950afe1 100755 --- a/ci/linux_openresty_common_runner.sh +++ b/ci/linux_openresty_common_runner.sh @@ -69,8 +69,6 @@ do_install() { # install vault cli capabilities install_vault_cli - # install k8s - install_k8s } script() { diff --git a/t/discovery/kubernetes/account.yaml b/t/discovery/kubernetes/account.yaml new file mode 100644 index 000000000000..22101b77e81d --- /dev/null +++ b/t/discovery/kubernetes/account.yaml @@ -0,0 +1,27 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: apisix-test + namespace: default +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: apisix-test +rules: + - apiGroups: [ \"\" ] + resources: [ endpoints ] + verbs: [ get,list,watch ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: apisix-test +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: apisix-test +subjects: + - kind: ServiceAccount + name: apisix-test + namespace: default diff --git a/t/discovery/kubernetes/kind.yaml b/t/discovery/kubernetes/kind.yaml new file mode 100644 index 000000000000..821e3827a433 --- /dev/null +++ b/t/discovery/kubernetes/kind.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerAddress: 127.0.0.1 + apiServerPort: 6443 diff --git a/t/discovery/kubernetes.t b/t/discovery/kubernetes/kubernetes.t similarity index 100% rename from t/discovery/kubernetes.t rename to t/discovery/kubernetes/kubernetes.t From d9ef13666c87143cc296dc1cbf1757b105e18467 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 16:23:09 +0800 Subject: [PATCH 26/73] feat: add kubernetes discovery module --- .github/workflows/centos7-ci.yml | 18 +++++++++--------- .github/workflows/cli.yml | 22 +++++++++++----------- ci/linux_openresty_common_runner.sh | 1 - t/discovery/kubernetes/account.yaml | 19 ++++++++++++++++++- t/discovery/kubernetes/kind.yaml | 17 +++++++++++++++++ 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 1c828e2d02d4..456e043152af 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -77,7 +77,7 @@ jobs: docker exec centos7Instance bash -c "cd apisix && rm -rf deps" docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." - - name: using kind setup kubernetes cluster for dicovery.kubernetes test + - name: Setup kubernetes cluster run: | KIND_VERSION="v0.11.1" KUBECTL_VERSION="v1.22.0" @@ -85,24 +85,24 @@ jobs: curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" chmod +x ./kind chmod +x ./kubectl - + ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml - + ./kubectl wait --for=condition=Ready nodes --all ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - + ./kubectl proxy -p 6645 - + KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - + KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token - + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} - echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > {KUBERNETES_CLIENT_TOKEN_FILE} - + echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > ${KUBERNETES_CLIENT_TOKEN_FILE} + echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' echo 'KUBERNETES_SERVICE_PORT=6443' echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index d4e8ae027a87..272dbc8cd99b 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -62,7 +62,7 @@ jobs: sudo --preserve-env=OPENRESTY_VERSION \ ./ci/${{ matrix.job_name }}_runner.sh do_install - - name: using kind setup kubernetes cluster for dicovery.kubernetes test + - name: Setup kubernetes cluster run: | KIND_VERSION="v0.11.1" KUBECTL_VERSION="v1.22.0" @@ -70,24 +70,24 @@ jobs: curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" chmod +x ./kind chmod +x ./kubectl - + ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml - + ./kubectl wait --for=condition=Ready nodes --all - + ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - + ./kubectl proxy -p 6645 - + KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - + KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" - + KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token - + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} - echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > {KUBERNETES_CLIENT_TOKEN_FILE} - + echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > ${KUBERNETES_CLIENT_TOKEN_FILE} + echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' echo 'KUBERNETES_SERVICE_PORT=6443' echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" diff --git a/ci/linux_openresty_common_runner.sh b/ci/linux_openresty_common_runner.sh index 91bd9950afe1..ced0b83ffdfe 100755 --- a/ci/linux_openresty_common_runner.sh +++ b/ci/linux_openresty_common_runner.sh @@ -68,7 +68,6 @@ do_install() { # install vault cli capabilities install_vault_cli - } script() { diff --git a/t/discovery/kubernetes/account.yaml b/t/discovery/kubernetes/account.yaml index 22101b77e81d..da7cf01f5cad 100644 --- a/t/discovery/kubernetes/account.yaml +++ b/t/discovery/kubernetes/account.yaml @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + kind: ServiceAccount apiVersion: v1 metadata: @@ -9,7 +26,7 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: apisix-test rules: - - apiGroups: [ \"\" ] + - apiGroups: [ "" ] resources: [ endpoints ] verbs: [ get,list,watch ] --- diff --git a/t/discovery/kubernetes/kind.yaml b/t/discovery/kubernetes/kind.yaml index 821e3827a433..3db903fcad64 100644 --- a/t/discovery/kubernetes/kind.yaml +++ b/t/discovery/kubernetes/kind.yaml @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: From 5903685850e26fa335ebb4d0a3b1ab29d1b2b9ce Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 16:28:00 +0800 Subject: [PATCH 27/73] feat: add kubernetes discovery module --- .github/workflows/build.yml | 31 +++++++++++++++++++++++++++++++ .github/workflows/cli.yml | 31 ------------------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a99778aac334..b570a001c935 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,5 +103,36 @@ jobs: sudo --preserve-env=OPENRESTY_VERSION \ ./ci/${{ matrix.os_name }}_runner.sh do_install + - name: Setup kubernetes cluster + run: | + KIND_VERSION="v0.11.1" + KUBECTL_VERSION="v1.22.0" + curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" + curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + chmod +x ./kind + chmod +x ./kubectl + + ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml + + ./kubectl wait --for=condition=Ready nodes --all + + ./kubectl apply -f ./t/discovery/kubernetes/account.yaml + + ./kubectl proxy -p 6645 + + KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') + + KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" + + KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token + + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} + echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > ${KUBERNETES_CLIENT_TOKEN_FILE} + + echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' + echo 'KUBERNETES_SERVICE_PORT=6443' + echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" + echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} + - name: Linux Script run: sudo ./ci/${{ matrix.os_name }}_runner.sh script diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 272dbc8cd99b..32f3feed5306 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -62,36 +62,5 @@ jobs: sudo --preserve-env=OPENRESTY_VERSION \ ./ci/${{ matrix.job_name }}_runner.sh do_install - - name: Setup kubernetes cluster - run: | - KIND_VERSION="v0.11.1" - KUBECTL_VERSION="v1.22.0" - curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" - curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - chmod +x ./kind - chmod +x ./kubectl - - ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml - - ./kubectl wait --for=condition=Ready nodes --all - - ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - - ./kubectl proxy -p 6645 - - KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - - KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" - - KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token - - mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} - echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > ${KUBERNETES_CLIENT_TOKEN_FILE} - - echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' - echo 'KUBERNETES_SERVICE_PORT=6443' - echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" - echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} - - name: Linux Script run: sudo ./ci/${{ matrix.job_name }}_runner.sh script From 4cf63f3c0a18cd3571fd8bbf32f2278ee19530e3 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 16:41:26 +0800 Subject: [PATCH 28/73] feat: add kubernetes discovery module --- .github/workflows/build.yml | 2 +- .github/workflows/centos7-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b570a001c935..673bcb0d350a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - ./kubectl proxy -p 6645 + ./kubectl proxy -p 6645 & KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 456e043152af..62f5bd2b4707 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -92,7 +92,7 @@ jobs: ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - ./kubectl proxy -p 6645 + ./kubectl proxy -p 6645 & KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') From 34e2ebb2f4f676e8dd3f218df642005e61093ec3 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 17:20:42 +0800 Subject: [PATCH 29/73] feat: add kubernetes discovery module --- .github/workflows/build.yml | 2 +- .github/workflows/centos7-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 673bcb0d350a..cc5053ce1f9d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - ./kubectl proxy -p 6645 & + ./kubectl proxy -p 6445 & KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 62f5bd2b4707..f97eec9b9af5 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -92,7 +92,7 @@ jobs: ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - ./kubectl proxy -p 6645 & + ./kubectl proxy -p 6445 & KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') From b05f7ebd90b898e71068f2553a352faf5f2ad255 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 18:47:14 +0800 Subject: [PATCH 30/73] feat: add kubernetes discovery module --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc5053ce1f9d..44330ab6457d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,7 +122,7 @@ jobs: KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" + KUBERNETES_CLIENT_TOKEN_DIR="/var/run/secrets/kubernetes.io/serviceaccount" KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token From dbc5ca4af0b822e7fbdae59b4f59ff8d98202f8e Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 18:48:08 +0800 Subject: [PATCH 31/73] feat: add kubernetes discovery module --- docs/zh/latest/discovery/kubernetes.md | 105 +++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 docs/zh/latest/discovery/kubernetes.md diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md new file mode 100644 index 000000000000..0aec2d992ece --- /dev/null +++ b/docs/zh/latest/discovery/kubernetes.md @@ -0,0 +1,105 @@ + + +# 基于 Kubernetes 的服务发现 + +Kubernetes 服务发现插件以 ListWatch 方式监听 Kubernetes 集群的 的 endpoints/v1 资源的实时变化, 并将其值存储在 shared.DICT 中, +以及提供对外查询接口 + +# Kubernetes 服务发现插件的配置 + +Kubernetes 服务发现的样例配置如下: + +```yaml +discovery: + kubernetes: + service: + #kubernetes apiserver schema , option [ http | https ], default https + schema: https + + # kubernetes apiserver host, you can set ipv4,ipv6 ,domain or environment variable + # default ${KUBERNETES_SERVICE_HOST} + host: 10.0.8.95 + + # kubernetes apiserver port, you can set number or environment variable + # default ${KUBERNETES_SERVICE_PORT} + port: 6443 + + client: + # kubernetes serviceaccount token or token_file + # default setup token_file and value is "/var/run/secrets/kubernetes.io/serviceaccount/token" + #token: + token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" + + # kubernetes discovery plugin support watch endpoints in specific namespace + # you can use [ equal | not_equal | match | not_match ] to filter your specified namespace + # [ match | not_match ] support regular expression (using ngx.re.match ) + namespace_selector: + equal: default +# not_equal: +# match: +# not_match: +``` + +如果 Kubernetes 服务插件运行在 Pod 内, 你可以使用最简配置: + +```yaml +discovery: + kubernetes: { } +``` + +如果 Kubernetes 服务插件运行在 Pod 外, 你需要新建或选取指定的 ServiceAccount, 获取其 Token 值, 并使用如下配置: + +```yaml +discovery: + kubernetes: + service: + host: [ ApiServer Host Value Here ] + port: [ ApiServer Port Value Here ] + schema: https + client: + token: [ ServiceAccount Token Value Here ] +``` + +# Kubernetes 服务发现插件的使用 + +Kubernetes 服务发现插件提供与其他服务发现相同的查询接口 -> nodes(service_name) + +其中 service_name 的 pattern 如下: +> _[namespace]/[name]:[portName]_ + +某些 endpoints 可能没有定义 portName, Kubernetes 服务发现插件会依次使用 targetPort, port 代替 + +# Q&A + +> Q: 为什么只支持配置 token 来访问 kubernetes apiserver ,可以使用 kubernetes config 吗 + +> A: 通常情况下,我们会使用三种方式与 kubernetes Apiserver 通信 : +> + mTLS +> + token +> + basic authentication \ +> 但因为 apisix.http 目前并没有支持 mTLS ,以及 basic authentication 并不被推荐使用,\ +> 所以只实现了 token 的方式, 这也意味着不支持 kubernetes config +------- +> Q: APISix 是多进程模型, 是否意味着每个 APISix 业务进程都会去 监听 kubernetes apiserver + +> A: Kubernetes 服务发现插件只使用特权进程监听 kubernetes 集群,然后将获取到结果存储在 ngx.shared.DICT中 业务进程是通过查询 ngx.shared.DICT 获取结果的 + +------ + From 29daca14ccedb3a3877b08e61b15cb1e598cab65 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 19:10:19 +0800 Subject: [PATCH 32/73] feat: add kubernetes discovery module --- .github/workflows/build.yml | 2 +- .github/workflows/centos7-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 44330ab6457d..e7cbd2d146bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,7 +114,7 @@ jobs: ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml - ./kubectl wait --for=condition=Ready nodes --all + ./kubectl wait --for=condition=Ready nodes --all --timeout=180s ./kubectl apply -f ./t/discovery/kubernetes/account.yaml diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index f97eec9b9af5..5765aa91351c 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -88,7 +88,7 @@ jobs: ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml - ./kubectl wait --for=condition=Ready nodes --all + ./kubectl wait --for=condition=Ready nodes --all --timeout=180s ./kubectl apply -f ./t/discovery/kubernetes/account.yaml From 5a45fdd53dd0320702b8917739e72d1f33c21ddf Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 12 Jan 2022 20:54:59 +0800 Subject: [PATCH 33/73] feat: add kubernetes discovery module --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7cbd2d146bd..17b40935419e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,7 +122,7 @@ jobs: KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - KUBERNETES_CLIENT_TOKEN_DIR="/var/run/secrets/kubernetes.io/serviceaccount" + KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token From e0f893fa850592c181b1c2d20a76f6d9ba0b06cd Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 17:09:38 +0800 Subject: [PATCH 34/73] feat: add kubernetes discovery module --- .github/workflows/build.yml | 31 ---- .github/workflows/centos7-ci.yml | 33 +--- .github/workflows/kubernetes-discovery.yml | 123 +++++++++++++ apisix/discovery/kubernetes/init.lua | 37 ++-- apisix/discovery/kubernetes/schema.lua | 3 + apisix/kubernetes.lua | 192 ++++++++++++--------- ci/kubernetes-discovery-ci.sh | 81 +++++++++ 7 files changed, 344 insertions(+), 156 deletions(-) create mode 100644 .github/workflows/kubernetes-discovery.yml create mode 100755 ci/kubernetes-discovery-ci.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c5d45140bc71..5e5891e5da76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,36 +103,5 @@ jobs: sudo --preserve-env=OPENRESTY_VERSION \ ./ci/${{ matrix.os_name }}_runner.sh do_install - - name: Setup kubernetes cluster - run: | - KIND_VERSION="v0.11.1" - KUBECTL_VERSION="v1.22.0" - curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" - curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - chmod +x ./kind - chmod +x ./kubectl - - ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml - - ./kubectl wait --for=condition=Ready nodes --all --timeout=180s - - ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - - ./kubectl proxy -p 6445 & - - KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - - KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" - - KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token - - mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} - echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > ${KUBERNETES_CLIENT_TOKEN_FILE} - - echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' - echo 'KUBERNETES_SERVICE_PORT=6443' - echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" - echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} - - name: Linux Script run: sudo ./ci/${{ matrix.os_name }}_runner.sh script diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 5765aa91351c..bb306f9160df 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -57,7 +57,7 @@ jobs: - name: Run centos7 docker and mapping apisix into container run: | - docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 -v /tmp:/tmp --dns-search apache.org docker.io/centos:7 /bin/bash + docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 --dns-search apache.org docker.io/centos:7 /bin/bash # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" - name: Run other docker containers for test @@ -77,37 +77,6 @@ jobs: docker exec centos7Instance bash -c "cd apisix && rm -rf deps" docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." - - name: Setup kubernetes cluster - run: | - KIND_VERSION="v0.11.1" - KUBECTL_VERSION="v1.22.0" - curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" - curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - chmod +x ./kind - chmod +x ./kubectl - - ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml - - ./kubectl wait --for=condition=Ready nodes --all --timeout=180s - - ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - - ./kubectl proxy -p 6445 & - - KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - - KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" - - KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token - - mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} - echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > ${KUBERNETES_CLIENT_TOKEN_FILE} - - echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' - echo 'KUBERNETES_SERVICE_PORT=6443' - echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" - echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} - - name: Run test cases run: | docker exec centos7Instance bash -c "cd apisix && ./ci/centos7-ci.sh run_case" diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml new file mode 100644 index 000000000000..7e083eebf168 --- /dev/null +++ b/.github/workflows/kubernetes-discovery.yml @@ -0,0 +1,123 @@ +name: CI Centos7 + +on: + push: + branches: [ master, 'release/**' ] + paths-ignore: + - 'docs/**' + - '**/*.md' + pull_request: + branches: [ master, 'release/**' ] + paths-ignore: + - 'docs/**' + - '**/*.md' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} + cancel-in-progress: true + +jobs: + build: + strategy: + fail-fast: false + matrix: + platform: + - ubuntu-18.04 + os_name: + - linux_openresty + - linux_openresty_1_17 + + runs-on: ${{ matrix.platform }} + timeout-minutes: 90 + env: + SERVER_NAME: ${{ matrix.os_name }} + OPENRESTY_VERSION: default + + steps: + - name: Setup kubernetes cluster + run: | + KIND_VERSION="v0.11.1" + KUBECTL_VERSION="v1.22.0" + curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" + curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + chmod +x ./kind + chmod +x ./kubectl + + ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml + + ./kubectl wait --for=condition=Ready nodes --all --timeout=180s + + ./kubectl apply -f ./t/discovery/kubernetes/account.yaml + + KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') + + KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" + + KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token + + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} + echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > ${KUBERNETES_CLIENT_TOKEN_FILE} + + echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' + echo 'KUBERNETES_SERVICE_PORT=6443' + echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" + echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} + + ./kubectl proxy -p 6445 & + + - name: Check out code + uses: actions/checkout@v2.4.0 + with: + submodules: recursive + + - name: Extract branch name + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + id: branch_env + shell: bash + run: | + echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})" + + - name: Install dependencies + run: | + sudo ./ci/kubernetes-discovery-ci.sh install_dependencies_in_ubuntu + + - name: Linux Script + run: | + sudo ./ci/kubernetes-discovery-ci.sh run_case + + - name: Build rpm package + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + run: | + export VERSION=${{ steps.branch_env.outputs.version }} + sudo gem install --no-document fpm + git clone -b v2.7.0 https://github.com/api7/apisix-build-tools.git + + # move codes under build tool + mkdir ./apisix-build-tools/apisix + for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done + + cd apisix-build-tools + make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix + cd .. + rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) + + - name: Run centos7 docker and mapping apisix into container + run: | + docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 -v /tmp:/tmp --dns-search apache.org docker.io/centos:7 /bin/bash + # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" + + - name: Install dependencies + run: | + docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery-ci.sh install_dependencies_in_centos" + + - name: Install rpm package + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + run: | + docker exec centos7Instance bash -c "cd apisix && rpm -iv --prefix=/apisix ./apisix-build-tools/output/apisix-${{ steps.branch_env.outputs.version }}-0.el7.x86_64.rpm" + # Dependencies are attached with rpm, so revert `make deps` + docker exec centos7Instance bash -c "cd apisix && rm -rf deps" + docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." + + - name: Run test cases + run: | + docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery-ci.sh run_case" diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 22f9f669e0eb..ca9caf79711c 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -40,7 +40,7 @@ local endpoint_lrucache = core.lrucache.new({ local endpoint_buffer = {} local empty_table = {} -local function sort_cmp(left, right) +local function sort_nodes_cmp(left, right) if left.host ~= right.host then return left.host < right.host end @@ -84,7 +84,7 @@ local function on_endpoint_modified(endpoint) for _, ports in pairs(endpoint_buffer) do for _, nodes in pairs(ports) do - core.table.sort(nodes, sort_cmp) + core.table.sort(nodes, sort_nodes_cmp) end end @@ -112,7 +112,15 @@ local function on_endpoint_deleted(endpoint) endpoint_dict:delete(endpoint_key) end -local function set_namespace_selector(conf, informer) +local function setup_label_selector(conf, informer) + local ls = conf.label_selector + if ls == nil or ls == "" then + return + end + informer.label_selector = ngx.escape_uri(ls) +end + +local function setup_namespace_selector(conf, informer) local ns = conf.namespace_selector if ns == nil then informer.namespace_selector = nil @@ -172,7 +180,7 @@ local function read_env(key) return true, key, nil end -local function read_conf(conf, apiserver) +local function setup_apiserver(conf, apiserver) apiserver.schema = conf.service.schema @@ -220,8 +228,6 @@ local function read_conf(conf, apiserver) "should set one of [client.token,client.token_file] but none" end - default_weight = conf.default_weight or 50 - return true, nil end @@ -270,7 +276,11 @@ function _M.init_worker() return end - local ok, err = read_conf(local_conf.discovery.kubernetes, kubernetes.apiserver) + local discovery_conf = local_conf.discovery.kubernetes + + default_weight = discovery_conf.default_weight or 50 + + local ok, err = setup_apiserver(discovery_conf, kubernetes.apiserver) if not ok then error(err) return @@ -279,14 +289,13 @@ function _M.init_worker() local endpoint_informer = kubernetes.informer_factory.new("", "v1", "Endpoints", "endpoints", "") - set_namespace_selector(local_conf.discovery.kubernetes, endpoint_informer) + setup_label_selector(discovery_conf, endpoint_informer) + setup_namespace_selector(discovery_conf, endpoint_informer) endpoint_informer.on_added = function(self, object, drive) - if self.namespace_selector ~= nil then - if self:namespace_selector(object.metadata.namespace) then - on_endpoint_modified(object) - end - else + if self.namespace_selector == nil then + on_endpoint_modified(object) + elseif self:namespace_selector(object.metadata.namespace) then on_endpoint_modified(object) end end @@ -313,7 +322,7 @@ function _M.init_worker() local timer_runner timer_runner = function(premature, informer) - if informer:list_watch() then + if not informer:list_watch() then local retry_interval = 40 ngx.sleep(retry_interval) end diff --git a/apisix/discovery/kubernetes/schema.lua b/apisix/discovery/kubernetes/schema.lua index 8bc32910eaef..4888de63c484 100644 --- a/apisix/discovery/kubernetes/schema.lua +++ b/apisix/discovery/kubernetes/schema.lua @@ -122,6 +122,9 @@ return { { required = { "not_match" } } }, }, + label_selector = { + type = "string", + } }, default = { service = { diff --git a/apisix/kubernetes.lua b/apisix/kubernetes.lua index 3f030e8beb33..e6166da9e2f3 100644 --- a/apisix/kubernetes.lua +++ b/apisix/kubernetes.lua @@ -28,9 +28,16 @@ local _constants = { ModifiedEvent = "MODIFIED", DeletedEvent = "DELETED", BookmarkEvent = "BOOKMARK", + ListDrive = "list", WatchDrive = "watch", - ErrorGone = 410, + + Success = "Success", + RequestError = "RequestError", + ReadBodyError = "ReadBodyError", + UnexpectedBody = "UnexpectedBody", + GmatchError = "GmatchError", + ResourceGone = "ResourceGone", } local _apiserver = { @@ -43,7 +50,7 @@ local _apiserver = { local empty_table = {} local function list(httpc, informer) - local res, err = httpc:request({ + local response, err = httpc:request({ path = informer.path, query = informer:list_query(), headers = { @@ -56,21 +63,22 @@ local function list(httpc, informer) core.log.info("--raw=" .. informer.path .. "?" .. informer:list_query()) - if not res then - return false, "RequestError", err or "" + if not response then + return false, _constants.RequestError, err or "" end - if res.status ~= 200 then - return false, res.reason, res:read_body() or "" + if response.status ~= 200 then + return false, response.reason, response:read_body() or "" end - local body, err = res:read_body() + + local body, err = response:read_body() if err then - return false, "ReadBodyError", err + return false, _constants.ReadBodyError, err end local data, _ = core.json.decode(body) if not data or data.kind ~= informer.list_kind then - return false, "UnexpectedBody", body + return false, _constants.UnexpectedBody, body end informer.version = data.metadata.resourceVersion @@ -86,18 +94,96 @@ local function list(httpc, informer) list(httpc, informer) end - return true, "Success", "" + return true, _constants.Success, "" +end + +local function split_event (body, dispatch_event) + local gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") + if not gmatch_iterator then + return nil, _constants.GmatchError, err + end + + local captures + local captured_size = 0 + local ok, reason + while true do + captures, err = gmatch_iterator() + + if err then + return nil, _constants.GmatchError, err + end + + if not captures then + break + end + + captured_size = captured_size + #captures[0] + + ok, reason, err = dispatch_event(captures[0]) + if not ok then + return nil, reason, err + end + end + + local remainder_body + if captured_size == #body then + remainder_body = "" + elseif captured_size == 0 then + remainder_body = body + elseif captured_size < #body then + remainder_body = string.sub(body, captured_size + 1) + end + + return remainder_body, _constants.Success, nil end local function watch(httpc, informer) - local max_watch_times = 5 - for _ = 0, max_watch_times do + + local dispatch_event = function(event_string) + local event, _ = core.json.decode(event_string) + + if not event or not event.type or not event.object then + return false, _constants.UnexpectedBody, event_string + end + + local type = event.type + + if type == _constants.ErrorEvent then + if event.object.code == 410 then + return false, _constants.ResourceGone, nil + end + return false, _constants.UnexpectedBody, event_string + end + + local object = event.object + informer.version = object.metadata.resourceVersion + + if type == _constants.AddedEvent then + if informer.on_added ~= nil then + informer:on_added(object, _constants.WatchDrive) + end + elseif type == _constants.DeletedEvent then + if informer.on_deleted ~= nil then + informer:on_deleted(object) + end + elseif type == _constants.ModifiedEvent then + if informer.on_modified ~= nil then + informer:on_modified(object) + end + -- elseif type == _constants.BookmarkEvent then + -- do nothing + end + return true, _constants.Success, nil + end + + local watch_times = 5 + for _ = 0, watch_times do local watch_seconds = 1800 + math.random(9, 999) informer.overtime = watch_seconds local http_seconds = watch_seconds + 120 httpc:set_timeouts(2000, 3000, http_seconds * 1000) - local res, err = httpc:request({ + local response, err = httpc:request({ path = informer.path, query = informer:watch_query(), headers = { @@ -111,93 +197,41 @@ local function watch(httpc, informer) core.log.info("--raw=" .. informer.path .. "?" .. informer:watch_query()) if err then - return false, "RequestError", err + return false, _constants.RequestError, err end - if res.status ~= 200 then - return false, res.reason, res:read_body() or "" + if response.status ~= 200 then + return false, response.reason, response:read_body() or "" end - local remainder_body = "" + local remainder_body local body - local reader = res.body_reader - local gmatch_iterator - local captures - local captured_size = 0 + local reason while true do - body, err = reader() + body, err = response.body_reader() if err then - return false, "ReadBodyError", err + return false, _constants.ReadBodyError, err end if not body then break end - if #remainder_body ~= 0 then + if remainder_body ~= nil and #remainder_body > 0 then body = remainder_body .. body end - gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") - if not gmatch_iterator then - return false, "GmatchError", err - end - - captures, err = gmatch_iterator() - - if err then - return false, "GmatchError", err - end - - if not captures then - break - end - - captured_size = captured_size + #captures[0] - local v, _ = core.json.decode(captures[0]) - - if not v or not v.object then - return false, "UnexpectedBody", captures[0] - end - - local type = v.type - if type == _constants.ErrorEvent then - if v.object.code == _constants.ErrorGone then - return true, "Success", nil + remainder_body, reason, err = split_event(body, dispatch_event) + if reason ~= _constants.Success then + if reason == _constants.ResourceGone then + return true, _constants.Success, nil end - return false, "UnexpectedBody", captures[0] - end - - local object = v.object - informer.version = object.metadata.resourceVersion - - if type == _constants.AddedEvent then - if informer.on_added ~= nil then - informer:on_added(object, _constants.WatchDrive) - end - elseif type == _constants.DeletedEvent then - if informer.on_deleted ~= nil then - informer:on_deleted(object) - end - elseif type == _constants.ModifiedEvent then - if informer.on_modified ~= nil then - informer:on_modified(object) - end - -- elseif type == _constants.BookmarkEvent then - -- do nothing - end - - if captured_size == #body then - remainder_body = "" - elseif captured_size == 0 then - remainder_body = body - else - remainder_body = string.sub(body, captured_size + 1) + return false, reason, err end end end - return true, "Success", "" + return true, _constants.Success, nil end local _informer_factory = { diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery-ci.sh new file mode 100755 index 000000000000..4a0ac9e08143 --- /dev/null +++ b/ci/kubernetes-discovery-ci.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. ./ci/common.sh + +install_dependencies_in_ubuntu() { + export_or_prefix + + # install openresty + ./utils/linux-install-openresty.sh + + # install luarocks + ./utils/linux-install-luarocks.sh + + # install test::nginx + yum install -y cpanminus perl + cpanm --notest Test::Nginx IPC::Run > build.log 2>&1 || (cat build.log && exit 1) + + # install dependencies + git clone https://github.com/iresty/test-nginx.git test-nginx + create_lua_deps +} + +install_dependencies_in_centos() { + export_or_prefix + + # install development tools + yum install -y wget tar gcc automake autoconf libtool make unzip \ + git which sudo openldap-devel + + # curl with http2 + wget https://github.com/moparisthebest/static-curl/releases/download/v7.79.1/curl-amd64 -O /usr/bin/curl + # install openresty to make apisix's rpm test work + yum install -y yum-utils && yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo + yum install -y openresty openresty-debug openresty-openssl111-debug-devel pcre pcre-devel + + # install luarocks + ./utils/linux-install-luarocks.sh + + # install test::nginx + yum install -y cpanminus perl + cpanm --notest Test::Nginx IPC::Run > build.log 2>&1 || (cat build.log && exit 1) + + # install dependencies + git clone https://github.com/iresty/test-nginx.git test-nginx + create_lua_deps +} + +run_case() { + export_or_prefix + make init + ./utils/set-dns.sh + # run test cases + prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result + rerun_flaky_tests /tmp/test.result +} + +case_opt=$1 +case $case_opt in + (install_dependencies) + install_dependencies + ;; + (run_case) + run_case + ;; +esac From 016a0497673daf826312c6371b8cfe125b908768 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 17:12:05 +0800 Subject: [PATCH 35/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index 7e083eebf168..48d37d2c6fdf 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -1,4 +1,4 @@ -name: CI Centos7 +name: Kuberbetes Discovery on: push: From 4d69480d19ffb026511b8eba627b9468e4df8d40 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 17:13:55 +0800 Subject: [PATCH 36/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index 48d37d2c6fdf..f21ee91f05e0 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -34,6 +34,18 @@ jobs: OPENRESTY_VERSION: default steps: + - name: Check out code + uses: actions/checkout@v2.4.0 + with: + submodules: recursive + + - name: Extract branch name + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + id: branch_env + shell: bash + run: | + echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})" + - name: Setup kubernetes cluster run: | KIND_VERSION="v0.11.1" @@ -65,18 +77,6 @@ jobs: ./kubectl proxy -p 6445 & - - name: Check out code - uses: actions/checkout@v2.4.0 - with: - submodules: recursive - - - name: Extract branch name - if: ${{ startsWith(github.ref, 'refs/heads/release/') }} - id: branch_env - shell: bash - run: | - echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})" - - name: Install dependencies run: | sudo ./ci/kubernetes-discovery-ci.sh install_dependencies_in_ubuntu From 2ff105dd21fd9a7b807a720d9b9e8477c17db3f5 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 17:25:00 +0800 Subject: [PATCH 37/73] feat: add kubernetes discovery module --- ci/kubernetes-discovery-ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery-ci.sh index 4a0ac9e08143..0929bbd1f9c9 100755 --- a/ci/kubernetes-discovery-ci.sh +++ b/ci/kubernetes-discovery-ci.sh @@ -28,7 +28,7 @@ install_dependencies_in_ubuntu() { ./utils/linux-install-luarocks.sh # install test::nginx - yum install -y cpanminus perl + apt install -y cpanminus perl cpanm --notest Test::Nginx IPC::Run > build.log 2>&1 || (cat build.log && exit 1) # install dependencies @@ -64,7 +64,7 @@ install_dependencies_in_centos() { run_case() { export_or_prefix make init - ./utils/set-dns.sh + # run test cases prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result rerun_flaky_tests /tmp/test.result From a8bf021ee3e6362ccebb9534f39b3a1d79d62c6b Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 17:56:21 +0800 Subject: [PATCH 38/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 32 ++++++++++++---------- ci/kubernetes-discovery-ci.sh | 18 ------------ 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index f21ee91f05e0..366df12ef099 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -54,36 +54,40 @@ jobs: curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" chmod +x ./kind chmod +x ./kubectl - + ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml - + ./kubectl wait --for=condition=Ready nodes --all --timeout=180s - + ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - + KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') - + KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount" - + KUBERNETES_CLIENT_TOKEN_FILE=${KUBERNETES_CLIENT_TOKEN_DIR}/token - + mkdir -p ${KUBERNETES_CLIENT_TOKEN_DIR} echo -n "$KUBERNETES_CLIENT_TOKEN_CONTENT" > ${KUBERNETES_CLIENT_TOKEN_FILE} - + echo 'KUBERNETES_SERVICE_HOST=127.0.0.1' echo 'KUBERNETES_SERVICE_PORT=6443' echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} - + ./kubectl proxy -p 6445 & - - name: Install dependencies + - name: Linux Before install run: | - sudo ./ci/kubernetes-discovery-ci.sh install_dependencies_in_ubuntu + sudo ./ci/linux_openresty_common_runner.sh before_install + + - name: Linux Install + run: | + sudo --preserve-env=OPENRESTY_VERSION ./ci/linux_openresty_common_runner.sh do_install - name: Linux Script run: | - sudo ./ci/kubernetes-discovery-ci.sh run_case + ./ci/kubernetes-discovery-ci.sh run_case - name: Build rpm package if: ${{ startsWith(github.ref, 'refs/heads/release/') }} @@ -91,11 +95,11 @@ jobs: export VERSION=${{ steps.branch_env.outputs.version }} sudo gem install --no-document fpm git clone -b v2.7.0 https://github.com/api7/apisix-build-tools.git - + # move codes under build tool mkdir ./apisix-build-tools/apisix for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done - + cd apisix-build-tools make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix cd .. diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery-ci.sh index 0929bbd1f9c9..0294c8769d59 100755 --- a/ci/kubernetes-discovery-ci.sh +++ b/ci/kubernetes-discovery-ci.sh @@ -18,24 +18,6 @@ . ./ci/common.sh -install_dependencies_in_ubuntu() { - export_or_prefix - - # install openresty - ./utils/linux-install-openresty.sh - - # install luarocks - ./utils/linux-install-luarocks.sh - - # install test::nginx - apt install -y cpanminus perl - cpanm --notest Test::Nginx IPC::Run > build.log 2>&1 || (cat build.log && exit 1) - - # install dependencies - git clone https://github.com/iresty/test-nginx.git test-nginx - create_lua_deps -} - install_dependencies_in_centos() { export_or_prefix From 58095a58022b8a3a272cf37b2ccd957d95457b09 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 19:28:41 +0800 Subject: [PATCH 39/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index 366df12ef099..116d7ff67774 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -77,15 +77,20 @@ jobs: ./kubectl proxy -p 6445 & + - name: Linux Get dependencies + run: | + sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3 libpcre3-dev libldap2-dev + - name: Linux Before install run: | - sudo ./ci/linux_openresty_common_runner.sh before_install + sudo ./ci/${{ matrix.os_name }}_runner.sh before_install - name: Linux Install run: | - sudo --preserve-env=OPENRESTY_VERSION ./ci/linux_openresty_common_runner.sh do_install + sudo --preserve-env=OPENRESTY_VERSION \ + ./ci/${{ matrix.os_name }}_runner.sh do_install - - name: Linux Script + - name: Run test cases run: | ./ci/kubernetes-discovery-ci.sh run_case From 3bfc39515bf529435b93524223bf4f3c70660ee3 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 19:55:58 +0800 Subject: [PATCH 40/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 9 ++------- ci/kubernetes-discovery-ci.sh | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index 116d7ff67774..ce722eaea1a5 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -77,18 +77,13 @@ jobs: ./kubectl proxy -p 6445 & - - name: Linux Get dependencies - run: | - sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3 libpcre3-dev libldap2-dev - - name: Linux Before install run: | - sudo ./ci/${{ matrix.os_name }}_runner.sh before_install + sudo cpanm --notest Test::Nginx >build.log 2>&1 || (cat build.log && exit 1) - name: Linux Install run: | - sudo --preserve-env=OPENRESTY_VERSION \ - ./ci/${{ matrix.os_name }}_runner.sh do_install + sudo --preserve-env=OPENRESTY_VERSION ./ci/${{ matrix.os_name }}_runner.sh do_install - name: Run test cases run: | diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery-ci.sh index 0294c8769d59..0cb014a5e318 100755 --- a/ci/kubernetes-discovery-ci.sh +++ b/ci/kubernetes-discovery-ci.sh @@ -45,7 +45,6 @@ install_dependencies_in_centos() { run_case() { export_or_prefix - make init # run test cases prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result From 920e849dfbca2a6b72b370b0eec1d6f36be2843a Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 19:59:05 +0800 Subject: [PATCH 41/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index ce722eaea1a5..c3e1d236ddd0 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -79,6 +79,7 @@ jobs: - name: Linux Before install run: | + sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3 libpcre3-dev libldap2-dev sudo cpanm --notest Test::Nginx >build.log 2>&1 || (cat build.log && exit 1) - name: Linux Install From f0b520f3589f238fce9dd888b2cda71b22301db8 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 20:14:27 +0800 Subject: [PATCH 42/73] feat: add kubernetes discovery module --- ci/kubernetes-discovery-ci.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery-ci.sh index 0cb014a5e318..c0031a165a10 100755 --- a/ci/kubernetes-discovery-ci.sh +++ b/ci/kubernetes-discovery-ci.sh @@ -45,8 +45,7 @@ install_dependencies_in_centos() { run_case() { export_or_prefix - - # run test cases + export PERL5LIB=.:$PERL5LIB prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result rerun_flaky_tests /tmp/test.result } From 72a7c36bfda5116567f841825388ee4bd51da5c3 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 20:31:21 +0800 Subject: [PATCH 43/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 4 ++-- ci/kubernetes-discovery-ci.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index c3e1d236ddd0..c2c6becf4d49 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: true jobs: - build: + kuberbets-discovery: strategy: fail-fast: false matrix: @@ -113,7 +113,7 @@ jobs: - name: Install dependencies run: | - docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery-ci.sh install_dependencies_in_centos" + docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery-ci.sh install_dependencies" - name: Install rpm package if: ${{ startsWith(github.ref, 'refs/heads/release/') }} diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery-ci.sh index c0031a165a10..026428104666 100755 --- a/ci/kubernetes-discovery-ci.sh +++ b/ci/kubernetes-discovery-ci.sh @@ -18,7 +18,7 @@ . ./ci/common.sh -install_dependencies_in_centos() { +install_dependencies() { export_or_prefix # install development tools From 7e4876b78b2aa3c66479ab38675c34e88192e8d7 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 20:48:52 +0800 Subject: [PATCH 44/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 48 +++-------------- ci/kubernetes-discovery-ci.sh | 61 ---------------------- 2 files changed, 6 insertions(+), 103 deletions(-) delete mode 100755 ci/kubernetes-discovery-ci.sh diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index c2c6becf4d49..9dab12857d7f 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -77,52 +77,16 @@ jobs: ./kubectl proxy -p 6445 & - - name: Linux Before install + - name: Linux Install run: | sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3 libpcre3-dev libldap2-dev sudo cpanm --notest Test::Nginx >build.log 2>&1 || (cat build.log && exit 1) - - - name: Linux Install - run: | sudo --preserve-env=OPENRESTY_VERSION ./ci/${{ matrix.os_name }}_runner.sh do_install - name: Run test cases run: | - ./ci/kubernetes-discovery-ci.sh run_case - - - name: Build rpm package - if: ${{ startsWith(github.ref, 'refs/heads/release/') }} - run: | - export VERSION=${{ steps.branch_env.outputs.version }} - sudo gem install --no-document fpm - git clone -b v2.7.0 https://github.com/api7/apisix-build-tools.git - - # move codes under build tool - mkdir ./apisix-build-tools/apisix - for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done - - cd apisix-build-tools - make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix - cd .. - rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) - - - name: Run centos7 docker and mapping apisix into container - run: | - docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" --dns 8.8.8.8 -v /tmp:/tmp --dns-search apache.org docker.io/centos:7 /bin/bash - # docker exec centos7Instance bash -c "cp -r /tmp/apisix ./" - - - name: Install dependencies - run: | - docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery-ci.sh install_dependencies" - - - name: Install rpm package - if: ${{ startsWith(github.ref, 'refs/heads/release/') }} - run: | - docker exec centos7Instance bash -c "cd apisix && rpm -iv --prefix=/apisix ./apisix-build-tools/output/apisix-${{ steps.branch_env.outputs.version }}-0.el7.x86_64.rpm" - # Dependencies are attached with rpm, so revert `make deps` - docker exec centos7Instance bash -c "cd apisix && rm -rf deps" - docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." - - - name: Run test cases - run: | - docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery-ci.sh run_case" +# . ./ci/common.sh + export_or_prefix + export PERL5LIB=.:$PERL5LIB + prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result +# rerun_flaky_tests /tmp/test.result diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery-ci.sh deleted file mode 100755 index 026428104666..000000000000 --- a/ci/kubernetes-discovery-ci.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -. ./ci/common.sh - -install_dependencies() { - export_or_prefix - - # install development tools - yum install -y wget tar gcc automake autoconf libtool make unzip \ - git which sudo openldap-devel - - # curl with http2 - wget https://github.com/moparisthebest/static-curl/releases/download/v7.79.1/curl-amd64 -O /usr/bin/curl - # install openresty to make apisix's rpm test work - yum install -y yum-utils && yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo - yum install -y openresty openresty-debug openresty-openssl111-debug-devel pcre pcre-devel - - # install luarocks - ./utils/linux-install-luarocks.sh - - # install test::nginx - yum install -y cpanminus perl - cpanm --notest Test::Nginx IPC::Run > build.log 2>&1 || (cat build.log && exit 1) - - # install dependencies - git clone https://github.com/iresty/test-nginx.git test-nginx - create_lua_deps -} - -run_case() { - export_or_prefix - export PERL5LIB=.:$PERL5LIB - prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result - rerun_flaky_tests /tmp/test.result -} - -case_opt=$1 -case $case_opt in - (install_dependencies) - install_dependencies - ;; - (run_case) - run_case - ;; -esac From 9d7055faba9567ca179cee6fee44677ac6a768ba Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 20:51:24 +0800 Subject: [PATCH 45/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index 9dab12857d7f..374b4fc95a75 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -85,8 +85,7 @@ jobs: - name: Run test cases run: | -# . ./ci/common.sh export_or_prefix export PERL5LIB=.:$PERL5LIB prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result -# rerun_flaky_tests /tmp/test.result + From 647bcf17444b4652cbbd3b0536dc63497e0fd1fd Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 21:16:00 +0800 Subject: [PATCH 46/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 40 +++++++++++- ci/kubernetes-discovery-ci.sh | 75 ++++++++++++++++++++++ 2 files changed, 112 insertions(+), 3 deletions(-) create mode 100755 ci/kubernetes-discovery-ci.sh diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index 374b4fc95a75..328e8045eca4 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -85,7 +85,41 @@ jobs: - name: Run test cases run: | - export_or_prefix - export PERL5LIB=.:$PERL5LIB - prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result + ./ci/kubernetes-discovery-ci.sh run_case + - name: Build rpm package + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + run: | + export VERSION=${{ steps.branch_env.outputs.version }} + sudo gem install --no-document fpm + git clone -b v2.7.0 https://github.com/api7/apisix-build-tools.git + + # move codes under build tool + mkdir ./apisix-build-tools/apisix + for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done + + cd apisix-build-tools + make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix + cd .. + rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) + + - name: Run centos7 docker and mapping apisix into container + run: | + docker run -itd -v /home/runner/work/apisix/apisix:/apisix -v /tmp:/tmp --name centos7Instance --net="host" --dns 8.8.8.8 --dns-search apache.org docker.io/centos:7 /bin/bash + + - name: Install dependencies + run: | + docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery.sh cleanup" + docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery.sh install_dependencies" + + - name: Install rpm package + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + run: | + docker exec centos7Instance bash -c "cd apisix && rpm -iv --prefix=/apisix ./apisix-build-tools/output/apisix-${{ steps.branch_env.outputs.version }}-0.el7.x86_64.rpm" + # Dependencies are attached with rpm, so revert `make deps` + docker exec centos7Instance bash -c "cd apisix && rm -rf deps" + docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." + + - name: Run test cases + run: | + docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery.sh run_case" diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery-ci.sh new file mode 100755 index 000000000000..dcbb6ce20e2f --- /dev/null +++ b/ci/kubernetes-discovery-ci.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. ./ci/common.sh + +cleanup() { + rm -rf deps + rm -rf test-nginx +} + +install_dependencies() { + export_or_prefix + + # install development tools + yum install -y wget tar gcc automake autoconf libtool make unzip \ + git which sudo openldap-devel + + # curl with http2 + wget https://github.com/moparisthebest/static-curl/releases/download/v7.79.1/curl-amd64 -O /usr/bin/curl + # install openresty to make apisix's rpm test work + yum install -y yum-utils && yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo + yum install -y openresty openresty-debug openresty-openssl111-debug-devel pcre pcre-devel + + # install luarocks + ./utils/linux-install-luarocks.sh + + # install test::nginx + yum install -y cpanminus perl + cpanm --notest Test::Nginx IPC::Run > build.log 2>&1 || (cat build.log && exit 1) + + # unless pulled recursively, the submodule directory will remain empty. So it's better to initialize and set the submodule to the particular commit. + if [ ! "$(ls -A . )" ]; then + git submodule init + git submodule update + fi + + # install dependencies + git clone https://github.com/iresty/test-nginx.git test-nginx + create_lua_deps +} + +run_case() { + export_or_prefix + export PERL5LIB=.:$PERL5LIB + prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result + rerun_flaky_tests /tmp/test.result +} + +case_opt=$1 +case $case_opt in + (install_dependencies) + install_dependencies + ;; + (run_case) + run_case + ;; + (cleanup) + cleanup + ;; +esac From 56e32d61d90c13e6ca52f1b5c1763ca8c1853bde Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 23:21:10 +0800 Subject: [PATCH 47/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 2 + ...iscovery-ci.sh => kubernetes-discovery.sh} | 0 t/discovery/kubernetes/endpoints.yaml | 58 ++++++++ t/discovery/kubernetes/kubernetes.t | 129 +++++++++++++----- 4 files changed, 157 insertions(+), 32 deletions(-) rename ci/{kubernetes-discovery-ci.sh => kubernetes-discovery.sh} (100%) create mode 100644 t/discovery/kubernetes/endpoints.yaml diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index 328e8045eca4..6aeb4645969a 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -60,6 +60,8 @@ jobs: ./kubectl wait --for=condition=Ready nodes --all --timeout=180s ./kubectl apply -f ./t/discovery/kubernetes/account.yaml + + ./kubectl apply -f ./t/discovery/kubernetes/endpoints.yaml KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') diff --git a/ci/kubernetes-discovery-ci.sh b/ci/kubernetes-discovery.sh similarity index 100% rename from ci/kubernetes-discovery-ci.sh rename to ci/kubernetes-discovery.sh diff --git a/t/discovery/kubernetes/endpoints.yaml b/t/discovery/kubernetes/endpoints.yaml new file mode 100644 index 000000000000..885f82503b5c --- /dev/null +++ b/t/discovery/kubernetes/endpoints.yaml @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: Namespace +apiVersion: v1 +metadata: + name: ns-a +--- + +kind: Endpoints +apiVersion: v1 +metadata: + name: ep + namespace: ns-a +subsets: [ ] +--- + +kind: Namespace +apiVersion: v1 +metadata: + name: ns-b +--- + +kind: Endpoints +apiVersion: v1 +metadata: + name: ep + namespace: ns-b +subsets: [ ] +--- + +kind: Namespace +apiVersion: v1 +metadata: + name: ns-c +--- + +kind: Endpoints +apiVersion: v1 +metadata: + name: ep + namespace: ns-c +subsets: [ ] +--- diff --git a/t/discovery/kubernetes/kubernetes.t b/t/discovery/kubernetes/kubernetes.t index 541cd6b59727..7c1581412272 100644 --- a/t/discovery/kubernetes/kubernetes.t +++ b/t/discovery/kubernetes/kubernetes.t @@ -54,7 +54,7 @@ _EOC_ our $scale_ns_c = <<_EOC_; [ { - "op": "replace_endpoints", + "op": "replace_subsets", "name": "ep", "namespace": "ns-c", "subsets": [ @@ -98,7 +98,7 @@ _EOC_ my $main_config = $block->main_config // <<_EOC_; env KUBERNETES_SERVICE_HOST=127.0.0.1; -env KUBERNETES_SERVICE_PORT=6443; +env KUBERNETES_SERVICE_PORT=45853; env KUBERNETES_CLIENT_TOKEN=$::token_value; env KUBERNETES_CLIENT_TOKEN_FILE=$::token_file; _EOC_ @@ -147,26 +147,7 @@ _EOC_ ["Host"] = "127.0.0.1:6445" } - if op.op == "create_namespace" then - method = "POST" - path = "/api/v1/namespaces" - local t = { metadata = { name = op.name } } - body = core.json.encode(t, true) - headers["Content-Type"] = "application/json" - end - - if op.op == "create_endpoints" then - method = "POST" - path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints" - local t = { - metadata = { name = op.name, namespace = op.namespace }, - subsets = op.subsets, - } - body = core.json.encode(t, true) - headers["Content-Type"] = "application/json" - end - - if op.op == "replace_endpoints" then + if op.op == "replace_subsets" then method = "PATCH" path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints/" .. op.name if #op.subsets == 0 then @@ -178,6 +159,14 @@ _EOC_ headers["Content-Type"] = "application/json-patch+json" end + if op.op == "replace_labels" then + method = "PATCH" + path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints/" .. op.name + local t = { { op = "replace", path = "/metadata/labels", value = op.labels } } + body = core.json.encode(t, true) + headers["Content-Type"] = "application/json-patch+json" + end + local httpc = http.new() core.log.info("begin to connect ", "127.0.0.1:6445") local ok, message = httpc:connect({ @@ -222,11 +211,7 @@ __DATA__ POST /operators [ { - "op": "create_namespace", - "name": "ns-a" - }, - { - "op": "create_endpoints", + "op": "replace_subsets", "namespace": "ns-a", "name": "ep", "subsets": [ @@ -269,7 +254,7 @@ POST /operators "name": "ns-b" }, { - "op": "create_endpoints", + "op": "replace_subsets", "namespace": "ns-b", "name": "ep", "subsets": [ @@ -312,7 +297,7 @@ POST /operators "name": "ns-c" }, { - "op": "create_endpoints", + "op": "replace_subsets", "namespace": "ns-c", "name": "ep", "subsets": [ @@ -381,7 +366,7 @@ discovery: kubernetes: service: host: "127.0.0.1" - port: "6443" + port: "45853" client: token: "${KUBERNETES_CLIENT_TOKEN}" --- request @@ -637,7 +622,87 @@ qr{ 0 0 0 0 2 2 } -=== TEST 14: scale endpoints +=== TEST 14: use label selector +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +discovery: + kubernetes: + client: + token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} + label_selector: |- + first=1,second +--- request eval +[ + +"POST /operators +[{\"op\":\"replace_labels\",\"name\":\"ep\",\"namespace\":\"ns-a\",\"labels\":{}}]", + +"POST /operators +[{\"op\":\"replace_labels\",\"name\":\"ep\",\"namespace\":\"ns-b\",\"labels\":{}}]", + +"POST /operators +[{\"op\":\"replace_labels\",\"name\":\"ep\",\"namespace\":\"ns-c\",\"labels\":{}}]", + +"GET /queries +[\"ns-a/ep:p1\",\"ns-b/ep:p1\",\"ns-c/ep:5001\"]", + +"POST /operators +[{\"op\":\"replace_labels\",\"name\":\"ep\",\"namespace\":\"ns-a\",\"labels\":{\"first\":\"1\" }}]", + +"GET /queries +[\"ns-a/ep:p1\",\"ns-b/ep:p1\",\"ns-c/ep:5001\"]", + +"POST /operators +[{\"op\":\"replace_labels\",\"name\":\"ep\",\"namespace\":\"ns-b\",\"labels\":{\"first\":\"1\",\"second\":\"o\" }}]", + +"GET /queries +[\"ns-a/ep:p1\",\"ns-b/ep:p1\",\"ns-c/ep:5001\"]", + +"POST /operators +[{\"op\":\"replace_labels\",\"name\":\"ep\",\"namespace\":\"ns-c\",\"labels\":{\"first\":\"2\",\"second\":\"o\" }}]", + +"GET /queries +[\"ns-a/ep:p1\",\"ns-b/ep:p1\",\"ns-c/ep:5001\"]", + +"POST /operators +[{\"op\":\"replace_labels\",\"name\":\"ep\",\"namespace\":\"ns-c\",\"labels\":{\"first\":\"1\" }}]", + +"GET /queries +[\"ns-a/ep:p1\",\"ns-b/ep:p1\",\"ns-c/ep:5001\"]", + +"POST /operators +[{\"op\":\"replace_labels\",\"name\":\"ep\",\"namespace\":\"ns-c\",\"labels\":{\"first\":\"1\",\"second\":\"o\" }}]", + +"GET /queries +[\"ns-a/ep:p1\",\"ns-b/ep:p1\",\"ns-c/ep:5001\"]", + +] +--- response_body eval +[ + "DONE\n", + "DONE\n", + "DONE\n", + "{ 0 0 0 }\n", + "DONE\n", + "{ 0 0 0 }\n", + "DONE\n", + "{ 0 2 0 }\n", + "DONE\n", + "{ 0 2 0 }\n", + "DONE\n", + "{ 0 2 0 }\n", + "DONE\n", + "{ 0 2 2 }\n", +] +--- no_error_log +[error] + + + +=== TEST 15: scale endpoints --- yaml_config eval: $::yaml_config --- request eval [ @@ -645,7 +710,7 @@ qr{ 0 0 0 0 2 2 } [\"ns-a/ep:p1\",\"ns-a/ep:p2\"]", "POST /operators -[{\"op\":\"replace_endpoints\",\"name\":\"ep\",\"namespace\":\"ns-a\",\"subsets\":[]}]", +[{\"op\":\"replace_subsets\",\"name\":\"ep\",\"namespace\":\"ns-a\",\"subsets\":[]}]", "GET /queries [\"ns-a/ep:p1\",\"ns-a/ep:p2\"]", From 51857a13526122450ccdad201298fb381da1596c Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 23:30:53 +0800 Subject: [PATCH 48/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 2 +- t/discovery/kubernetes/kubernetes.t | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index 6aeb4645969a..ae0c792314cc 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -87,7 +87,7 @@ jobs: - name: Run test cases run: | - ./ci/kubernetes-discovery-ci.sh run_case + ./ci/kubernetes-discovery.sh run_case - name: Build rpm package if: ${{ startsWith(github.ref, 'refs/heads/release/') }} diff --git a/t/discovery/kubernetes/kubernetes.t b/t/discovery/kubernetes/kubernetes.t index 7c1581412272..f1c6a0280884 100644 --- a/t/discovery/kubernetes/kubernetes.t +++ b/t/discovery/kubernetes/kubernetes.t @@ -98,7 +98,7 @@ _EOC_ my $main_config = $block->main_config // <<_EOC_; env KUBERNETES_SERVICE_HOST=127.0.0.1; -env KUBERNETES_SERVICE_PORT=45853; +env KUBERNETES_SERVICE_PORT=6443; env KUBERNETES_CLIENT_TOKEN=$::token_value; env KUBERNETES_CLIENT_TOKEN_FILE=$::token_file; _EOC_ @@ -366,7 +366,7 @@ discovery: kubernetes: service: host: "127.0.0.1" - port: "45853" + port: "6443" client: token: "${KUBERNETES_CLIENT_TOKEN}" --- request From 0b9a708e0956c144749d528a70a068edb61de722 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Tue, 1 Feb 2022 23:45:14 +0800 Subject: [PATCH 49/73] feat: add kubernetes discovery module --- ci/kubernetes-discovery.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/kubernetes-discovery.sh b/ci/kubernetes-discovery.sh index dcbb6ce20e2f..440f36ce3bcc 100755 --- a/ci/kubernetes-discovery.sh +++ b/ci/kubernetes-discovery.sh @@ -19,6 +19,7 @@ . ./ci/common.sh cleanup() { + rm -rf test-result rm -rf deps rm -rf test-nginx } @@ -57,8 +58,8 @@ install_dependencies() { run_case() { export_or_prefix export PERL5LIB=.:$PERL5LIB - prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee /tmp/test.result - rerun_flaky_tests /tmp/test.result + prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee test-result + rerun_flaky_tests test-result } case_opt=$1 From c4791a423543a062be7662d5d840cd44d1e2d4a2 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 11:19:05 +0800 Subject: [PATCH 50/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-discovery.yml | 10 +-- apisix/kubernetes.lua | 4 +- docs/zh/latest/discovery/kubernetes.md | 75 +++++++++++----------- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-discovery.yml index ae0c792314cc..fc531e016135 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-discovery.yml @@ -1,4 +1,4 @@ -name: Kuberbetes Discovery +name: Kubernetes Discovery on: push: @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: true jobs: - kuberbets-discovery: + kubernetes-discovery: strategy: fail-fast: false matrix: @@ -60,7 +60,7 @@ jobs: ./kubectl wait --for=condition=Ready nodes --all --timeout=180s ./kubectl apply -f ./t/discovery/kubernetes/account.yaml - + ./kubectl apply -f ./t/discovery/kubernetes/endpoints.yaml KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') @@ -95,11 +95,11 @@ jobs: export VERSION=${{ steps.branch_env.outputs.version }} sudo gem install --no-document fpm git clone -b v2.7.0 https://github.com/api7/apisix-build-tools.git - + # move codes under build tool mkdir ./apisix-build-tools/apisix for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done - + cd apisix-build-tools make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix cd .. diff --git a/apisix/kubernetes.lua b/apisix/kubernetes.lua index e6166da9e2f3..dda915b54ce3 100644 --- a/apisix/kubernetes.lua +++ b/apisix/kubernetes.lua @@ -176,8 +176,8 @@ local function watch(httpc, informer) return true, _constants.Success, nil end - local watch_times = 5 - for _ = 0, watch_times do + local watch_times = 8 + for _ = 1, watch_times do local watch_seconds = 1800 + math.random(9, 999) informer.overtime = watch_seconds local http_seconds = watch_seconds + 120 diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md index 0aec2d992ece..e45d1a0e9566 100644 --- a/docs/zh/latest/discovery/kubernetes.md +++ b/docs/zh/latest/discovery/kubernetes.md @@ -19,8 +19,8 @@ # 基于 Kubernetes 的服务发现 -Kubernetes 服务发现插件以 ListWatch 方式监听 Kubernetes 集群的 的 endpoints/v1 资源的实时变化, 并将其值存储在 shared.DICT 中, -以及提供对外查询接口 +Kubernetes 服务发现插件以 ListWatch 方式监听 Kubernetes 集群的 的 v1.endpoints 的实时变化, +并将其值存储在 ngx.shared.DICT 中, 同时遵循 APISix Discovery 规范提供对外查询接口 # Kubernetes 服务发现插件的配置 @@ -30,31 +30,32 @@ Kubernetes 服务发现的样例配置如下: discovery: kubernetes: service: - #kubernetes apiserver schema , option [ http | https ], default https - schema: https + # kubernetes apiserver schema, options [ http | https ] + schema: https #default https - # kubernetes apiserver host, you can set ipv4,ipv6 ,domain or environment variable - # default ${KUBERNETES_SERVICE_HOST} - host: 10.0.8.95 + # kubernetes apiserver host, options [ ipv4 | ipv6 | domain | env variable] + host: 10.0.8.95 #default ${KUBERNETES_SERVICE_HOST} - # kubernetes apiserver port, you can set number or environment variable - # default ${KUBERNETES_SERVICE_PORT} - port: 6443 + # kubernetes apiserver port, you can enter port number or environment variable + port: 6443 #default ${KUBERNETES_SERVICE_PORT} client: # kubernetes serviceaccount token or token_file - # default setup token_file and value is "/var/run/secrets/kubernetes.io/serviceaccount/token" - #token: token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" + #token: - # kubernetes discovery plugin support watch endpoints in specific namespace - # you can use [ equal | not_equal | match | not_match ] to filter your specified namespace - # [ match | not_match ] support regular expression (using ngx.re.match ) + # kubernetes discovery plugin support use namespace_selector + # you can use one of [ equal | not_equal | match | not_match ] filter namespace namespace_selector: equal: default -# not_equal: -# match: -# not_match: + #not_equal: + #match: + #not_match: + + # kubernetes discovery plugin support use label_selector + # for the expression of label_selector, please refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + label_selector: |- + first="a",second="b" ``` 如果 Kubernetes 服务插件运行在 Pod 内, 你可以使用最简配置: @@ -70,36 +71,36 @@ discovery: discovery: kubernetes: service: - host: [ ApiServer Host Value Here ] - port: [ ApiServer Port Value Here ] + host: # Enter ApiServer Host Value Here + port: # Enter ApiServer Port Value Here schema: https client: - token: [ ServiceAccount Token Value Here ] + token: # Enter ServiceAccount Token Value Here + #token_file: # Enter File Path Here ``` # Kubernetes 服务发现插件的使用 -Kubernetes 服务发现插件提供与其他服务发现相同的查询接口 -> nodes(service_name) - -其中 service_name 的 pattern 如下: +Kubernetes 服务发现插件提供与其他服务发现插件相同的查询接口 -> nodes(service_name) \ +service_name 的 pattern 如下: > _[namespace]/[name]:[portName]_ -某些 endpoints 可能没有定义 portName, Kubernetes 服务发现插件会依次使用 targetPort, port 代替 +如果 kubernetes Endpoint 没有定义 portName, Kubernetes 服务发现插件会依次使用 targetPort, port 代替 # Q&A -> Q: 为什么只支持配置 token 来访问 kubernetes apiserver ,可以使用 kubernetes config 吗 +> Q: 为什么只支持配置 token 来访问 kubernetes apiserver \ +> A: 通常情况下,我们会使用三种方式与 kubernetes apiserver 通信 : +> +>+ mTLS +>+ token +>+ basic authentication +> +> 因为 lua-resty-http 目前不支持 mTLS ,以及 basic authentication 不被推荐使用,\ +> 所以当前只实现了 token 认证方式 -> A: 通常情况下,我们会使用三种方式与 kubernetes Apiserver 通信 : -> + mTLS -> + token -> + basic authentication \ -> 但因为 apisix.http 目前并没有支持 mTLS ,以及 basic authentication 并不被推荐使用,\ -> 所以只实现了 token 的方式, 这也意味着不支持 kubernetes config ------- -> Q: APISix 是多进程模型, 是否意味着每个 APISix 业务进程都会去 监听 kubernetes apiserver - -> A: Kubernetes 服务发现插件只使用特权进程监听 kubernetes 集群,然后将获取到结果存储在 ngx.shared.DICT中 业务进程是通过查询 ngx.shared.DICT 获取结果的 - ------- +> Q: APISix 是多进程模型, 是否意味着每个 APISix 业务进程都会去监听 kubernetes apiserver \ +> A: Kubernetes 服务发现插件只使用特权进程监听 kubernetes 集群,然后将结果存储在 ngx.shared.DICT中, \ +> 业务进程是通过查询 ngx.shared.DICT 获取结果的 From 3e3bb9706a1a36ba04289ceda98c2dd75158b915 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 12:07:35 +0800 Subject: [PATCH 51/73] feat: add kubernetes discovery module --- t/discovery/kubernetes/kubernetes.t | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/t/discovery/kubernetes/kubernetes.t b/t/discovery/kubernetes/kubernetes.t index f1c6a0280884..14a17cfa4ffc 100644 --- a/t/discovery/kubernetes/kubernetes.t +++ b/t/discovery/kubernetes/kubernetes.t @@ -78,7 +78,15 @@ _EOC_ } -use t::APISIX 'no_plan'; +use t::APISIX; + +my $out = eval { `resty -e "local s=ngx.socket.tcp();print(s:connect(\"127.0.0.1\",6443))"` }; + +if ($out !~ m/function:/) { + plan(skip_all => "kubernetes not patched"); +} else { + plan('no_plan'); +} repeat_each(1); log_level('debug'); From d68ceb3c322938fbd6ed30548f88109f4b51aba4 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 12:08:58 +0800 Subject: [PATCH 52/73] feat: add kubernetes discovery module --- t/discovery/kubernetes/kubernetes.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/discovery/kubernetes/kubernetes.t b/t/discovery/kubernetes/kubernetes.t index 14a17cfa4ffc..b6979a51c5a4 100644 --- a/t/discovery/kubernetes/kubernetes.t +++ b/t/discovery/kubernetes/kubernetes.t @@ -82,7 +82,7 @@ use t::APISIX; my $out = eval { `resty -e "local s=ngx.socket.tcp();print(s:connect(\"127.0.0.1\",6443))"` }; -if ($out !~ m/function:/) { +if ($out !~ 1 ) { plan(skip_all => "kubernetes not patched"); } else { plan('no_plan'); From 5ba1f75e6be7365a8937fbdbc90c350d4063dcd9 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 12:27:48 +0800 Subject: [PATCH 53/73] feat: add kubernetes discovery module --- ci/kubernetes-discovery.sh | 2 ++ t/discovery/kubernetes/kubernetes.t | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/kubernetes-discovery.sh b/ci/kubernetes-discovery.sh index 440f36ce3bcc..398845a4d395 100755 --- a/ci/kubernetes-discovery.sh +++ b/ci/kubernetes-discovery.sh @@ -19,6 +19,7 @@ . ./ci/common.sh cleanup() { + rm -rf kubernetes_discovery_ci rm -rf test-result rm -rf deps rm -rf test-nginx @@ -58,6 +59,7 @@ install_dependencies() { run_case() { export_or_prefix export PERL5LIB=.:$PERL5LIB + echo "true" > kubernetes_discovery_ci prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee test-result rerun_flaky_tests test-result } diff --git a/t/discovery/kubernetes/kubernetes.t b/t/discovery/kubernetes/kubernetes.t index b6979a51c5a4..cc568b66276d 100644 --- a/t/discovery/kubernetes/kubernetes.t +++ b/t/discovery/kubernetes/kubernetes.t @@ -80,12 +80,11 @@ _EOC_ use t::APISIX; -my $out = eval { `resty -e "local s=ngx.socket.tcp();print(s:connect(\"127.0.0.1\",6443))"` }; - -if ($out !~ 1 ) { - plan(skip_all => "kubernetes not patched"); -} else { +my $kubernetes_discovery_ci = eval {`cat kubernetes_discovery_ci 2>/dev/null`}; +if ($kubernetes_discovery_ci) { plan('no_plan'); +} else { + plan(skip_all => "kubernetes not patched"); } repeat_each(1); From eec2cfe0f76835e3dda61e58c8528760e325d510 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 13:07:49 +0800 Subject: [PATCH 54/73] feat: add kubernetes discovery module --- t/discovery/kubernetes/kubernetes.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/discovery/kubernetes/kubernetes.t b/t/discovery/kubernetes/kubernetes.t index cc568b66276d..9b39c731b065 100644 --- a/t/discovery/kubernetes/kubernetes.t +++ b/t/discovery/kubernetes/kubernetes.t @@ -84,7 +84,7 @@ my $kubernetes_discovery_ci = eval {`cat kubernetes_discovery_ci 2>/dev/null`}; if ($kubernetes_discovery_ci) { plan('no_plan'); } else { - plan(skip_all => "kubernetes not patched"); + plan(skip_all => "not drive by kubernetes-discovery workflow"); } repeat_each(1); From f74a45a6e034d2bad24d31765c891411b0a1ee1c Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 21:07:59 +0800 Subject: [PATCH 55/73] feat: add kubernetes discovery module --- ...rnetes-discovery.yml => kubernetes-ci.yml} | 45 ++---------------- Makefile | 1 + .../{ => discovery/kubernetes}/kubernetes.lua | 0 ci/kubernetes-discovery.sh | 47 +------------------ docs/zh/latest/config.json | 3 +- .../configs}/account.yaml | 0 .../configs/endpoint.yaml | 0 .../configs}/kind.yaml | 0 .../discovery}/kubernetes.t | 9 +--- 9 files changed, 9 insertions(+), 96 deletions(-) rename .github/workflows/{kubernetes-discovery.yml => kubernetes-ci.yml} (54%) rename apisix/{ => discovery/kubernetes}/kubernetes.lua (100%) rename {t/discovery/kubernetes => t_kubernetes/configs}/account.yaml (100%) rename t/discovery/kubernetes/endpoints.yaml => t_kubernetes/configs/endpoint.yaml (100%) rename {t/discovery/kubernetes => t_kubernetes/configs}/kind.yaml (100%) rename {t/discovery/kubernetes => t_kubernetes/discovery}/kubernetes.t (98%) diff --git a/.github/workflows/kubernetes-discovery.yml b/.github/workflows/kubernetes-ci.yml similarity index 54% rename from .github/workflows/kubernetes-discovery.yml rename to .github/workflows/kubernetes-ci.yml index fc531e016135..c6593c57d0c6 100644 --- a/.github/workflows/kubernetes-discovery.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -1,4 +1,4 @@ -name: Kubernetes Discovery +name: CI Kubernetes on: push: @@ -55,13 +55,13 @@ jobs: chmod +x ./kind chmod +x ./kubectl - ./kind create cluster --name apisix-test --config ./t/discovery/kubernetes/kind.yaml + ./kind create cluster --name apisix-test --config ./t_kubernetes/configs/kind.yaml ./kubectl wait --for=condition=Ready nodes --all --timeout=180s - ./kubectl apply -f ./t/discovery/kubernetes/account.yaml + ./kubectl apply -f ./t_kubernetes/configs/account.yaml - ./kubectl apply -f ./t/discovery/kubernetes/endpoints.yaml + ./kubectl apply -f ./t_kubernetes/configs/endpoints.yaml KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') @@ -88,40 +88,3 @@ jobs: - name: Run test cases run: | ./ci/kubernetes-discovery.sh run_case - - - name: Build rpm package - if: ${{ startsWith(github.ref, 'refs/heads/release/') }} - run: | - export VERSION=${{ steps.branch_env.outputs.version }} - sudo gem install --no-document fpm - git clone -b v2.7.0 https://github.com/api7/apisix-build-tools.git - - # move codes under build tool - mkdir ./apisix-build-tools/apisix - for dir in `ls|grep -v "^apisix-build-tools$"`;do cp -r $dir ./apisix-build-tools/apisix/;done - - cd apisix-build-tools - make package type=rpm app=apisix version=${VERSION} checkout=release/${VERSION} image_base=centos image_tag=7 local_code_path=./apisix - cd .. - rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec) - - - name: Run centos7 docker and mapping apisix into container - run: | - docker run -itd -v /home/runner/work/apisix/apisix:/apisix -v /tmp:/tmp --name centos7Instance --net="host" --dns 8.8.8.8 --dns-search apache.org docker.io/centos:7 /bin/bash - - - name: Install dependencies - run: | - docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery.sh cleanup" - docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery.sh install_dependencies" - - - name: Install rpm package - if: ${{ startsWith(github.ref, 'refs/heads/release/') }} - run: | - docker exec centos7Instance bash -c "cd apisix && rpm -iv --prefix=/apisix ./apisix-build-tools/output/apisix-${{ steps.branch_env.outputs.version }}-0.el7.x86_64.rpm" - # Dependencies are attached with rpm, so revert `make deps` - docker exec centos7Instance bash -c "cd apisix && rm -rf deps" - docker exec centos7Instance bash -c "cd apisix && mv usr/bin . && mv usr/local/apisix/* ." - - - name: Run test cases - run: | - docker exec centos7Instance bash -c "cd apisix && ./ci/kubernetes-discovery.sh run_case" diff --git a/Makefile b/Makefile index 6e26c5360b43..219007b5a15a 100644 --- a/Makefile +++ b/Makefile @@ -280,6 +280,7 @@ install: runtime $(ENV_INSTALL) apisix/discovery/dns/*.lua $(ENV_INST_LUADIR)/apisix/discovery/dns $(ENV_INSTALL) apisix/discovery/eureka/*.lua $(ENV_INST_LUADIR)/apisix/discovery/eureka $(ENV_INSTALL) apisix/discovery/nacos/*.lua $(ENV_INST_LUADIR)/apisix/discovery/nacos + $(ENV_INSTALL) apisix/discovery/nacos/*.lua $(ENV_INST_LUADIR)/apisix/discovery/kubernetes $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/http $(ENV_INSTALL) apisix/http/*.lua $(ENV_INST_LUADIR)/apisix/http/ diff --git a/apisix/kubernetes.lua b/apisix/discovery/kubernetes/kubernetes.lua similarity index 100% rename from apisix/kubernetes.lua rename to apisix/discovery/kubernetes/kubernetes.lua diff --git a/ci/kubernetes-discovery.sh b/ci/kubernetes-discovery.sh index 398845a4d395..839929ba68f9 100755 --- a/ci/kubernetes-discovery.sh +++ b/ci/kubernetes-discovery.sh @@ -18,61 +18,16 @@ . ./ci/common.sh -cleanup() { - rm -rf kubernetes_discovery_ci - rm -rf test-result - rm -rf deps - rm -rf test-nginx -} - -install_dependencies() { - export_or_prefix - - # install development tools - yum install -y wget tar gcc automake autoconf libtool make unzip \ - git which sudo openldap-devel - - # curl with http2 - wget https://github.com/moparisthebest/static-curl/releases/download/v7.79.1/curl-amd64 -O /usr/bin/curl - # install openresty to make apisix's rpm test work - yum install -y yum-utils && yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo - yum install -y openresty openresty-debug openresty-openssl111-debug-devel pcre pcre-devel - - # install luarocks - ./utils/linux-install-luarocks.sh - - # install test::nginx - yum install -y cpanminus perl - cpanm --notest Test::Nginx IPC::Run > build.log 2>&1 || (cat build.log && exit 1) - - # unless pulled recursively, the submodule directory will remain empty. So it's better to initialize and set the submodule to the particular commit. - if [ ! "$(ls -A . )" ]; then - git submodule init - git submodule update - fi - - # install dependencies - git clone https://github.com/iresty/test-nginx.git test-nginx - create_lua_deps -} - run_case() { export_or_prefix export PERL5LIB=.:$PERL5LIB - echo "true" > kubernetes_discovery_ci - prove -Itest-nginx/lib -r t/discovery/kubernetes/kubernetes.t | tee test-result + prove -Itest-nginx/lib -I./ -r t_kubernetes | tee test-result rerun_flaky_tests test-result } case_opt=$1 case $case_opt in - (install_dependencies) - install_dependencies - ;; (run_case) run_case ;; - (cleanup) - cleanup - ;; esac diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json index 22026bedaad2..180a71f25d7a 100644 --- a/docs/zh/latest/config.json +++ b/docs/zh/latest/config.json @@ -174,7 +174,8 @@ "discovery", "discovery/dns", "discovery/nacos", - "discovery/eureka" + "discovery/eureka", + "discovery/kubernetes" ] }, { diff --git a/t/discovery/kubernetes/account.yaml b/t_kubernetes/configs/account.yaml similarity index 100% rename from t/discovery/kubernetes/account.yaml rename to t_kubernetes/configs/account.yaml diff --git a/t/discovery/kubernetes/endpoints.yaml b/t_kubernetes/configs/endpoint.yaml similarity index 100% rename from t/discovery/kubernetes/endpoints.yaml rename to t_kubernetes/configs/endpoint.yaml diff --git a/t/discovery/kubernetes/kind.yaml b/t_kubernetes/configs/kind.yaml similarity index 100% rename from t/discovery/kubernetes/kind.yaml rename to t_kubernetes/configs/kind.yaml diff --git a/t/discovery/kubernetes/kubernetes.t b/t_kubernetes/discovery/kubernetes.t similarity index 98% rename from t/discovery/kubernetes/kubernetes.t rename to t_kubernetes/discovery/kubernetes.t index 9b39c731b065..f1c6a0280884 100644 --- a/t/discovery/kubernetes/kubernetes.t +++ b/t_kubernetes/discovery/kubernetes.t @@ -78,14 +78,7 @@ _EOC_ } -use t::APISIX; - -my $kubernetes_discovery_ci = eval {`cat kubernetes_discovery_ci 2>/dev/null`}; -if ($kubernetes_discovery_ci) { - plan('no_plan'); -} else { - plan(skip_all => "not drive by kubernetes-discovery workflow"); -} +use t::APISIX 'no_plan'; repeat_each(1); log_level('debug'); From b3bed5f614cfe7ea608122b7d0ab48dc89b749e7 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 21:08:07 +0800 Subject: [PATCH 56/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/init.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index ca9caf79711c..a6a2a28d6402 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -27,7 +27,7 @@ local process = require("ngx.process") local core = require("apisix.core") local util = require("apisix.cli.util") local local_conf = require("apisix.core.config_local").local_conf() -local kubernetes = require("apisix.kubernetes") +local kubernetes = require("apisix.discovery.kubernetes.kubernetes") local endpoint_dict = ngx.shared.discovery local default_weight = 0 From fe030ba5ec1394d550ba17b7ef18ac9d840d6b3b Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 21:09:02 +0800 Subject: [PATCH 57/73] feat: add kubernetes discovery module --- ci/{kubernetes-discovery.sh => kubernetes-ci.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ci/{kubernetes-discovery.sh => kubernetes-ci.sh} (100%) diff --git a/ci/kubernetes-discovery.sh b/ci/kubernetes-ci.sh similarity index 100% rename from ci/kubernetes-discovery.sh rename to ci/kubernetes-ci.sh From 6552848b9ee5ff17545ee4060c6e63992e6d3932 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 21:10:40 +0800 Subject: [PATCH 58/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index c6593c57d0c6..083b6c286a01 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -87,4 +87,4 @@ jobs: - name: Run test cases run: | - ./ci/kubernetes-discovery.sh run_case + ./ci/kubernetes-ci.sh run_case From 06442ac5275c69164f437eaa091a12c15206e1e1 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 21:14:04 +0800 Subject: [PATCH 59/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index 083b6c286a01..bf5e3a6860d8 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -61,7 +61,7 @@ jobs: ./kubectl apply -f ./t_kubernetes/configs/account.yaml - ./kubectl apply -f ./t_kubernetes/configs/endpoints.yaml + ./kubectl apply -f ./t_kubernetes/configs/endpoint.yaml KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}') From 7b4523cc1b4d7945f0d062072cd31f9005683050 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 21:16:34 +0800 Subject: [PATCH 60/73] feat: add kubernetes discovery module --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 219007b5a15a..6be616214aab 100644 --- a/Makefile +++ b/Makefile @@ -280,7 +280,7 @@ install: runtime $(ENV_INSTALL) apisix/discovery/dns/*.lua $(ENV_INST_LUADIR)/apisix/discovery/dns $(ENV_INSTALL) apisix/discovery/eureka/*.lua $(ENV_INST_LUADIR)/apisix/discovery/eureka $(ENV_INSTALL) apisix/discovery/nacos/*.lua $(ENV_INST_LUADIR)/apisix/discovery/nacos - $(ENV_INSTALL) apisix/discovery/nacos/*.lua $(ENV_INST_LUADIR)/apisix/discovery/kubernetes + $(ENV_INSTALL) apisix/discovery/kubernetes/*.lua $(ENV_INST_LUADIR)/apisix/discovery/kubernetes $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/http $(ENV_INSTALL) apisix/http/*.lua $(ENV_INST_LUADIR)/apisix/http/ From 329782a1c895717da50c0b8c78802bbcf9f78039 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 2 Feb 2022 21:23:51 +0800 Subject: [PATCH 61/73] feat: add kubernetes discovery module --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6be616214aab..c461cc7074b0 100644 --- a/Makefile +++ b/Makefile @@ -275,7 +275,7 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery $(ENV_INSTALL) apisix/discovery/*.lua $(ENV_INST_LUADIR)/apisix/discovery/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul_kv,dns,eureka,nacos} + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul_kv,dns,eureka,nacos,kubernetes} $(ENV_INSTALL) apisix/discovery/consul_kv/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul_kv $(ENV_INSTALL) apisix/discovery/dns/*.lua $(ENV_INST_LUADIR)/apisix/discovery/dns $(ENV_INSTALL) apisix/discovery/eureka/*.lua $(ENV_INST_LUADIR)/apisix/discovery/eureka From ab81437ad0fb0cf0a6596aaf5233c7733dbf6833 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Sat, 5 Feb 2022 10:19:18 +0800 Subject: [PATCH 62/73] feat: add kubernetes discovery module --- .../discovery/kubernetes/informer_factory.lua | 342 ++++++++++++++++ apisix/discovery/kubernetes/init.lua | 140 ++++--- apisix/discovery/kubernetes/kubernetes.lua | 364 ------------------ t_kubernetes/discovery/kubernetes.t | 26 +- 4 files changed, 421 insertions(+), 451 deletions(-) create mode 100644 apisix/discovery/kubernetes/informer_factory.lua delete mode 100644 apisix/discovery/kubernetes/kubernetes.lua diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua new file mode 100644 index 000000000000..ecbce4ed1ed9 --- /dev/null +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -0,0 +1,342 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local ngx = ngx +local ipairs = ipairs +local string = string +local math = math +local setmetatable = setmetatable +local core = require("apisix.core") +local http = require("resty.http") + +local empty_table = {} + +local function list_query(informer) + local arguments = { + limit = informer.limit, + } + + if informer.continue ~= nil and informer.continue ~= "" then + arguments.continue = informer.continue + end + + if informer.label_selector and informer.label_selector ~= "" then + arguments.labelSelector = informer.label_selector + end + + if informer.field_selector and informer.field_selector ~= "" then + arguments.fieldSelector = informer.field_selector + end + + return ngx.encode_args(arguments) +end + +local function list(httpc, apiserver, informer) + local response, err = httpc:request({ + path = informer.path, + query = list_query(informer), + headers = { + ["Host"] = apiserver.host .. ":" .. apiserver.port, + ["Authorization"] = "Bearer " .. apiserver.token, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + }) + + core.log.info("--raw=", informer.path, "?", list_query(informer)) + + if not response then + return false, "RequestError", err or "" + end + + if response.status ~= 200 then + return false, response.reason, response:read_body() or "" + end + + local body, err = response:read_body() + if err then + return false, "ReadBodyError", err + end + + local data, _ = core.json.decode(body) + if not data or data.kind ~= informer.list_kind then + return false, "UnexpectedBody", body + end + + informer.version = data.metadata.resourceVersion + + if informer.on_added ~= nil then + for _, item in ipairs(data.items or empty_table) do + informer:on_added(item, "list") + end + end + + informer.continue = data.metadata.continue + if informer.continue ~= nil and informer.continue ~= "" then + list(httpc, informer) + end + + return true, "Success", "" +end + +local function watch_query(informer) + local arguments = { + watch = "true", + allowWatchBookmarks = "true", + timeoutSeconds = informer.overtime, + } + + if informer.version ~= nil and informer.version ~= "" then + arguments.resourceVersion = informer.version + end + + if informer.label_selector and informer.label_selector ~= "" then + arguments.labelSelector = informer.label_selector + end + + if informer.field_selector and informer.field_selector ~= "" then + arguments.fieldSelector = informer.field_selector + end + + return ngx.encode_args(arguments) +end + +local function split_event (body, dispatch_event) + local gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") + if not gmatch_iterator then + return nil, "GmatchError", err + end + + local captures + local captured_size = 0 + local ok, reason + while true do + captures, err = gmatch_iterator() + + if err then + return nil, "GmatchError", err + end + + if not captures then + break + end + + captured_size = captured_size + #captures[0] + + ok, reason, err = dispatch_event(captures[0]) + if not ok then + return nil, reason, err + end + end + + local remainder_body + if captured_size == #body then + remainder_body = "" + elseif captured_size == 0 then + remainder_body = body + elseif captured_size < #body then + remainder_body = string.sub(body, captured_size + 1) + end + + return remainder_body, "Success", nil +end + +local function watch(httpc, apiserver, informer) + + local dispatch_event = function(event_string) + local event, _ = core.json.decode(event_string) + + if not event or not event.type or not event.object then + return false, "UnexpectedBody", event_string + end + + local type = event.type + + if type == "ERROR" then + if event.object.code == 410 then + return false, "ResourceGone", nil + end + return false, "UnexpectedBody", event_string + end + + local object = event.object + informer.version = object.metadata.resourceVersion + + if type == "ADDED" then + if informer.on_added ~= nil then + informer:on_added(object, "watch") + end + elseif type == "DELETED" then + if informer.on_deleted ~= nil then + informer:on_deleted(object) + end + elseif type == "MODIFIED" then + if informer.on_modified ~= nil then + informer:on_modified(object) + end + -- elseif type == "BOOKMARK" then + -- do nothing + end + return true, "Success", nil + end + + local watch_times = 8 + for _ = 1, watch_times do + local watch_seconds = 1800 + math.random(9, 999) + informer.overtime = watch_seconds + local http_seconds = watch_seconds + 120 + httpc:set_timeouts(2000, 3000, http_seconds * 1000) + + local response, err = httpc:request({ + path = informer.path, + query = watch_query(informer), + headers = { + ["Host"] = apiserver.host .. ":" .. apiserver.port, + ["Authorization"] = "Bearer " .. apiserver.token, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + }) + + core.log.info("--raw=", informer.path, "?", watch_query(informer)) + + if err then + return false, "RequestError", err + end + + if response.status ~= 200 then + return false, response.reason, response:read_body() or "" + end + + local remainder_body + local body + local reason + + while true do + body, err = response.body_reader() + if err then + return false, "ReadBodyError", err + end + + if not body then + break + end + + if remainder_body ~= nil and #remainder_body > 0 then + body = remainder_body .. body + end + + remainder_body, reason, err = split_event(body, dispatch_event) + if reason ~= "Success" then + if reason == "ResourceGone" then + return true, "Success", nil + end + return false, reason, err + end + end + end + return true, "Success", nil +end + +local function list_watch(self, apiserver) + local ok + local reason, message + local httpc = http.new() + + self.fetch_state = "connecting" + core.log.info("begin to connect ", apiserver.host, ":", apiserver.port) + + ok, message = httpc:connect({ + scheme = apiserver.schema, + host = apiserver.host, + port = apiserver.port, + ssl_verify = false + }) + + if not ok then + self.fetch_state = "connecting" + core.log.error("connect apiserver failed , apiserver.host: ", apiserver.host, + ", apiserver.port: ", apiserver.port, ", message : ", message) + return false + end + + core.log.info("begin to list ", self.kind) + self.fetch_state = "listing" + if self.pre_List ~= nil then + self:pre_list() + end + + ok, reason, message = list(httpc, apiserver, self) + if not ok then + self.fetch_state = "list failed" + core.log.error("list failed, kind: ", self.kind, + ", reason: ", reason, ", message : ", message) + return false + end + + self.fetch_state = "list finished" + if self.post_List ~= nil then + self:post_list() + end + + core.log.info("begin to watch ", self.kind) + self.fetch_state = "watching" + ok, reason, message = watch(httpc, apiserver, self) + if not ok then + self.fetch_state = "watch failed" + core.log.error("watch failed, kind: ", self.kind, + ", reason: ", reason, ", message : ", message) + return false + end + + self.fetch_state = "watch finished" + return true +end + +local _M = { +} + +function _M.new(group, version, kind, plural, namespace) + local path = "" + if group == "" then + path = path .. "/api/" .. version + else + path = path .. "/apis/" .. group .. "/" .. version + end + + if namespace ~= "" then + path = path .. "/namespace/" .. namespace + end + path = path .. "/" .. plural + + return setmetatable({}, { __index = { + kind = kind, + list_kind = kind .. "List", + plural = plural, + path = path, + limit = 120, + label_selector = "", + field_selector = "", + overtime = "1800", + version = "", + continue = "", + list_watch = list_watch + } + }) +end + +return _M diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index a6a2a28d6402..33233ecf16db 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -23,11 +23,12 @@ local tonumber = tonumber local tostring = tostring local os = os local error = error +local pcall = pcall local process = require("ngx.process") local core = require("apisix.core") local util = require("apisix.cli.util") local local_conf = require("apisix.core.config_local").local_conf() -local kubernetes = require("apisix.discovery.kubernetes.kubernetes") +local informer_factory = require("apisix.discovery.kubernetes.informer_factory") local endpoint_dict = ngx.shared.discovery local default_weight = 0 @@ -47,8 +48,13 @@ local function sort_nodes_cmp(left, right) return left.port < right.port end -local function on_endpoint_modified(endpoint) - core.log.debug(core.json.encode(endpoint, true)) +local function on_endpoint_modified(informer, endpoint) + if informer.namespace_selector ~= nil and + not informer:namespace_selector(endpoint.metadata.namespace) then + return + end + + core.log.debug(core.json.delay_encode(endpoint)) core.table.clear(endpoint_buffer) local subsets = endpoint.subsets @@ -105,32 +111,50 @@ local function on_endpoint_modified(endpoint) end end -local function on_endpoint_deleted(endpoint) - core.log.debug(core.json.encode(endpoint, true)) +local function on_endpoint_deleted(informer, endpoint) + if informer.namespace_selector ~= nil and + not informer:namespace_selector(endpoint.metadata.namespace) then + return + end + + core.log.debug(core.json.delay_encode(endpoint)) local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name endpoint_dict:delete(endpoint_key .. "#version") endpoint_dict:delete(endpoint_key) end +local function pre_list(informer) + endpoint_dict:flush_all() +end + +local function post_list(informer) + endpoint_dict:flush_expired() +end + local function setup_label_selector(conf, informer) - local ls = conf.label_selector - if ls == nil or ls == "" then - return - end - informer.label_selector = ngx.escape_uri(ls) + informer.label_selector = conf.label_selector end local function setup_namespace_selector(conf, informer) local ns = conf.namespace_selector if ns == nil then informer.namespace_selector = nil - elseif ns.equal then - informer.field_selector = "metadata.namespace%3D" .. ns.equal + return + end + + if ns.equal then + informer.field_selector = "metadata.namespace=" .. ns.equal informer.namespace_selector = nil - elseif ns.not_equal then - informer.field_selector = "metadata.namespace%21%3D" .. ns.not_equal + return + end + + if ns.not_equal then + informer.field_selector = "metadata.namespace!=" .. ns.not_equal informer.namespace_selector = nil - elseif ns.match then + return + end + + if ns.match then informer.namespace_selector = function(self, namespace) local match = conf.namespace_selector.match local m, err @@ -145,7 +169,10 @@ local function setup_namespace_selector(conf, informer) end return false end - elseif ns.not_match then + return + end + + if ns.not_match then informer.namespace_selector = function(self, namespace) local not_match = conf.namespace_selector.not_match local m, err @@ -160,6 +187,7 @@ local function setup_namespace_selector(conf, informer) end return true end + return end end @@ -180,55 +208,61 @@ local function read_env(key) return true, key, nil end -local function setup_apiserver(conf, apiserver) +local function get_apiserver(conf) + local apiserver = { + schema = "", + host = "", + port = "", + token = "" + } apiserver.schema = conf.service.schema local ok, value, message ok, value, message = read_env(conf.service.host) if not ok then - return false, message + return nil, message end apiserver.host = value if apiserver.host == "" then - return false, "get empty host value" + return nil, "get empty host value" end ok, value, message = read_env(conf.service.port) if not ok then - return false, message + return nil, message end apiserver.port = tonumber(value) if not apiserver.port or apiserver.port <= 0 or apiserver.port > 65535 then - return false, "get invalid port value: " .. apiserver.port + return nil, "get invalid port value: " .. apiserver.port end -- we should not check if the apiserver.token is empty here if conf.client.token then ok, value, message = read_env(conf.client.token) if not ok then - return false, message + return nil, message end apiserver.token = value elseif conf.client.token_file and conf.client.token_file ~= "" then ok, value, message = read_env(conf.client.token_file) if not ok then - return false, message + return nil, message end local apiserver_token_file = value apiserver.token, message = util.read_file(apiserver_token_file) if not apiserver.token then - return false, message + return nil, message end else - return false, "invalid kubernetes discovery configuration:" .. + return nil, "invalid kubernetes discovery configuration:" .. "should set one of [client.token,client.token_file] but none" end - return true, nil + return apiserver, nil end local function create_endpoint_lrucache(endpoint_key, endpoint_port) @@ -280,56 +314,38 @@ function _M.init_worker() default_weight = discovery_conf.default_weight or 50 - local ok, err = setup_apiserver(discovery_conf, kubernetes.apiserver) - if not ok then + local apiserver, err = get_apiserver(discovery_conf) + if not apiserver then error(err) return end - local endpoint_informer = kubernetes.informer_factory.new("", "v1", + local endpoints_informer = informer_factory.new("", "v1", "Endpoints", "endpoints", "") - setup_label_selector(discovery_conf, endpoint_informer) - setup_namespace_selector(discovery_conf, endpoint_informer) - - endpoint_informer.on_added = function(self, object, drive) - if self.namespace_selector == nil then - on_endpoint_modified(object) - elseif self:namespace_selector(object.metadata.namespace) then - on_endpoint_modified(object) - end - end + setup_namespace_selector(discovery_conf, endpoints_informer) + setup_label_selector(discovery_conf, endpoints_informer) - endpoint_informer.on_modified = endpoint_informer.on_added - - endpoint_informer.on_deleted = function(self, object) - if self.namespace_selector ~= nil then - if self:namespace_selector(object.metadata.namespace) then - on_endpoint_deleted(object) - end - else - on_endpoint_deleted(object) - end - end - - endpoint_informer.pre_list = function(self) - endpoint_dict:flush_all() - end - - endpoint_informer.post_list = function(self) - endpoint_dict:flush_expired() - end + endpoints_informer.on_added = on_endpoint_modified + endpoints_informer.on_modified = on_endpoint_modified + endpoints_informer.on_deleted = on_endpoint_deleted + endpoints_informer.pre_list = pre_list + endpoints_informer.post_list = post_list local timer_runner - timer_runner = function(premature, informer) - if not informer:list_watch() then + timer_runner = function(premature) + if premature then + return + end + local status, ok = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver) + if not status or not ok then local retry_interval = 40 ngx.sleep(retry_interval) end - ngx.timer.at(0, timer_runner, informer) + ngx.timer.at(0, timer_runner) end - ngx.timer.at(0, timer_runner, endpoint_informer) + ngx.timer.at(0, timer_runner) end return _M diff --git a/apisix/discovery/kubernetes/kubernetes.lua b/apisix/discovery/kubernetes/kubernetes.lua deleted file mode 100644 index dda915b54ce3..000000000000 --- a/apisix/discovery/kubernetes/kubernetes.lua +++ /dev/null @@ -1,364 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -local ngx = ngx -local ipairs = ipairs -local string = string -local math = math -local core = require("apisix.core") -local http = require("resty.http") - -local _constants = { - ErrorEvent = "ERROR", - AddedEvent = "ADDED", - ModifiedEvent = "MODIFIED", - DeletedEvent = "DELETED", - BookmarkEvent = "BOOKMARK", - - ListDrive = "list", - WatchDrive = "watch", - - Success = "Success", - RequestError = "RequestError", - ReadBodyError = "ReadBodyError", - UnexpectedBody = "UnexpectedBody", - GmatchError = "GmatchError", - ResourceGone = "ResourceGone", -} - -local _apiserver = { - schema = "", - host = "", - port = "", - token = "" -} - -local empty_table = {} - -local function list(httpc, informer) - local response, err = httpc:request({ - path = informer.path, - query = informer:list_query(), - headers = { - ["Host"] = _apiserver.host .. ":" .. _apiserver.port, - ["Authorization"] = "Bearer " .. _apiserver.token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } - }) - - core.log.info("--raw=" .. informer.path .. "?" .. informer:list_query()) - - if not response then - return false, _constants.RequestError, err or "" - end - - if response.status ~= 200 then - return false, response.reason, response:read_body() or "" - end - - local body, err = response:read_body() - if err then - return false, _constants.ReadBodyError, err - end - - local data, _ = core.json.decode(body) - if not data or data.kind ~= informer.list_kind then - return false, _constants.UnexpectedBody, body - end - - informer.version = data.metadata.resourceVersion - - if informer.on_added ~= nil then - for _, item in ipairs(data.items or empty_table) do - informer:on_added(item, _constants.ListDrive) - end - end - - informer.continue = data.metadata.continue - if informer.continue ~= nil and informer.continue ~= "" then - list(httpc, informer) - end - - return true, _constants.Success, "" -end - -local function split_event (body, dispatch_event) - local gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") - if not gmatch_iterator then - return nil, _constants.GmatchError, err - end - - local captures - local captured_size = 0 - local ok, reason - while true do - captures, err = gmatch_iterator() - - if err then - return nil, _constants.GmatchError, err - end - - if not captures then - break - end - - captured_size = captured_size + #captures[0] - - ok, reason, err = dispatch_event(captures[0]) - if not ok then - return nil, reason, err - end - end - - local remainder_body - if captured_size == #body then - remainder_body = "" - elseif captured_size == 0 then - remainder_body = body - elseif captured_size < #body then - remainder_body = string.sub(body, captured_size + 1) - end - - return remainder_body, _constants.Success, nil -end - -local function watch(httpc, informer) - - local dispatch_event = function(event_string) - local event, _ = core.json.decode(event_string) - - if not event or not event.type or not event.object then - return false, _constants.UnexpectedBody, event_string - end - - local type = event.type - - if type == _constants.ErrorEvent then - if event.object.code == 410 then - return false, _constants.ResourceGone, nil - end - return false, _constants.UnexpectedBody, event_string - end - - local object = event.object - informer.version = object.metadata.resourceVersion - - if type == _constants.AddedEvent then - if informer.on_added ~= nil then - informer:on_added(object, _constants.WatchDrive) - end - elseif type == _constants.DeletedEvent then - if informer.on_deleted ~= nil then - informer:on_deleted(object) - end - elseif type == _constants.ModifiedEvent then - if informer.on_modified ~= nil then - informer:on_modified(object) - end - -- elseif type == _constants.BookmarkEvent then - -- do nothing - end - return true, _constants.Success, nil - end - - local watch_times = 8 - for _ = 1, watch_times do - local watch_seconds = 1800 + math.random(9, 999) - informer.overtime = watch_seconds - local http_seconds = watch_seconds + 120 - httpc:set_timeouts(2000, 3000, http_seconds * 1000) - - local response, err = httpc:request({ - path = informer.path, - query = informer:watch_query(), - headers = { - ["Host"] = _apiserver.host .. ":" .. _apiserver.port, - ["Authorization"] = "Bearer " .. _apiserver.token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } - }) - - core.log.info("--raw=" .. informer.path .. "?" .. informer:watch_query()) - - if err then - return false, _constants.RequestError, err - end - - if response.status ~= 200 then - return false, response.reason, response:read_body() or "" - end - - local remainder_body - local body - local reason - - while true do - body, err = response.body_reader() - if err then - return false, _constants.ReadBodyError, err - end - - if not body then - break - end - - if remainder_body ~= nil and #remainder_body > 0 then - body = remainder_body .. body - end - - remainder_body, reason, err = split_event(body, dispatch_event) - if reason ~= _constants.Success then - if reason == _constants.ResourceGone then - return true, _constants.Success, nil - end - return false, reason, err - end - end - end - return true, _constants.Success, nil -end - -local _informer_factory = { -} - -function _informer_factory.new(group, version, kind, plural, namespace) - local _t = { - kind = kind, - list_kind = kind .. "List", - plural = plural, - path = "", - version = "", - continue = "", - overtime = "1800", - limit = 120, - label_selector = "", - field_selector = "", - } - - if group == "" then - _t.path = _t.path .. "/api/" .. version - else - _t.path = _t.path .. "/apis/" .. group .. "/" .. version - end - - if namespace ~= "" then - _t.path = _t.path .. "/namespace/" .. namespace - end - _t.path = _t.path .. "/" .. plural - - function _t.list_query(self) - local uri = "limit=" .. self.limit - - if self.continue ~= nil and self.continue ~= "" then - uri = uri .. "&continue=" .. self.continue - end - - if self.label_selector and self.label_selector ~= "" then - uri = uri .. "&labelSelector=" .. self.label_selector - end - - if self.field_selector and self.field_selector ~= "" then - uri = uri .. "&fieldSelector=" .. self.field_selector - end - - return uri - end - - function _t.watch_query(self) - local uri = "watch=true&allowWatchBookmarks=true&timeoutSeconds=" .. self.overtime - - if self.version ~= nil and self.version ~= "" then - uri = uri .. "&resourceVersion=" .. self.version - end - - if self.label_selector and self.label_selector ~= "" then - uri = uri .. "&labelSelector=" .. self.label_selector - end - - if self.field_selector and self.field_selector ~= "" then - uri = uri .. "&fieldSelector=" .. self.field_selector - end - - return uri - end - - function _t.list_watch(self) - local ok - local reason, message - local httpc = http.new() - - self.fetch_state = "connecting" - core.log.info("begin to connect ", _apiserver.host, ":", _apiserver.port) - - ok, message = httpc:connect({ - scheme = _apiserver.schema, - host = _apiserver.host, - port = _apiserver.port, - ssl_verify = false - }) - - if not ok then - self.fetch_state = "connecting" - core.log.error("connect apiserver failed , _apiserver.host: ", _apiserver.host, - ", _apiserver.port: ", _apiserver.port, ", message : ", message) - return false - end - - core.log.info("begin to list ", self.kind) - self.fetch_state = "listing" - if self.pre_List ~= nil then - self:pre_list() - end - - ok, reason, message = list(httpc, self) - if not ok then - self.fetch_state = "list failed" - core.log.error("list failed, kind: ", self.kind, - ", reason: ", reason, ", message : ", message) - return false - end - - self.fetch_state = "list finished" - if self.post_List ~= nil then - self:post_list() - end - - core.log.info("begin to watch ", self.kind) - self.fetch_state = "watching" - ok, reason, message = watch(httpc, self) - if not ok then - self.fetch_state = "watch failed" - core.log.error("watch failed, kind: ", self.kind, - ", reason: ", reason, ", message : ", message) - return false - end - - self.fetch_state = "watch finished" - return true - end - - return _t -end - -return { - version = "0.0.1", - informer_factory = _informer_factory, - apiserver = _apiserver, - __index = _constants -} diff --git a/t_kubernetes/discovery/kubernetes.t b/t_kubernetes/discovery/kubernetes.t index f1c6a0280884..9d44e6a4e6ab 100644 --- a/t_kubernetes/discovery/kubernetes.t +++ b/t_kubernetes/discovery/kubernetes.t @@ -598,31 +598,7 @@ qr{ 0 0 0 0 2 2 } -=== TEST 13: use namespace selector not_match with regex ---- yaml_config -apisix: - node_listen: 1984 - config_center: yaml - enable_admin: false -discovery: - kubernetes: - client: - token_file: ${KUBERNETES_CLIENT_TOKEN_FILE} - namespace_selector: - not_match: ["ns-[ab]"] ---- request -GET /queries -["ns-a/ep:p1","ns-a/ep:p2","ns-b/ep:p1","ns-b/ep:p2","ns-c/ep:5001","ns-c/ep:5002"] ---- more_headers -Content-type: application/json ---- response_body eval -qr{ 0 0 0 0 2 2 } ---- no_error_log -[error] - - - -=== TEST 14: use label selector +=== TEST 13: use label selector --- yaml_config apisix: node_listen: 1984 From 871e429b4f51319a1411d7cee98fc775f5d6fdd7 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Sat, 5 Feb 2022 19:09:33 +0800 Subject: [PATCH 63/73] feat: add kubernetes discovery module --- .github/workflows/kubernetes-ci.yml | 2 +- apisix/discovery/kubernetes/informer_factory.lua | 2 +- apisix/discovery/kubernetes/init.lua | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index bf5e3a6860d8..b622d34fef3b 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -28,7 +28,7 @@ jobs: - linux_openresty_1_17 runs-on: ${{ matrix.platform }} - timeout-minutes: 90 + timeout-minutes: 15 env: SERVER_NAME: ${{ matrix.os_name }} OPENRESTY_VERSION: default diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index ecbce4ed1ed9..72eebbb959be 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -268,7 +268,7 @@ local function list_watch(self, apiserver) }) if not ok then - self.fetch_state = "connecting" + self.fetch_state = "connect failed" core.log.error("connect apiserver failed , apiserver.host: ", apiserver.host, ", apiserver.port: ", apiserver.port, ", message : ", message) return false diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 33233ecf16db..3b9a70723a0a 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -266,7 +266,7 @@ local function get_apiserver(conf) end local function create_endpoint_lrucache(endpoint_key, endpoint_port) - local endpoint_content, _, _ = endpoint_dict:get_stale(endpoint_key) + local endpoint_content = endpoint_dict:get_stale(endpoint_key) if not endpoint_content then core.log.error("get empty endpoint content from discovery DIC, this should not happen ", endpoint_key) @@ -296,7 +296,7 @@ function _M.nodes(service_name) local endpoint_key = match[1] local endpoint_port = match[2] - local endpoint_version, _, _ = endpoint_dict:get_stale(endpoint_key .. "#version") + local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version") if not endpoint_version then core.log.info("get empty endpoint version from discovery DICT ", endpoint_key) return nil From 8f3d9b2a7d0e73b8ce1f7a9a7d96186ed130ffb9 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Sat, 5 Feb 2022 19:24:04 +0800 Subject: [PATCH 64/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/init.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 3b9a70723a0a..3c222fde35e7 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -310,6 +310,11 @@ function _M.init_worker() return end + if not endpoint_dict then + error("get empty endpoint_dict") + return + end + local discovery_conf = local_conf.discovery.kubernetes default_weight = discovery_conf.default_weight or 50 From e4d9f05d7c2edb84425ae4eaacf9299316186210 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Sun, 6 Feb 2022 23:20:40 +0800 Subject: [PATCH 65/73] feat: add kubernetes discovery module --- .../discovery/kubernetes/informer_factory.lua | 148 ++++++++++-------- apisix/discovery/kubernetes/init.lua | 87 +++++----- 2 files changed, 133 insertions(+), 102 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 72eebbb959be..5b0a18d450e5 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -19,7 +19,7 @@ local ngx = ngx local ipairs = ipairs local string = string local math = math -local setmetatable = setmetatable +local type = type local core = require("apisix.core") local http = require("resty.http") @@ -72,7 +72,7 @@ local function list(httpc, apiserver, informer) return false, "ReadBodyError", err end - local data, _ = core.json.decode(body) + local data = core.json.decode(body) if not data or data.kind ~= informer.list_kind then return false, "UnexpectedBody", body end @@ -90,7 +90,7 @@ local function list(httpc, apiserver, informer) list(httpc, informer) end - return true, "Success", "" + return true end local function watch_query(informer) @@ -115,7 +115,7 @@ local function watch_query(informer) return ngx.encode_args(arguments) end -local function split_event (body, dispatch_event) +local function split_event (body, callback, ...) local gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") if not gmatch_iterator then return nil, "GmatchError", err @@ -137,7 +137,7 @@ local function split_event (body, dispatch_event) captured_size = captured_size + #captures[0] - ok, reason, err = dispatch_event(captures[0]) + ok, reason, err = callback(captures[0], ...) if not ok then return nil, reason, err end @@ -152,48 +152,47 @@ local function split_event (body, dispatch_event) remainder_body = string.sub(body, captured_size + 1) end - return remainder_body, "Success", nil + return remainder_body, "Success" end -local function watch(httpc, apiserver, informer) - - local dispatch_event = function(event_string) - local event, _ = core.json.decode(event_string) +local function dispatch_event(event_string, informer) + local event, _ = core.json.decode(event_string) - if not event or not event.type or not event.object then - return false, "UnexpectedBody", event_string - end + if not event or not event.type or not event.object then + return false, "UnexpectedBody", event_string + end - local type = event.type + local tp = event.type - if type == "ERROR" then - if event.object.code == 410 then - return false, "ResourceGone", nil - end - return false, "UnexpectedBody", event_string + if tp == "ERROR" then + if event.object.code == 410 then + return false, "ResourceGone", nil end + return false, "UnexpectedBody", event_string + end - local object = event.object - informer.version = object.metadata.resourceVersion + local object = event.object + informer.version = object.metadata.resourceVersion - if type == "ADDED" then - if informer.on_added ~= nil then - informer:on_added(object, "watch") - end - elseif type == "DELETED" then - if informer.on_deleted ~= nil then - informer:on_deleted(object) - end - elseif type == "MODIFIED" then - if informer.on_modified ~= nil then - informer:on_modified(object) - end - -- elseif type == "BOOKMARK" then - -- do nothing + if tp == "ADDED" then + if informer.on_added ~= nil then + informer:on_added(object, "watch") + end + elseif tp == "DELETED" then + if informer.on_deleted ~= nil then + informer:on_deleted(object) end - return true, "Success", nil + elseif tp == "MODIFIED" then + if informer.on_modified ~= nil then + informer:on_modified(object) + end + -- elseif type == "BOOKMARK" then + -- do nothing end + return true +end +local function watch(httpc, apiserver, informer) local watch_times = 8 for _ = 1, watch_times do local watch_seconds = 1800 + math.random(9, 999) @@ -240,24 +239,24 @@ local function watch(httpc, apiserver, informer) body = remainder_body .. body end - remainder_body, reason, err = split_event(body, dispatch_event) + remainder_body, reason, err = split_event(body, dispatch_event, informer) if reason ~= "Success" then if reason == "ResourceGone" then - return true, "Success", nil + return true end return false, reason, err end end end - return true, "Success", nil + return true end -local function list_watch(self, apiserver) +local function list_watch(informer, apiserver) local ok local reason, message local httpc = http.new() - self.fetch_state = "connecting" + informer.fetch_state = "connecting" core.log.info("begin to connect ", apiserver.host, ":", apiserver.port) ok, message = httpc:connect({ @@ -268,42 +267,42 @@ local function list_watch(self, apiserver) }) if not ok then - self.fetch_state = "connect failed" + informer.fetch_state = "connect failed" core.log.error("connect apiserver failed , apiserver.host: ", apiserver.host, ", apiserver.port: ", apiserver.port, ", message : ", message) return false end - core.log.info("begin to list ", self.kind) - self.fetch_state = "listing" - if self.pre_List ~= nil then - self:pre_list() + core.log.info("begin to list ", informer.kind) + informer.fetch_state = "listing" + if informer.pre_List ~= nil then + informer:pre_list() end - ok, reason, message = list(httpc, apiserver, self) + ok, reason, message = list(httpc, apiserver, informer) if not ok then - self.fetch_state = "list failed" - core.log.error("list failed, kind: ", self.kind, + informer.fetch_state = "list failed" + core.log.error("list failed, kind: ", informer.kind, ", reason: ", reason, ", message : ", message) return false end - self.fetch_state = "list finished" - if self.post_List ~= nil then - self:post_list() + informer.fetch_state = "list finished" + if informer.post_List ~= nil then + informer:post_list() end - core.log.info("begin to watch ", self.kind) - self.fetch_state = "watching" - ok, reason, message = watch(httpc, apiserver, self) + core.log.info("begin to watch ", informer.kind) + informer.fetch_state = "watching" + ok, reason, message = watch(httpc, apiserver, informer) if not ok then - self.fetch_state = "watch failed" - core.log.error("watch failed, kind: ", self.kind, + informer.fetch_state = "watch failed" + core.log.error("watch failed, kind: ", informer.kind, ", reason: ", reason, ", message : ", message) return false end - self.fetch_state = "watch finished" + informer.fetch_state = "watch finished" return true end @@ -311,19 +310,45 @@ local _M = { } function _M.new(group, version, kind, plural, namespace) + local tp + tp = type(group) + if tp ~= nil and tp ~= "string" then + return nil, "group should set to string or nil type but " .. tp + end + + tp = type(namespace) + if tp ~= nil and tp ~= "string" then + return nil, "namespace should set to string or nil type but " .. tp + end + + tp = type(version) + if tp ~= "string" or version == "" then + return nil, "version should set to non-empty string" + end + + tp = type(kind) + if tp ~= "string" or kind == "" then + return nil, "kind should set to non-empty string" + end + + tp = type(plural) + if tp ~= "string" or plural == "" then + return nil, "plural should set to non-empty string" + end + local path = "" - if group == "" then + if group == nil or group == "" then path = path .. "/api/" .. version else path = path .. "/apis/" .. group .. "/" .. version end - if namespace ~= "" then + if namespace ~= nil and namespace ~= "" then path = path .. "/namespace/" .. namespace end path = path .. "/" .. plural - return setmetatable({}, { __index = { + return { kind = kind, list_kind = kind .. "List", plural = plural, @@ -336,7 +361,6 @@ function _M.new(group, version, kind, plural, namespace) continue = "", list_watch = list_watch } - }) end return _M diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 3c222fde35e7..ac65f433b992 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -29,7 +29,10 @@ local core = require("apisix.core") local util = require("apisix.cli.util") local local_conf = require("apisix.core.config_local").local_conf() local informer_factory = require("apisix.discovery.kubernetes.informer_factory") -local endpoint_dict = ngx.shared.discovery +local endpoint_dict = ngx.shared["discovery"] +if not endpoint_dict then + error("failed to get ngx.shared.dict discovery when load kubernetes discovery plugin ") +end local default_weight = 0 @@ -198,14 +201,14 @@ local function read_env(key) -- '$', '{', '}' == 36,123,125 if a == 36 and b == 123 and c == 125 then local env = string.sub(key, 3, #key - 1) - local val = os.getenv(env) - if not val then - return false, nil, "not found environment variable " .. env + local value = os.getenv(env) + if not value then + return nil, "not found environment variable " .. env end - return true, val, nil + return value, nil end end - return true, key, nil + return key, nil end local function get_apiserver(conf) @@ -217,52 +220,56 @@ local function get_apiserver(conf) } apiserver.schema = conf.service.schema + if apiserver.schema ~= "http" and apiserver.schema ~= "https" then + return nil, "service.schema should set to one of [http,https] but " .. apiserver.schema + end - local ok, value, message - ok, value, message = read_env(conf.service.host) - if not ok then - return nil, message + local err + apiserver.host, err = read_env(conf.service.host) + if err then + return nil, err end - apiserver.host = value if apiserver.host == "" then - return nil, "get empty host value" + return nil, "service.host should set to non-empty string" end - ok, value, message = read_env(conf.service.port) - if not ok then - return nil, message + local port + port, err = read_env(conf.service.port) + if err then + return nil, err end - apiserver.port = tonumber(value) + apiserver.port = tonumber(port) if not apiserver.port or apiserver.port <= 0 or apiserver.port > 65535 then - return nil, "get invalid port value: " .. apiserver.port + return nil, "invalid port value: ", apiserver.port end - -- we should not check if the apiserver.token is empty here if conf.client.token then - ok, value, message = read_env(conf.client.token) - if not ok then - return nil, message + apiserver.token, err = read_env(conf.client.token) + if err then + return nil, err end - apiserver.token = value elseif conf.client.token_file and conf.client.token_file ~= "" then - ok, value, message = read_env(conf.client.token_file) - if not ok then - return nil, message + local file + file, err = read_env(conf.client.token_file) + if err then + return nil, err end - local apiserver_token_file = value - apiserver.token, message = util.read_file(apiserver_token_file) - if not apiserver.token then - return nil, message + apiserver.token, err = util.read_file(file) + if err then + return nil, err end else - return nil, "invalid kubernetes discovery configuration:" .. - "should set one of [client.token,client.token_file] but none" + return nil, "one of [client.token,client.token_file] should be set but none" end - return apiserver, nil + if apiserver.schema == "https" and apiserver.token == "" then + return nil, "service.host should set to non-empty string when service.schema is https" + end + + return apiserver end local function create_endpoint_lrucache(endpoint_key, endpoint_port) @@ -273,10 +280,11 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port) return nil end - local endpoint, _ = core.json.decode(endpoint_content) + local endpoint = core.json.decode(endpoint_content) if not endpoint then core.log.error("decode endpoint content failed, this should not happen, content: ", endpoint_content) + return nil end return endpoint[endpoint_port] @@ -310,23 +318,22 @@ function _M.init_worker() return end - if not endpoint_dict then - error("get empty endpoint_dict") - return - end - local discovery_conf = local_conf.discovery.kubernetes default_weight = discovery_conf.default_weight or 50 local apiserver, err = get_apiserver(discovery_conf) - if not apiserver then + if err then error(err) return end - local endpoints_informer = informer_factory.new("", "v1", + local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "") + if err then + error(err) + return + end setup_namespace_selector(discovery_conf, endpoints_informer) setup_label_selector(discovery_conf, endpoints_informer) From 1dd2c37daaa8b90cc9bf879fc5b6628d3d6e440d Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Sun, 6 Feb 2022 23:26:55 +0800 Subject: [PATCH 66/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/informer_factory.lua | 3 +++ apisix/discovery/kubernetes/init.lua | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 5b0a18d450e5..fbe2bc115183 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -189,6 +189,7 @@ local function dispatch_event(event_string, informer) -- elseif type == "BOOKMARK" then -- do nothing end + return true end @@ -248,6 +249,7 @@ local function watch(httpc, apiserver, informer) end end end + return true end @@ -303,6 +305,7 @@ local function list_watch(informer, apiserver) end informer.fetch_state = "watch finished" + return true end diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index ac65f433b992..6f65239ee104 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -48,6 +48,7 @@ local function sort_nodes_cmp(left, right) if left.host ~= right.host then return left.host < right.host end + return left.port < right.port end @@ -208,7 +209,8 @@ local function read_env(key) return value, nil end end - return key, nil + + return key end local function get_apiserver(conf) @@ -309,6 +311,7 @@ function _M.nodes(service_name) core.log.info("get empty endpoint version from discovery DICT ", endpoint_key) return nil end + return endpoint_lrucache(service_name, endpoint_version, create_endpoint_lrucache, endpoint_key, endpoint_port) end From 7849b8beba676bbc5ca97b9f1329571ed49846f0 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 9 Feb 2022 10:29:49 +0800 Subject: [PATCH 67/73] feat: add kubernetes discovery module --- .../discovery/kubernetes/informer_factory.lua | 34 +++++++------ apisix/discovery/kubernetes/init.lua | 48 +++++++++++++------ 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index fbe2bc115183..bfd12a2a8f73 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -30,7 +30,7 @@ local function list_query(informer) limit = informer.limit, } - if informer.continue ~= nil and informer.continue ~= "" then + if informer.continue and informer.continue ~= "" then arguments.continue = informer.continue end @@ -45,6 +45,7 @@ local function list_query(informer) return ngx.encode_args(arguments) end + local function list(httpc, apiserver, informer) local response, err = httpc:request({ path = informer.path, @@ -79,20 +80,21 @@ local function list(httpc, apiserver, informer) informer.version = data.metadata.resourceVersion - if informer.on_added ~= nil then + if informer.on_added then for _, item in ipairs(data.items or empty_table) do informer:on_added(item, "list") end end informer.continue = data.metadata.continue - if informer.continue ~= nil and informer.continue ~= "" then + if informer.continue and informer.continue ~= "" then list(httpc, informer) end return true end + local function watch_query(informer) local arguments = { watch = "true", @@ -100,7 +102,7 @@ local function watch_query(informer) timeoutSeconds = informer.overtime, } - if informer.version ~= nil and informer.version ~= "" then + if informer.version and informer.version ~= "" then arguments.resourceVersion = informer.version end @@ -115,6 +117,7 @@ local function watch_query(informer) return ngx.encode_args(arguments) end + local function split_event (body, callback, ...) local gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") if not gmatch_iterator then @@ -155,6 +158,7 @@ local function split_event (body, callback, ...) return remainder_body, "Success" end + local function dispatch_event(event_string, informer) local event, _ = core.json.decode(event_string) @@ -175,15 +179,15 @@ local function dispatch_event(event_string, informer) informer.version = object.metadata.resourceVersion if tp == "ADDED" then - if informer.on_added ~= nil then + if informer.on_added then informer:on_added(object, "watch") end elseif tp == "DELETED" then - if informer.on_deleted ~= nil then + if informer.on_deleted then informer:on_deleted(object) end elseif tp == "MODIFIED" then - if informer.on_modified ~= nil then + if informer.on_modified then informer:on_modified(object) end -- elseif type == "BOOKMARK" then @@ -193,6 +197,7 @@ local function dispatch_event(event_string, informer) return true end + local function watch(httpc, apiserver, informer) local watch_times = 8 for _ = 1, watch_times do @@ -236,7 +241,7 @@ local function watch(httpc, apiserver, informer) break end - if remainder_body ~= nil and #remainder_body > 0 then + if remainder_body and #remainder_body > 0 then body = remainder_body .. body end @@ -253,6 +258,7 @@ local function watch(httpc, apiserver, informer) return true end + local function list_watch(informer, apiserver) local ok local reason, message @@ -277,7 +283,7 @@ local function list_watch(informer, apiserver) core.log.info("begin to list ", informer.kind) informer.fetch_state = "listing" - if informer.pre_List ~= nil then + if informer.pre_List then informer:pre_list() end @@ -290,7 +296,7 @@ local function list_watch(informer, apiserver) end informer.fetch_state = "list finished" - if informer.post_List ~= nil then + if informer.post_List then informer:post_list() end @@ -315,12 +321,12 @@ local _M = { function _M.new(group, version, kind, plural, namespace) local tp tp = type(group) - if tp ~= nil and tp ~= "string" then + if tp ~= "nil" and tp ~= "string" then return nil, "group should set to string or nil type but " .. tp end tp = type(namespace) - if tp ~= nil and tp ~= "string" then + if tp ~= "nil" and tp ~= "string" then return nil, "namespace should set to string or nil type but " .. tp end @@ -340,13 +346,13 @@ function _M.new(group, version, kind, plural, namespace) end local path = "" - if group == nil or group == "" then + if group or group == "" then path = path .. "/api/" .. version else path = path .. "/apis/" .. group .. "/" .. version end - if namespace ~= nil and namespace ~= "" then + if namespace and namespace ~= "" then path = path .. "/namespace/" .. namespace end path = path .. "/" .. plural diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 6f65239ee104..5787251c08b4 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -29,12 +29,9 @@ local core = require("apisix.core") local util = require("apisix.cli.util") local local_conf = require("apisix.core.config_local").local_conf() local informer_factory = require("apisix.discovery.kubernetes.informer_factory") -local endpoint_dict = ngx.shared["discovery"] -if not endpoint_dict then - error("failed to get ngx.shared.dict discovery when load kubernetes discovery plugin ") -end -local default_weight = 0 +local endpoint_dict +local default_weight local endpoint_lrucache = core.lrucache.new({ ttl = 300, @@ -52,8 +49,9 @@ local function sort_nodes_cmp(left, right) return left.port < right.port end + local function on_endpoint_modified(informer, endpoint) - if informer.namespace_selector ~= nil and + if informer.namespace_selector and not informer:namespace_selector(endpoint.metadata.namespace) then return end @@ -63,7 +61,7 @@ local function on_endpoint_modified(informer, endpoint) local subsets = endpoint.subsets for _, subset in ipairs(subsets or empty_table) do - if subset.addresses ~= nil then + if subset.addresses then local addresses = subset.addresses for _, port in ipairs(subset.ports or empty_table) do local port_name @@ -115,8 +113,9 @@ local function on_endpoint_modified(informer, endpoint) end end + local function on_endpoint_deleted(informer, endpoint) - if informer.namespace_selector ~= nil and + if informer.namespace_selector and not informer:namespace_selector(endpoint.metadata.namespace) then return end @@ -127,18 +126,22 @@ local function on_endpoint_deleted(informer, endpoint) endpoint_dict:delete(endpoint_key) end + local function pre_list(informer) endpoint_dict:flush_all() end + local function post_list(informer) endpoint_dict:flush_expired() end + local function setup_label_selector(conf, informer) informer.label_selector = conf.label_selector end + local function setup_namespace_selector(conf, informer) local ns = conf.namespace_selector if ns == nil then @@ -195,6 +198,7 @@ local function setup_namespace_selector(conf, informer) end end + local function read_env(key) if #key > 3 then local a, b = string.byte(key, 1, 2) @@ -213,6 +217,7 @@ local function read_env(key) return key end + local function get_apiserver(conf) local apiserver = { schema = "", @@ -244,7 +249,7 @@ local function get_apiserver(conf) apiserver.port = tonumber(port) if not apiserver.port or apiserver.port <= 0 or apiserver.port > 65535 then - return nil, "invalid port value: ", apiserver.port + return nil, "invalid port value: " .. apiserver.port end if conf.client.token then @@ -274,6 +279,7 @@ local function get_apiserver(conf) return apiserver end + local function create_endpoint_lrucache(endpoint_key, endpoint_port) local endpoint_content = endpoint_dict:get_stale(endpoint_key) if not endpoint_content then @@ -316,7 +322,14 @@ function _M.nodes(service_name) create_endpoint_lrucache, endpoint_key, endpoint_port) end + function _M.init_worker() + -- TODO: maybe we can read dict name from discovery config + endpoint_dict = ngx.shared.discovery + if not endpoint_dict then + error("failed to get ngx.shared.dict discovery") + end + if process.type() ~= "privileged agent" then return end @@ -352,12 +365,19 @@ function _M.init_worker() if premature then return end - local status, ok = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver) - if not status or not ok then - local retry_interval = 40 - ngx.sleep(retry_interval) + + local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver) + + local retry_interval = 0 + if not ok then + core.log.error("list_watch failed, kind: ", endpoints_informer.kind, + ", reason: ", "RuntimeException", ", message : ", status) + retry_interval = 40 + elseif not status then + retry_interval = 40 end - ngx.timer.at(0, timer_runner) + + ngx.timer.at(retry_interval, timer_runner) end ngx.timer.at(0, timer_runner) From 02ef870748a5372af5a861cca916183781f04d0f Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Wed, 9 Feb 2022 15:29:26 +0800 Subject: [PATCH 68/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/informer_factory.lua | 4 ++-- apisix/discovery/kubernetes/init.lua | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index bfd12a2a8f73..911ec872439a 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -276,7 +276,7 @@ local function list_watch(informer, apiserver) if not ok then informer.fetch_state = "connect failed" - core.log.error("connect apiserver failed , apiserver.host: ", apiserver.host, + core.log.error("connect apiserver failed, apiserver.host: ", apiserver.host, ", apiserver.port: ", apiserver.port, ", message : ", message) return false end @@ -346,7 +346,7 @@ function _M.new(group, version, kind, plural, namespace) end local path = "" - if group or group == "" then + if group == nil or group == "" then path = path .. "/api/" .. version else path = path .. "/apis/" .. group .. "/" .. version diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 5787251c08b4..eb02b9eaa91d 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -327,7 +327,7 @@ function _M.init_worker() -- TODO: maybe we can read dict name from discovery config endpoint_dict = ngx.shared.discovery if not endpoint_dict then - error("failed to get ngx.shared.dict discovery") + error("failed to get Nginx shared dict: discovery, please check your APISIX version") end if process.type() ~= "privileged agent" then From fa017d2f3bdb522ddd3e30cff9d250886a5e1873 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 14 Feb 2022 09:04:34 +0800 Subject: [PATCH 69/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/informer_factory.lua | 13 +++++++------ apisix/discovery/kubernetes/init.lua | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 911ec872439a..e8a02c762606 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -121,7 +121,7 @@ end local function split_event (body, callback, ...) local gmatch_iterator, err = ngx.re.gmatch(body, "{\"type\":.*}\n", "jao") if not gmatch_iterator then - return nil, "GmatchError", err + return false, nil, "GmatchError", err end local captures @@ -131,7 +131,7 @@ local function split_event (body, callback, ...) captures, err = gmatch_iterator() if err then - return nil, "GmatchError", err + return false, nil, "GmatchError", err end if not captures then @@ -142,7 +142,7 @@ local function split_event (body, callback, ...) ok, reason, err = callback(captures[0], ...) if not ok then - return nil, reason, err + return false, nil, reason, err end end @@ -155,7 +155,7 @@ local function split_event (body, callback, ...) remainder_body = string.sub(body, captured_size + 1) end - return remainder_body, "Success" + return true, remainder_body end @@ -245,8 +245,9 @@ local function watch(httpc, apiserver, informer) body = remainder_body .. body end - remainder_body, reason, err = split_event(body, dispatch_event, informer) - if reason ~= "Success" then + local ok + ok, remainder_body, reason, err = split_event(body, dispatch_event, informer) + if not ok and reason ~= "Success" then if reason == "ResourceGone" then return true end diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index eb02b9eaa91d..9a8770344d97 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -327,7 +327,7 @@ function _M.init_worker() -- TODO: maybe we can read dict name from discovery config endpoint_dict = ngx.shared.discovery if not endpoint_dict then - error("failed to get Nginx shared dict: discovery, please check your APISIX version") + error("failed to get nginx shared dict: discovery, please check your APISIX version") end if process.type() ~= "privileged agent" then From 57d7f910010c269ead215f44bed3e1e77ff39e0e Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 14 Feb 2022 17:52:20 +0800 Subject: [PATCH 70/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/informer_factory.lua | 1 + apisix/discovery/kubernetes/init.lua | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index e8a02c762606..cd1391afa865 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -227,6 +227,7 @@ local function watch(httpc, apiserver, informer) return false, response.reason, response:read_body() or "" end + local ok local remainder_body local body local reason diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 9a8770344d97..ba83588a79d1 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -273,7 +273,7 @@ local function get_apiserver(conf) end if apiserver.schema == "https" and apiserver.token == "" then - return nil, "service.host should set to non-empty string when service.schema is https" + return nil, "apiserver.token should set to non-empty string when service.schema is https" end return apiserver @@ -304,7 +304,7 @@ local _M = { function _M.nodes(service_name) local pattern = "^(.*):(.*)$" -- namespace/name:port_name - local match, _ = ngx.re.match(service_name, pattern, "jo") + local match = ngx.re.match(service_name, pattern, "jo") if not match then core.log.info("get unexpected upstream service_name:　", service_name) return nil From 5e85c1d509b821e86d6159492c8e9ea6b74e9d8e Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 14 Feb 2022 17:54:30 +0800 Subject: [PATCH 71/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/informer_factory.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index cd1391afa865..0c13a2b38178 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -88,7 +88,7 @@ local function list(httpc, apiserver, informer) informer.continue = data.metadata.continue if informer.continue and informer.continue ~= "" then - list(httpc, informer) + list(httpc, apiserver, informer) end return true From b2940abdef4c90f6a08a8dba901689dfe0771416 Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Mon, 14 Feb 2022 18:05:08 +0800 Subject: [PATCH 72/73] feat: add kubernetes discovery module --- apisix/discovery/kubernetes/informer_factory.lua | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 0c13a2b38178..8b50fc338d8d 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -160,7 +160,7 @@ end local function dispatch_event(event_string, informer) - local event, _ = core.json.decode(event_string) + local event = core.json.decode(event_string) if not event or not event.type or not event.object then return false, "UnexpectedBody", event_string @@ -246,9 +246,8 @@ local function watch(httpc, apiserver, informer) body = remainder_body .. body end - local ok ok, remainder_body, reason, err = split_event(body, dispatch_event, informer) - if not ok and reason ~= "Success" then + if not ok then if reason == "ResourceGone" then return true end From 22766936beae21de74fc4fb99aaa394b7345494d Mon Sep 17 00:00:00 2001 From: zhixiongdu Date: Sat, 19 Feb 2022 11:17:04 +0800 Subject: [PATCH 73/73] docs: improve kubernetes.md --- docs/zh/latest/discovery/kubernetes.md | 60 ++++++++++++++++---------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md index e45d1a0e9566..9ef14ffdce82 100644 --- a/docs/zh/latest/discovery/kubernetes.md +++ b/docs/zh/latest/discovery/kubernetes.md @@ -19,38 +19,52 @@ # 基于 Kubernetes 的服务发现 -Kubernetes 服务发现插件以 ListWatch 方式监听 Kubernetes 集群的 的 v1.endpoints 的实时变化, -并将其值存储在 ngx.shared.DICT 中, 同时遵循 APISix Discovery 规范提供对外查询接口 +Kubernetes 服务发现插件以 ListWatch 方式监听 Kubernetes 集群 v1.endpoints 的实时变化, +并将其值存储在 ngx.shared.dict 中, 同时遵循 APISIX Discovery 规范提供查询接口 # Kubernetes 服务发现插件的配置 -Kubernetes 服务发现的样例配置如下: +Kubernetes 服务发现插件的样例配置如下: ```yaml discovery: kubernetes: service: - # kubernetes apiserver schema, options [ http | https ] + # apiserver schema, options [http, https] schema: https #default https - # kubernetes apiserver host, options [ ipv4 | ipv6 | domain | env variable] - host: 10.0.8.95 #default ${KUBERNETES_SERVICE_HOST} + # apiserver host, options [ipv4, ipv6, domain, environment variable] + host: ${KUBERNETES_SERVICE_HOST} #default ${KUBERNETES_SERVICE_HOST} - # kubernetes apiserver port, you can enter port number or environment variable - port: 6443 #default ${KUBERNETES_SERVICE_PORT} + # apiserver port, options [port number, environment variable] + port: ${KUBERNETES_SERVICE_PORT} #default ${KUBERNETES_SERVICE_PORT} client: - # kubernetes serviceaccount token or token_file - token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" - #token: + # serviceaccount token or token_file + token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + #token: |- + # eyJhbGciOiJSUzI1NiIsImtpZCI6Ikx5ME1DNWdnbmhQNkZCNlZYMXBsT3pYU3BBS2swYzBPSkN3ZnBESGpkUEEif + # 6Ikx5ME1DNWdnbmhQNkZCNlZYMXBsT3pYU3BBS2swYzBPSkN3ZnBESGpkUEEifeyJhbGciOiJSUzI1NiIsImtpZCI # kubernetes discovery plugin support use namespace_selector - # you can use one of [ equal | not_equal | match | not_match ] filter namespace + # you can use one of [equal, not_equal, match, not_match] filter namespace namespace_selector: + # only save endpoints with namespace equal default equal: default - #not_equal: + + # only save endpoints with namespace not equal default + #not_equal: default + + # only save endpoints with namespace match one of [default, ^my-[a-z]+$] #match: + #- default + #- ^my-[a-z]+$ + + # only save endpoints with namespace not match one of [default, ^my-[a-z]+$] #not_match: + #- default + #- ^my-[a-z]+$ # kubernetes discovery plugin support use label_selector # for the expression of label_selector, please refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels @@ -71,12 +85,12 @@ discovery: discovery: kubernetes: service: - host: # Enter ApiServer Host Value Here - port: # Enter ApiServer Port Value Here schema: https + host: # enter apiserver host value here + port: # enter apiServer port value here client: - token: # Enter ServiceAccount Token Value Here - #token_file: # Enter File Path Here + token: # enter serviceaccount token value here + #token_file: # enter file path here ``` # Kubernetes 服务发现插件的使用 @@ -89,18 +103,18 @@ service_name 的 pattern 如下: # Q&A -> Q: 为什么只支持配置 token 来访问 kubernetes apiserver \ -> A: 通常情况下,我们会使用三种方式与 kubernetes apiserver 通信 : +> Q: 为什么只支持配置 token 来访问 Kubernetes ApiServer \ +> A: 通常情况下,我们会使用三种方式与 Kubernetes ApiServer 通信 : > >+ mTLS >+ token >+ basic authentication > -> 因为 lua-resty-http 目前不支持 mTLS ,以及 basic authentication 不被推荐使用,\ +> 因为 lua-resty-http 目前不支持 mTLS, 以及 basic authentication 不被推荐使用,\ > 所以当前只实现了 token 认证方式 ------- -> Q: APISix 是多进程模型, 是否意味着每个 APISix 业务进程都会去监听 kubernetes apiserver \ -> A: Kubernetes 服务发现插件只使用特权进程监听 kubernetes 集群,然后将结果存储在 ngx.shared.DICT中, \ -> 业务进程是通过查询 ngx.shared.DICT 获取结果的 +> Q: APISIX 是多进程模型, 是否意味着每个 APISIX 工作进程都会监听 Kubernetes v1.endpoints \ +> A: Kubernetes 服务发现插件只使用特权进程监听 Kubernetes v1.endpoints, 然后将结果存储\ +> 在 ngx.shared.dict 中, 业务进程是通过查询 ngx.shared.dict 来获取结果的