Skip to content

Commit

Permalink
feat(gRPC): support more features (#8485)
Browse files Browse the repository at this point in the history
  • Loading branch information
spacewander authored Dec 20, 2022
1 parent e66b694 commit 24d8e7c
Show file tree
Hide file tree
Showing 11 changed files with 793 additions and 13 deletions.
12 changes: 8 additions & 4 deletions apisix/cli/snippet.lua
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,11 @@ lua_ssl_trusted_certificate {* trusted_ca_cert *};
server {
{% if control_plane then %}
{% if directive_prefix == "grpc" then %}
listen {* control_plane.listen *} ssl http2;
{% else %}
listen {* control_plane.listen *} ssl;
{% end %}
ssl_certificate {* control_plane.cert *};
ssl_certificate_key {* control_plane.cert_key *};
Expand All @@ -45,9 +49,13 @@ server {
ssl_client_certificate {* control_plane.client_ca_cert *};
{% end %}
{% else %}
{% if directive_prefix == "grpc" then %}
listen unix:{* home *}/conf/config_listen.sock http2;
{% else %}
listen unix:{* home *}/conf/config_listen.sock;
{% end %}
{% end %}
access_log off;
Expand Down Expand Up @@ -104,10 +112,6 @@ server {

local function is_grpc_used(env, etcd)
local is_grpc_available = env.use_apisix_base
if etcd.user then
-- TODO: support user/password
is_grpc_available = false
end
return is_grpc_available and etcd.use_grpc
end

Expand Down
10 changes: 5 additions & 5 deletions apisix/core/etcd.lua
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,7 @@ local function _new(etcd_conf)
end

if etcd_conf.use_grpc then
-- TODO: let lua-resty-etcd support more use cases
if etcd.user or ngx_get_phase() == "init" then
if ngx_get_phase() == "init" then
etcd_conf.use_grpc = false
else
local ok = pcall(require, "resty.grpc")
Expand Down Expand Up @@ -125,9 +124,6 @@ local function new()
end

local etcd_conf = clone_tab(local_conf.etcd)
if etcd_conf.use_grpc then
return new_without_proxy()
end

if local_conf.apisix.ssl and local_conf.apisix.ssl.ssl_trusted_certificate then
etcd_conf.trusted_ca = local_conf.apisix.ssl.ssl_trusted_certificate
Expand Down Expand Up @@ -203,6 +199,10 @@ _M.new = new


local function switch_proxy()
if ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then
return new_without_proxy()
end

local etcd_cli, prefix, err = new()
if not etcd_cli or err then
return etcd_cli, prefix, err
Expand Down
2 changes: 1 addition & 1 deletion rockspec/apisix-master-0.rockspec
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ dependencies = {
"lua-resty-ctxdump = 0.1-0",
"api7-lua-resty-dns-client = 7.0.1",
"lua-resty-template = 2.0",
"lua-resty-etcd = 1.10.1",
"lua-resty-etcd = 1.10.2",
"api7-lua-resty-http = 0.2.0",
"lua-resty-balancer = 0.04",
"lua-resty-ngxvar = 0.5.2",
Expand Down
16 changes: 15 additions & 1 deletion t/bin/gen_snippet.lua
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ package.path = pkg_path .. pkg_path_org
local file = require("apisix.cli.file")
local schema = require("apisix.cli.schema")
local snippet = require("apisix.cli.snippet")
local util = require("apisix.cli.util")
local yaml_conf, err = file.read_yaml_conf("t/servroot")
if not yaml_conf then
error(err)
Expand All @@ -45,10 +46,23 @@ if not ok then
error(err)
end

local or_info, err = util.execute_cmd("openresty -V 2>&1")
if not or_info then
error("failed to exec cmd \'openresty -V 2>&1\', err: " .. err)
end

local use_apisix_base = true
if not or_info:find("apisix-nginx-module", 1, true) then
use_apisix_base = false
end

local res, err
if arg[1] == "conf_server" then
res, err = snippet.generate_conf_server(
{apisix_home = "t/servroot/"},
{
apisix_home = "t/servroot/",
use_apisix_base = use_apisix_base,
},
yaml_conf)
end

Expand Down
70 changes: 70 additions & 0 deletions t/cli/test_etcd_grpc.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# 'make init' operates scripts and related configuration files in the current directory
# The 'apisix' command is a command in the /usr/local/apisix,
# and the configuration file for the operation is in the /usr/local/apisix/conf

. ./t/cli/common.sh

exit_if_not_customed_nginx

# check etcd while enable auth
git checkout conf/config.yaml

export ETCDCTL_API=3
etcdctl version
etcdctl --endpoints=127.0.0.1:2379 user add "root:apache-api6"
etcdctl --endpoints=127.0.0.1:2379 role add root
etcdctl --endpoints=127.0.0.1:2379 user grant-role root root
etcdctl --endpoints=127.0.0.1:2379 user get root
etcdctl --endpoints=127.0.0.1:2379 auth enable
etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6 del /apisix --prefix

echo '
deployment:
role: traditional
role_traditional:
config_provider: etcd
etcd:
host:
- http://127.0.0.1:2379
prefix: /apisix
timeout: 30
use_grpc: true
user: root
password: apache-api6
' > conf/config.yaml

make run
sleep 1

code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9180/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1')
make stop

if [ ! $code -eq 200 ]; then
echo "failed: could not work with etcd"
exit 1
fi

echo "passed: work well with etcd auth enabled"

etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6 auth disable
etcdctl --endpoints=127.0.0.1:2379 role delete root
etcdctl --endpoints=127.0.0.1:2379 user delete root
153 changes: 153 additions & 0 deletions t/cli/test_etcd_grpc_healthcheck.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# 'make init' operates scripts and related configuration files in the current directory
# The 'apisix' command is a command in the /usr/local/apisix,
# and the configuration file for the operation is in the /usr/local/apisix/conf

. ./t/cli/common.sh

exit_if_not_customed_nginx

custom_clean_up() {
clean_up

# stop etcd docker container
docker-compose -f ./t/cli/docker-compose-etcd-cluster.yaml down
}

trap custom_clean_up EXIT

# create 3 node etcd cluster in docker
ETCD_NAME_0=etcd0
ETCD_NAME_1=etcd1
ETCD_NAME_2=etcd2
HEALTH_CHECK_RETRY_TIMEOUT=10

if [ -f "logs/error.log" ]; then
rm logs/error.log
fi
touch logs/error.log

echo '
deployment:
role: traditional
role_traditional:
config_provider: etcd
etcd:
use_grpc: true
host:
- "http://0.0.0.0:23790"
- "http://0.0.0.0:23791"
- "http://0.0.0.0:23792"
health_check_timeout: '"$HEALTH_CHECK_RETRY_TIMEOUT"'
timeout: 2
' > conf/config.yaml

docker-compose -f ./t/cli/docker-compose-etcd-cluster.yaml up -d

# case 1: Check apisix not got effected when one etcd node disconnected
make init && make run

docker stop ${ETCD_NAME_0}
code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9180/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1')
if [ ! $code -eq 200 ]; then
echo "failed: apisix got effect when one etcd node out of a cluster disconnected"
exit 1
fi
docker start ${ETCD_NAME_0}

docker stop ${ETCD_NAME_1}
code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9180/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1')
if [ ! $code -eq 200 ]; then
echo "failed: apisix got effect when one etcd node out of a cluster disconnected"
exit 1
fi
docker start ${ETCD_NAME_1}

make stop

echo "passed: apisix not got effected when one etcd node disconnected"

# case 2: Check when all etcd nodes disconnected, apisix trying to reconnect with backoff, and could successfully recover when reconnected
make init && make run

docker stop ${ETCD_NAME_0} && docker stop ${ETCD_NAME_1} && docker stop ${ETCD_NAME_2}

sleep_till=$(date +%s -d "$DATE + $HEALTH_CHECK_RETRY_TIMEOUT second")

code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9180/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1')
if [ $code -eq 200 ]; then
echo "failed: apisix not got effect when all etcd nodes disconnected"
exit 1
fi

docker start ${ETCD_NAME_0} && docker start ${ETCD_NAME_1} && docker start ${ETCD_NAME_2}

# case 3: sleep till etcd health check try to check again
current_time=$(date +%s)
sleep_seconds=$(( $sleep_till - $current_time + 3))
if [ "$sleep_seconds" -gt 0 ]; then
sleep $sleep_seconds
fi

code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9180/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1')
if [ ! $code -eq 200 ]; then
echo "failed: apisix could not recover when etcd node recover"
docker ps
cat logs/error.log
exit 1
fi

make stop

echo "passed: when all etcd nodes disconnected, apisix trying to reconnect with backoff, and could successfully recover when reconnected"

# case 4: stop one etcd node (result: start successful)
docker stop ${ETCD_NAME_0}

out=$(make init 2>&1)
if echo "$out" | grep "23790" | grep "connection refused"; then
echo "passed: APISIX successfully to start, stop only one etcd node"
else
echo "failed: stop only one etcd node APISIX should start normally"
exit 1
fi

# case 5: stop two etcd nodes (result: start failure)
docker stop ${ETCD_NAME_1}

out=$(make init 2>&1 || true)
if echo "$out" | grep "23791" | grep "connection refused"; then
echo "passed: APISIX failed to start, etcd cluster must have two or more healthy nodes"
else
echo "failed: two etcd nodes have been stopped, APISIX should fail to start"
exit 1
fi

# case 6: stop all etcd nodes (result: start failure)
docker stop ${ETCD_NAME_2}

out=$(make init 2>&1 || true)
if echo "$out" | grep "23792" | grep "connection refused"; then
echo "passed: APISIX failed to start, all etcd nodes have stopped"
else
echo "failed: all etcd nodes have stopped, APISIX should not be able to start"
exit 1
fi
Loading

0 comments on commit 24d8e7c

Please sign in to comment.