From 51913745453333ecd570bbce483c28f0ca1f4b54 Mon Sep 17 00:00:00 2001 From: Alex Zhang Date: Tue, 3 Nov 2020 13:53:39 +0800 Subject: [PATCH] feat: support TLS connection with etcd. (#2548) Support the TLS connection when communicating with etcd cluster. We added a configuration item to custom the certificate verification. Whether to setup TLS connection or not depends on the endpoints' scheme, for instance, when endpoints are: ``` etcd: host: - "https://127.0.0.1:2379" - "https://127.0.0.1:3379" ``` APISIX will originate TLS connection automatically, and the Server Name Indication extention will be set by the endpoint host (`127.0.0.1` in above case). Note by default APISIX will verify the certificate, close the verification in configuration explicitly if you want to bypass it. ``` etcd: tls: verfiy: false ``` --- .github/workflows/build.yml | 15 ++- apisix/core/config_etcd.lua | 6 + apisix/core/etcd.lua | 6 + bin/apisix | 16 +++ conf/config-default.yaml | 10 +- doc/install-dependencies.md | 5 + doc/zh-cn/install-dependencies.md | 5 + rockspec/apisix-master-0.rockspec | 2 +- t/certs/etcd.key | 28 +++++ t/certs/etcd.pem | 22 ++++ t/core/config_etcd.t | 184 ++++++++++++++++++++++++++++++ 11 files changed, 294 insertions(+), 5 deletions(-) create mode 100644 t/certs/etcd.key create mode 100644 t/certs/etcd.pem diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5952a1b48795..f414fbbde09a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,6 @@ jobs: ALLOW_NONE_AUTHENTICATION: yes ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379 - steps: - name: Check out code uses: actions/checkout@v2 @@ -65,6 +64,20 @@ jobs: redis-cli -h 127.0.0.1 -p 5000 ping redis-cli -h 127.0.0.1 -p 5000 cluster nodes + - name: Running etcd server with TLS + if: matrix.os_name == 'linux_openresty' + run: | + sudo docker run -d -p 12379:12379 -p 12380:12380 \ + -e ALLOW_NONE_AUTHENTICATION=yes \ + -e ETCD_ADVERTISE_CLIENT_URLS=https://0.0.0.0:12379 \ + -e ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:12379 \ + -e ETCD_CERT_FILE=/certs/etcd.pem \ + -e ETCD_KEY_FILE=/certs/etcd.key \ + -e GITHUB_ACTIONS=true \ + -e CI=true \ + -v ${{ github.workspace }}/t/certs:/certs \ + bitnami/etcd:3.4.0 + - name: Linux Install run: sudo ./.travis/${{ matrix.os_name }}_runner.sh do_install diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua index bb71313691d2..6a8094fea101 100644 --- a/apisix/core/config_etcd.lua +++ b/apisix/core/config_etcd.lua @@ -519,6 +519,12 @@ function _M.new(key, opts) etcd_conf.prefix = nil etcd_conf.protocol = "v3" etcd_conf.api_prefix = "/v3" + etcd_conf.ssl_verify = true + + -- default to verify etcd cluster certificate + if etcd_conf.tls and etcd_conf.tls.verify == false then + etcd_conf.ssl_verify = false + end local automatic = opts and opts.automatic local item_schema = opts and opts.item_schema diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua index 715ff25b161d..c28eb58f1630 100644 --- a/apisix/core/etcd.lua +++ b/apisix/core/etcd.lua @@ -37,6 +37,12 @@ local function new() etcd_conf.prefix = nil etcd_conf.protocol = "v3" etcd_conf.api_prefix = "/v3" + etcd_conf.ssl_verify = true + + -- default to verify etcd cluster certificate + if etcd_conf.tls and etcd_conf.tls.verify == false then + etcd_conf.ssl_verify = false + end local etcd_cli etcd_cli, err = etcd.new(etcd_conf) diff --git a/bin/apisix b/bin/apisix index f714ed45fcb1..5cb8e39e4a7c 100755 --- a/bin/apisix +++ b/bin/apisix @@ -560,6 +560,22 @@ local function init_etcd(show_output) end local host_count = #(yaml_conf.etcd.host) + local scheme + for i = 1, host_count do + local host = yaml_conf.etcd.host[i] + local fields = split(host, "://") + if not fields then + io.stderr:write("malformed etcd endpoint: ", host, "\n") + os.exit(1) + end + + if not scheme then + scheme = fields[1] + elseif scheme ~= fields[1] then + print([[WARNING: mixed protocols among etcd endpoints]]) + end + end + local dkjson = require("dkjson") -- check the etcd cluster version diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 8ae4ebc7e26c..768ff081a0c6 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -144,11 +144,15 @@ nginx_config: # config for render the template to genarate n etcd: host: # it's possible to define multiple etcd hosts addresses of the same etcd cluster. - - "http://127.0.0.1:2379" # multiple etcd address + - "http://127.0.0.1:2379" # multiple etcd address, if your etcd cluster enables TLS, please use https scheme, + # e.g. "https://127.0.0.1:2379". prefix: "/apisix" # apisix configurations prefix timeout: 30 # 30 seconds - # user: root # root username for etcd - # password: 5tHkHhYkjr6cQY # root password for etcd + # user: root # root username for etcd + # password: 5tHkHhYkjr6cQY # root password for etcd + tls: + verify: true # whether to verify the etcd endpoint certificate when setup a TLS connection to etcd, + # the default value is true, e.g. the certificate will be verified strictly. # discovery: # service discovery center # eureka: diff --git a/doc/install-dependencies.md b/doc/install-dependencies.md index 827d698a54ef..122b0c803788 100644 --- a/doc/install-dependencies.md +++ b/doc/install-dependencies.md @@ -30,6 +30,8 @@ Note ==== - Since v2.0 Apache APISIX would not support the v2 protocol storage to etcd anymore, and the minimum etcd version supported is v3.4.0. +- Now by default Apache APISIX uses HTTP protocol to talk with etcd cluster, which is insecure. Please configure certificate and correspsonding private key for your etcd cluster, and use "https" scheme explicitly in the etcd endpoints list in your Apache APISIX configuration, if you want to keep the data secure and integral. See the etcd section in `conf/config-default.yaml` for more details. + - If you want use Tengine instead of OpenResty, please take a look at this installation step script [Install Tengine at Ubuntu](../.travis/linux_tengine_runner.sh). @@ -141,4 +143,7 @@ brew install openresty/brew/openresty etcd luarocks curl git # start etcd server etcd & + +# enable TLS for etcd server +etcd --cert-file=/path/to/cert --key-file=/path/to/pkey --advertise-client-urls https://127.0.0.1:2379 ``` diff --git a/doc/zh-cn/install-dependencies.md b/doc/zh-cn/install-dependencies.md index 8ef9370a892e..519065099d3d 100644 --- a/doc/zh-cn/install-dependencies.md +++ b/doc/zh-cn/install-dependencies.md @@ -29,6 +29,8 @@ ==== - Apache APISIX 从 v2.0 开始不再支持 `v2` 版本的 etcd，并且 etcd 最低支持版本为 v3.4.0。 +- 目前 Apache APISIX 默认使用 HTTP 协议与 etcd 集群通信，这并不安全，如果希望保障数据的安全性和完整性。 请为您的 etcd 集群配置证书及对应私钥，并在您的 Apache APISIX etcd endpoints 配置列表中明确使用 `https` 协议前缀。请查阅 `conf/config-default.yaml` 中 etcd 一节相关的配置来了解更多细节。 + - 如果你要想使用 Tengine 替代 OpenResty，请参考 [Install Tengine at Ubuntu](../../.travis/linux_tengine_runner.sh)。 @@ -140,4 +142,7 @@ brew install openresty/brew/openresty etcd luarocks curl git # 开启 etcd server etcd & + +# 为 etcd 服务启用 TLS +etcd --cert-file=/path/to/cert --key-file=/path/to/pkey --advertise-client-urls https://127.0.0.1:2379 ``` diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index aa85f5a152e0..7c9d4d06a852 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -32,7 +32,7 @@ description = { dependencies = { "lua-resty-template = 1.9", - "lua-resty-etcd = 1.2", + "lua-resty-etcd = 1.4.1", "lua-resty-balancer = 0.02rc5", "lua-resty-ngxvar = 0.5", "lua-resty-jit-uuid = 0.0.7", diff --git a/t/certs/etcd.key b/t/certs/etcd.key new file mode 100644 index 000000000000..c2bd6514fbbe --- /dev/null +++ b/t/certs/etcd.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCf6sMQke4OUrPf +lZXqKZC4uaQLMxJB2DgacBf4Q3kXcVNCNHtc2U3U5cMD8tr3/Jt5MKKun5jQrbXV +FF4eVr4Iv9jgPBwQc2kBUC9QL/alsMhEuXMeqdGQcCK3F0CLJdV3zUlKUDU0kg+O +Exnbl1CHXrIbpD7zLy1i3s8p39v1pYFYf4WlrQxvfa/xo97gXY5dJv8RryryLzRc +uhHYBvX5MHCGpbrY61JxpfZqBo8CmLuHl1tmbeXpdHdQB11LKiuL6HtKflNjc6rg +5r8bXl1nZbM/KOZEE+muA1LVoaTyHzY/aGXz0bNy4QRUO+De9JFcTDgnXnNZVG5x +cyyDBpc9AgMBAAECggEAatcEtehZPJaCeClPPF/Cwbe9YoIfe4BCk186lHI3z7K1 +5nB7zt+bwVY0AUpagv3wvXoB5lrYVOsJpa9y5iAb3GqYMc/XDCKfD/KLea5hwfcn +BctEn0LjsPVKLDrLs2t2gBDWG2EU+udunwQh7XTdp2Nb6V3FdOGbGAg2LgrSwP1g +0r4z14F70oWGYyTQ5N8UGuyryVrzQH525OYl38Yt7R6zJ/44FVi/2TvdfHM5ss39 +SXWi00Q30fzaBEf4AdHVwVCRKctwSbrIOyM53kiScFDmBGRblCWOxXbiFV+d3bjX +gf2zxs7QYZrFOzOO7kLtHGua4itEB02497v+1oKDwQKBgQDOBvCVGRe2WpItOLnj +SF8iz7Sm+jJGQz0D9FhWyGPvrN7IXGrsXavA1kKRz22dsU8xdKk0yciOB13Wb5y6 +yLsr/fPBjAhPb4h543VHFjpAQcxpsH51DE0b2oYOWMmz+rXGB5Jy8EkP7Q4njIsc +2wLod1dps8OT8zFx1jX3Us6iUQKBgQDGtKkfsvWi3HkwjFTR+/Y0oMz7bSruE5Z8 +g0VOHPkSr4XiYgLpQxjbNjq8fwsa/jTt1B57+By4xLpZYD0BTFuf5po+igSZhH8s +QS5XnUnbM7d6Xr/da7ZkhSmUbEaMeHONSIVpYNgtRo4bB9Mh0l1HWdoevw/w5Ryt +L/OQiPhfLQKBgQCh1iG1fPh7bbnVe/HI71iL58xoPbCwMLEFIjMiOFcINirqCG6V +LR91Ytj34JCihl1G4/TmWnsH1hGIGDRtJLCiZeHL70u32kzCMkI1jOhFAWqoutMa +7obDkmwraONIVW/kFp6bWtSJhhTQTD4adI9cPCKWDXdcCHSWj0Xk+U8HgQKBgBng +t1HYhaLzIZlP/U/nh3XtJyTrX7bnuCZ5FhKJNWrYjxAfgY+NXHRYCKg5x2F5j70V +be7pLhxmCnrPTMKZhik56AaTBOxVVBaYWoewhUjV4GRAaK5Wc8d9jB+3RizPFwVk +V3OU2DJ1SNZ+W2HBOsKrEfwFF/dgby6i2w6MuAP1AoGBAIxvxUygeT/6P0fHN22P +zAHFI4v2925wYdb7H//D8DIADyBwv18N6YH8uH7L+USZN7e4p2k8MGGyvTXeC6aX +IeVtU6fH57Ddn59VPbF20m8RCSkmBvSdcbyBmqlZSBE+fKwCliKl6u/GH0BNAWKz +r8yiEiskqRmy7P7MY9hDmEbG +-----END PRIVATE KEY----- diff --git a/t/certs/etcd.pem b/t/certs/etcd.pem new file mode 100644 index 000000000000..2f878bdff958 --- /dev/null +++ b/t/certs/etcd.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDrzCCApegAwIBAgIJAI3Meu/gJVTLMA0GCSqGSIb3DQEBCwUAMG4xCzAJBgNV +BAYTAkNOMREwDwYDVQQIDAhaaGVqaWFuZzERMA8GA1UEBwwISGFuZ3pob3UxDTAL +BgNVBAoMBHRlc3QxDTALBgNVBAsMBHRlc3QxGzAZBgNVBAMMEmV0Y2QuY2x1c3Rl +ci5sb2NhbDAeFw0yMDEwMjgwMzMzMDJaFw0yMTEwMjgwMzMzMDJaMG4xCzAJBgNV +BAYTAkNOMREwDwYDVQQIDAhaaGVqaWFuZzERMA8GA1UEBwwISGFuZ3pob3UxDTAL +BgNVBAoMBHRlc3QxDTALBgNVBAsMBHRlc3QxGzAZBgNVBAMMEmV0Y2QuY2x1c3Rl +ci5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ/qwxCR7g5S +s9+VleopkLi5pAszEkHYOBpwF/hDeRdxU0I0e1zZTdTlwwPy2vf8m3kwoq6fmNCt +tdUUXh5Wvgi/2OA8HBBzaQFQL1Av9qWwyES5cx6p0ZBwIrcXQIsl1XfNSUpQNTSS +D44TGduXUIdeshukPvMvLWLezynf2/WlgVh/haWtDG99r/Gj3uBdjl0m/xGvKvIv +NFy6EdgG9fkwcIalutjrUnGl9moGjwKYu4eXW2Zt5el0d1AHXUsqK4voe0p+U2Nz +quDmvxteXWdlsz8o5kQT6a4DUtWhpPIfNj9oZfPRs3LhBFQ74N70kVxMOCdec1lU +bnFzLIMGlz0CAwEAAaNQME4wHQYDVR0OBBYEFFHeljijrr+SPxlH5fjHRPcC7bv2 +MB8GA1UdIwQYMBaAFFHeljijrr+SPxlH5fjHRPcC7bv2MAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBAG6NNTK7sl9nJxeewVuogCdMtkcdnx9onGtCOeiQ +qvh5Xwn9akZtoLMVEdceU0ihO4wILlcom3OqHs9WOd6VbgW5a19Thh2toxKidHz5 +rAaBMyZsQbFb6+vFshZwoCtOLZI/eIZfUUMFqMXlEPrKru1nSddNdai2+zi5rEnM +HCot43+3XYuqkvWlOjoi9cP+C4epFYrxpykVbcrtbd7TK+wZNiK3xtDPnVzjdNWL +geAEl9xrrk0ss4nO/EreTQgS46gVU+tLC+b23m2dU7dcKZ7RDoiA9bdVc4a2IsaS +2MvLL4NZ2nUh8hAEHiLtGMAV3C6xNbEyM07hEpDW6vk6tqk= +-----END CERTIFICATE----- diff --git a/t/core/config_etcd.t b/t/core/config_etcd.t index 7944537d7435..ce158cf69a95 100644 --- a/t/core/config_etcd.t +++ b/t/core/config_etcd.t @@ -47,3 +47,187 @@ GET /t qr{failed to fetch data from etcd: connection refused, etcd key: .*routes} --- grep_error_log_out eval qr/(failed to fetch data from etcd: connection refused, etcd key: .*routes\n){1,}/ + + + +=== TEST 2: originate TLS connection to etcd cluster without TLS configuration +--- yaml_config +apisix: + node_listen: 1984 +etcd: + host: + - "https://127.0.0.1:2379" +--- config + location /t { + content_by_lua_block { + ngx.sleep(4) + ngx.say("ok") + } + } +--- timeout: 5 +--- request +GET /t +--- grep_error_log chop +failed to fetch data from etcd: handshake failed +--- grep_error_log_out eval +qr/(failed to fetch data from etcd: handshake failed){1,}/ + + + +=== TEST 3: originate plain connection to etcd cluster which enables TLS +--- yaml_config +apisix: + node_listen: 1984 +etcd: + host: + - "http://127.0.0.1:12379" +--- config + location /t { + content_by_lua_block { + ngx.sleep(4) + ngx.say("ok") + } + } +--- timeout: 5 +--- request +GET /t +--- grep_error_log chop +failed to fetch data from etcd: closed +--- grep_error_log_out eval +qr/(failed to fetch data from etcd: closed){1,}/ + + + +=== TEST 4: originate TLS connection to etcd cluster and verify TLS certificate (default behavior) +--- yaml_config +apisix: + node_listen: 1984 +etcd: + host: + - "https://127.0.0.1:12379" +--- config + location /t { + content_by_lua_block { + ngx.sleep(4) + ngx.say("ok") + } + } +--- timeout: 5 +--- request +GET /t +--- grep_error_log chop +failed to fetch data from etcd: 18: self signed certificate +--- grep_error_log_out eval +qr/(failed to fetch data from etcd: 18: self signed certificate){1,}/ + + + +=== TEST 5: set route(id: 1) to etcd cluster with TLS +--- yaml_config +apisix: + node_listen: 1984 + admin_key: null +etcd: + host: + - "https://127.0.0.1:12379" + tls: + verify: false +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "methods": ["GET"], + "upstream": { + "nodes": { + "127.0.0.1:8080": 1 + }, + "type": "roundrobin" + }, + "desc": "new route", + "uri": "/index.html" + }]], + [[{ + "node": { + "value": { + "methods": [ + "GET" + ], + "uri": "/index.html", + "desc": "new route", + "upstream": { + "nodes": { + "127.0.0.1:8080": 1 + }, + "type": "roundrobin" + } + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + + ngx.status = code + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 6: get route(id: 1) from etcd cluster with TLS +--- yaml_config +apisix: + node_listen: 1984 + admin_key: null +etcd: + host: + - "https://127.0.0.1:12379" + tls: + verify: false +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_GET, + nil, + [[{ + "node": { + "value": { + "methods": [ + "GET" + ], + "uri": "/index.html", + "desc": "new route", + "upstream": { + "nodes": { + "127.0.0.1:8080": 1 + }, + "type": "roundrobin" + } + }, + "key": "/apisix/routes/1" + }, + "action": "get" + }]] + ) + + ngx.status = code + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error]