feat: support TLS connection with etcd. (#2548)

Support the TLS connection when communicating with etcd cluster. We added a configuration item to custom the certificate verification. Whether to setup TLS connection or not depends on the endpoints' scheme, for instance, when endpoints are:

```
etcd:
  host:
    - "https://127.0.0.1:2379"
    - "https://127.0.0.1:3379"
```

APISIX will originate TLS connection automatically, and the Server Name Indication extention will be set by the endpoint host (`127.0.0.1` in above case). Note by default APISIX will verify the certificate, close the verification in configuration explicitly if you want to bypass it.

```
etcd:
  tls:
    verfiy: false
```

.github/workflows/build.yml

@@ -35,7 +35,6 @@ jobs:
           ALLOW_NONE_AUTHENTICATION: yes
           ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
 
-
     steps:
       - name: Check out code
         uses: actions/checkout@v2
@@ -65,6 +64,20 @@ jobs:
           redis-cli -h 127.0.0.1 -p 5000 ping
           redis-cli -h 127.0.0.1 -p 5000 cluster nodes
 
+      - name: Running etcd server with TLS
+        if: matrix.os_name == 'linux_openresty'
+        run: |
+            sudo docker run -d -p 12379:12379 -p 12380:12380 \
+            -e ALLOW_NONE_AUTHENTICATION=yes \
+            -e ETCD_ADVERTISE_CLIENT_URLS=https://0.0.0.0:12379 \
+            -e ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:12379 \
+            -e ETCD_CERT_FILE=/certs/etcd.pem \
+            -e ETCD_KEY_FILE=/certs/etcd.key \
+            -e GITHUB_ACTIONS=true \
+            -e CI=true \
+            -v ${{ github.workspace }}/t/certs:/certs \
+            bitnami/etcd:3.4.0
+
       - name: Linux Install
         run: sudo ./.travis/${{ matrix.os_name }}_runner.sh do_install
 

apisix/core/config_etcd.lua

@@ -519,6 +519,12 @@ function _M.new(key, opts)
     etcd_conf.prefix = nil
     etcd_conf.protocol = "v3"
     etcd_conf.api_prefix = "/v3"
+    etcd_conf.ssl_verify = true
+
+    -- default to verify etcd cluster certificate
+    if etcd_conf.tls and etcd_conf.tls.verify == false then
+        etcd_conf.ssl_verify = false
+    end
 
     local automatic = opts and opts.automatic
     local item_schema = opts and opts.item_schema

apisix/core/etcd.lua

@@ -37,6 +37,12 @@ local function new()
     etcd_conf.prefix = nil
     etcd_conf.protocol = "v3"
     etcd_conf.api_prefix = "/v3"
+    etcd_conf.ssl_verify = true
+
+    -- default to verify etcd cluster certificate
+    if etcd_conf.tls and etcd_conf.tls.verify == false then
+        etcd_conf.ssl_verify = false
+    end
 
     local etcd_cli
     etcd_cli, err = etcd.new(etcd_conf)

bin/apisix

@@ -560,6 +560,22 @@ local function init_etcd(show_output)
     end
 
     local host_count = #(yaml_conf.etcd.host)
+    local scheme
+    for i = 1, host_count do
+        local host = yaml_conf.etcd.host[i]
+        local fields = split(host, "://")
+        if not fields then
+            io.stderr:write("malformed etcd endpoint: ", host, "\n")
+            os.exit(1)
+        end
+
+        if not scheme then
+            scheme = fields[1]
+        elseif scheme ~= fields[1] then
+            print([[WARNING: mixed protocols among etcd endpoints]])
+        end
+    end
+
     local dkjson = require("dkjson")
 
     -- check the etcd cluster version

conf/config-default.yaml

@@ -144,11 +144,15 @@ nginx_config:                     # config for render the template to genarate n
 
 etcd:
   host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
-    - "http://127.0.0.1:2379"     # multiple etcd address
+    - "http://127.0.0.1:2379"     # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
+                                  # e.g. "https://127.0.0.1:2379".
   prefix: "/apisix"               # apisix configurations prefix
   timeout: 30                     # 30 seconds
-  # user: root                     # root username for etcd
-  # password: 5tHkHhYkjr6cQY        # root password for etcd
+  # user: root                    # root username for etcd
+  # password: 5tHkHhYkjr6cQY      # root password for etcd
+  tls:
+      verify: true                # whether to verify the etcd endpoint certificate when setup a TLS connection to etcd,
+                                  # the default value is true, e.g. the certificate will be verified strictly.
 
 # discovery:                          # service discovery center
 #   eureka:

doc/install-dependencies.md

@@ -30,6 +30,8 @@ Note
 ====
 - Since v2.0 Apache APISIX would not support the v2 protocol storage to etcd anymore, and the minimum etcd version supported is v3.4.0.
 
+- Now by default Apache APISIX uses HTTP protocol to talk with etcd cluster, which is insecure. Please configure certificate and correspsonding private key for your etcd cluster, and use "https" scheme explicitly in the etcd endpoints list in your Apache APISIX configuration, if you want to keep the data secure and integral. See the etcd section in `conf/config-default.yaml` for more details.
+
 - If you want use Tengine instead of OpenResty, please take a look at this installation step script [Install Tengine at Ubuntu](../.travis/linux_tengine_runner.sh).
 
 
@@ -141,4 +143,7 @@ brew install openresty/brew/openresty etcd luarocks curl git
 
 # start etcd server
 etcd &
+
+# enable TLS for etcd server
+etcd --cert-file=/path/to/cert --key-file=/path/to/pkey --advertise-client-urls https://127.0.0.1:2379
 ```

doc/zh-cn/install-dependencies.md

@@ -29,6 +29,8 @@
 ====
 - Apache APISIX 从 v2.0 开始不再支持 `v2` 版本的 etcd，并且 etcd 最低支持版本为 v3.4.0。
 
+- 目前 Apache APISIX 默认使用 HTTP 协议与 etcd 集群通信，这并不安全，如果希望保障数据的安全性和完整性。 请为您的 etcd 集群配置证书及对应私钥，并在您的 Apache APISIX etcd endpoints 配置列表中明确使用 `https` 协议前缀。请查阅 `conf/config-default.yaml` 中 etcd 一节相关的配置来了解更多细节。
+
 - 如果你要想使用 Tengine 替代 OpenResty，请参考 [Install Tengine at Ubuntu](../../.travis/linux_tengine_runner.sh)。
 
 
@@ -140,4 +142,7 @@ brew install openresty/brew/openresty etcd luarocks curl git
 
 # 开启 etcd server
 etcd &
+
+# 为 etcd 服务启用 TLS
+etcd --cert-file=/path/to/cert --key-file=/path/to/pkey --advertise-client-urls https://127.0.0.1:2379
 ```

rockspec/apisix-master-0.rockspec

@@ -32,7 +32,7 @@ description = {
 
 dependencies = {
     "lua-resty-template = 1.9",
-    "lua-resty-etcd = 1.2",
+    "lua-resty-etcd = 1.4.1",
     "lua-resty-balancer = 0.02rc5",
     "lua-resty-ngxvar = 0.5",
     "lua-resty-jit-uuid = 0.0.7",

t/certs/etcd.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCf6sMQke4OUrPf
+lZXqKZC4uaQLMxJB2DgacBf4Q3kXcVNCNHtc2U3U5cMD8tr3/Jt5MKKun5jQrbXV
+FF4eVr4Iv9jgPBwQc2kBUC9QL/alsMhEuXMeqdGQcCK3F0CLJdV3zUlKUDU0kg+O
+Exnbl1CHXrIbpD7zLy1i3s8p39v1pYFYf4WlrQxvfa/xo97gXY5dJv8RryryLzRc
+uhHYBvX5MHCGpbrY61JxpfZqBo8CmLuHl1tmbeXpdHdQB11LKiuL6HtKflNjc6rg
+5r8bXl1nZbM/KOZEE+muA1LVoaTyHzY/aGXz0bNy4QRUO+De9JFcTDgnXnNZVG5x
+cyyDBpc9AgMBAAECggEAatcEtehZPJaCeClPPF/Cwbe9YoIfe4BCk186lHI3z7K1
+5nB7zt+bwVY0AUpagv3wvXoB5lrYVOsJpa9y5iAb3GqYMc/XDCKfD/KLea5hwfcn
+BctEn0LjsPVKLDrLs2t2gBDWG2EU+udunwQh7XTdp2Nb6V3FdOGbGAg2LgrSwP1g
+0r4z14F70oWGYyTQ5N8UGuyryVrzQH525OYl38Yt7R6zJ/44FVi/2TvdfHM5ss39
+SXWi00Q30fzaBEf4AdHVwVCRKctwSbrIOyM53kiScFDmBGRblCWOxXbiFV+d3bjX
+gf2zxs7QYZrFOzOO7kLtHGua4itEB02497v+1oKDwQKBgQDOBvCVGRe2WpItOLnj
+SF8iz7Sm+jJGQz0D9FhWyGPvrN7IXGrsXavA1kKRz22dsU8xdKk0yciOB13Wb5y6
+yLsr/fPBjAhPb4h543VHFjpAQcxpsH51DE0b2oYOWMmz+rXGB5Jy8EkP7Q4njIsc
+2wLod1dps8OT8zFx1jX3Us6iUQKBgQDGtKkfsvWi3HkwjFTR+/Y0oMz7bSruE5Z8
+g0VOHPkSr4XiYgLpQxjbNjq8fwsa/jTt1B57+By4xLpZYD0BTFuf5po+igSZhH8s
+QS5XnUnbM7d6Xr/da7ZkhSmUbEaMeHONSIVpYNgtRo4bB9Mh0l1HWdoevw/w5Ryt
+L/OQiPhfLQKBgQCh1iG1fPh7bbnVe/HI71iL58xoPbCwMLEFIjMiOFcINirqCG6V
+LR91Ytj34JCihl1G4/TmWnsH1hGIGDRtJLCiZeHL70u32kzCMkI1jOhFAWqoutMa
+7obDkmwraONIVW/kFp6bWtSJhhTQTD4adI9cPCKWDXdcCHSWj0Xk+U8HgQKBgBng
+t1HYhaLzIZlP/U/nh3XtJyTrX7bnuCZ5FhKJNWrYjxAfgY+NXHRYCKg5x2F5j70V
+be7pLhxmCnrPTMKZhik56AaTBOxVVBaYWoewhUjV4GRAaK5Wc8d9jB+3RizPFwVk
+V3OU2DJ1SNZ+W2HBOsKrEfwFF/dgby6i2w6MuAP1AoGBAIxvxUygeT/6P0fHN22P
+zAHFI4v2925wYdb7H//D8DIADyBwv18N6YH8uH7L+USZN7e4p2k8MGGyvTXeC6aX
+IeVtU6fH57Ddn59VPbF20m8RCSkmBvSdcbyBmqlZSBE+fKwCliKl6u/GH0BNAWKz
+r8yiEiskqRmy7P7MY9hDmEbG
+-----END PRIVATE KEY-----

t/certs/etcd.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDrzCCApegAwIBAgIJAI3Meu/gJVTLMA0GCSqGSIb3DQEBCwUAMG4xCzAJBgNV
+BAYTAkNOMREwDwYDVQQIDAhaaGVqaWFuZzERMA8GA1UEBwwISGFuZ3pob3UxDTAL
+BgNVBAoMBHRlc3QxDTALBgNVBAsMBHRlc3QxGzAZBgNVBAMMEmV0Y2QuY2x1c3Rl
+ci5sb2NhbDAeFw0yMDEwMjgwMzMzMDJaFw0yMTEwMjgwMzMzMDJaMG4xCzAJBgNV
+BAYTAkNOMREwDwYDVQQIDAhaaGVqaWFuZzERMA8GA1UEBwwISGFuZ3pob3UxDTAL
+BgNVBAoMBHRlc3QxDTALBgNVBAsMBHRlc3QxGzAZBgNVBAMMEmV0Y2QuY2x1c3Rl
+ci5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ/qwxCR7g5S
+s9+VleopkLi5pAszEkHYOBpwF/hDeRdxU0I0e1zZTdTlwwPy2vf8m3kwoq6fmNCt
+tdUUXh5Wvgi/2OA8HBBzaQFQL1Av9qWwyES5cx6p0ZBwIrcXQIsl1XfNSUpQNTSS
+D44TGduXUIdeshukPvMvLWLezynf2/WlgVh/haWtDG99r/Gj3uBdjl0m/xGvKvIv
+NFy6EdgG9fkwcIalutjrUnGl9moGjwKYu4eXW2Zt5el0d1AHXUsqK4voe0p+U2Nz
+quDmvxteXWdlsz8o5kQT6a4DUtWhpPIfNj9oZfPRs3LhBFQ74N70kVxMOCdec1lU
+bnFzLIMGlz0CAwEAAaNQME4wHQYDVR0OBBYEFFHeljijrr+SPxlH5fjHRPcC7bv2
+MB8GA1UdIwQYMBaAFFHeljijrr+SPxlH5fjHRPcC7bv2MAwGA1UdEwQFMAMBAf8w
+DQYJKoZIhvcNAQELBQADggEBAG6NNTK7sl9nJxeewVuogCdMtkcdnx9onGtCOeiQ
+qvh5Xwn9akZtoLMVEdceU0ihO4wILlcom3OqHs9WOd6VbgW5a19Thh2toxKidHz5
+rAaBMyZsQbFb6+vFshZwoCtOLZI/eIZfUUMFqMXlEPrKru1nSddNdai2+zi5rEnM
+HCot43+3XYuqkvWlOjoi9cP+C4epFYrxpykVbcrtbd7TK+wZNiK3xtDPnVzjdNWL
+geAEl9xrrk0ss4nO/EreTQgS46gVU+tLC+b23m2dU7dcKZ7RDoiA9bdVc4a2IsaS
+2MvLL4NZ2nUh8hAEHiLtGMAV3C6xNbEyM07hEpDW6vk6tqk=
+-----END CERTIFICATE-----

t/core/config_etcd.t

@@ -47,3 +47,187 @@ GET /t
 qr{failed to fetch data from etcd: connection refused,  etcd key: .*routes}
 --- grep_error_log_out eval
 qr/(failed to fetch data from etcd: connection refused,  etcd key: .*routes\n){1,}/
+
+
+
+=== TEST 2: originate TLS connection to etcd cluster without TLS configuration
+--- yaml_config
+apisix:
+  node_listen: 1984
+etcd:
+  host:
+    - "https://127.0.0.1:2379"
+--- config
+    location /t {
+        content_by_lua_block {
+            ngx.sleep(4)
+            ngx.say("ok")
+        }
+    }
+--- timeout: 5
+--- request
+GET /t
+--- grep_error_log chop
+failed to fetch data from etcd: handshake failed
+--- grep_error_log_out eval
+qr/(failed to fetch data from etcd: handshake failed){1,}/
+
+
+
+=== TEST 3: originate plain connection to etcd cluster which enables TLS
+--- yaml_config
+apisix:
+  node_listen: 1984
+etcd:
+  host:
+    - "http://127.0.0.1:12379"
+--- config
+    location /t {
+        content_by_lua_block {
+            ngx.sleep(4)
+            ngx.say("ok")
+        }
+    }
+--- timeout: 5
+--- request
+GET /t
+--- grep_error_log chop
+failed to fetch data from etcd: closed
+--- grep_error_log_out eval
+qr/(failed to fetch data from etcd: closed){1,}/
+
+
+
+=== TEST 4: originate TLS connection to etcd cluster and verify TLS certificate (default behavior)
+--- yaml_config
+apisix:
+  node_listen: 1984
+etcd:
+  host:
+    - "https://127.0.0.1:12379"
+--- config
+    location /t {
+        content_by_lua_block {
+            ngx.sleep(4)
+            ngx.say("ok")
+        }
+    }
+--- timeout: 5
+--- request
+GET /t
+--- grep_error_log chop
+failed to fetch data from etcd: 18: self signed certificate
+--- grep_error_log_out eval
+qr/(failed to fetch data from etcd: 18: self signed certificate){1,}/
+
+
+
+=== TEST 5: set route(id: 1) to etcd cluster with TLS
+--- yaml_config
+apisix:
+  node_listen: 1984
+  admin_key: null
+etcd:
+  host:
+    - "https://127.0.0.1:12379"
+  tls:
+    verify: false
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "methods": ["GET"],
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:8080": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "desc": "new route",
+                    "uri": "/index.html"
+                }]],
+                [[{
+                    "node": {
+                        "value": {
+                            "methods": [
+                                "GET"
+                            ],
+                            "uri": "/index.html",
+                            "desc": "new route",
+                            "upstream": {
+                                "nodes": {
+                                    "127.0.0.1:8080": 1
+                                },
+                                "type": "roundrobin"
+                            }
+                        },
+                        "key": "/apisix/routes/1"
+                    },
+                    "action": "set"
+                }]]
+                )
+
+            ngx.status = code
+            ngx.say(body)
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 6: get route(id: 1) from etcd cluster with TLS
+--- yaml_config
+apisix:
+  node_listen: 1984
+  admin_key: null
+etcd:
+  host:
+    - "https://127.0.0.1:12379"
+  tls:
+    verify: false
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_GET,
+                 nil,
+                [[{
+                    "node": {
+                        "value": {
+                            "methods": [
+                                "GET"
+                            ],
+                            "uri": "/index.html",
+                            "desc": "new route",
+                            "upstream": {
+                                "nodes": {
+                                    "127.0.0.1:8080": 1
+                                },
+                                "type": "roundrobin"
+                            }
+                        },
+                        "key": "/apisix/routes/1"
+                    },
+                    "action": "get"
+                }]]
+                )
+
+            ngx.status = code
+            ngx.say(body)
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]