feat: support mTLS with etcd

apisix/core/config_etcd.lua

@@ -506,8 +506,15 @@ do
 
         -- default to verify etcd cluster certificate
         etcd_conf.ssl_verify = true
-        if etcd_conf.tls and etcd_conf.tls.verify == false then
-            etcd_conf.ssl_verify = false
+        if etcd_conf.tls then
+            if etcd_conf.tls.verify == false then
+                etcd_conf.ssl_verify = false
+            end
+
+            if etcd_conf.tls.cert then
+                etcd_conf.ssl_cert_path = etcd_conf.tls.cert
+                etcd_conf.ssl_key_path = etcd_conf.tls.key
+            end
         end
 
         local err

apisix/core/etcd.lua

@@ -24,6 +24,7 @@ local tonumber         = tonumber
 local _M = {}
 
 
+-- this function create the etcd client instance used in the Admin API
 local function new()
     local local_conf, err = fetch_local_conf()
     if not local_conf then
@@ -40,8 +41,16 @@ local function new()
     etcd_conf.ssl_verify = true
 
     -- default to verify etcd cluster certificate
-    if etcd_conf.tls and etcd_conf.tls.verify == false then
-        etcd_conf.ssl_verify = false
+    etcd_conf.ssl_verify = true
+    if etcd_conf.tls then
+        if etcd_conf.tls.verify == false then
+            etcd_conf.ssl_verify = false
+        end
+
+        if etcd_conf.tls.cert then
+            etcd_conf.ssl_cert_path = etcd_conf.tls.cert
+            etcd_conf.ssl_key_path = etcd_conf.tls.key
+        end
     end
 
     local etcd_cli

apisix/patch.lua

@@ -119,7 +119,12 @@ local luasocket_wrapper = {
         return self.sock:settimeout(time)
     end,
 
-    sslhandshake = function (self, reused_session, server_name, verify, send_status_req)
+    tlshandshake = function (self, options)
+        local reused_session = options.reused_session
+        local server_name = options.server_name
+        local verify = options.verify
+        local send_status_req = options.ocsp_status_req
+
         if reused_session then
             log(WARN, "reused_session is not supported yet")
         end
@@ -132,6 +137,8 @@ local luasocket_wrapper = {
             mode = "client",
             protocol = "any",
             verify = verify and "peer" or "none",
+            certificate = options.client_cert_path,
+            key = options.client_priv_key_path,
             options = {
                 "all",
                 "no_sslv2",
@@ -157,6 +164,15 @@ local luasocket_wrapper = {
 
         self.sock = sec_sock
         return true
+    end,
+
+    sslhandshake = function (self, reused_session, server_name, verify, send_status_req)
+        return self:tlshandshake({
+            reused_session = reused_session,
+            server_name = server_name,
+            verify = verify,
+            ocsp_status_req = send_status_req,
+        })
     end
 }
 

rockspec/apisix-master-0.rockspec

@@ -34,7 +34,7 @@ dependencies = {
     "lua-resty-ctxdump = 0.1-0",
     "lua-resty-dns-client = 5.2.0",
     "lua-resty-template = 1.9",
-    "lua-resty-etcd = 1.4.3",
+    "lua-resty-etcd = 1.5.0",
     "lua-resty-balancer = 0.02rc5",
     "lua-resty-ngxvar = 0.5.2",
     "lua-resty-jit-uuid = 0.0.7",

t/APISIX.pm

@@ -298,8 +298,9 @@ _EOC_
             apisix.stream_balancer_phase()
         }
     }
+_EOC_
 
-    init_by_lua_block {
+    my $stream_init_by_lua_block = $block->stream_init_by_lua_block // <<_EOC_;
         if os.getenv("APISIX_ENABLE_LUACOV") == "1" then
             require("luacov.runner")("t/apisix.luacov")
             jit.off()
@@ -309,8 +310,12 @@ _EOC_
 
         apisix = require("apisix")
         apisix.stream_init()
-    }
+_EOC_
 
+    $stream_config .= <<_EOC_;
+    init_by_lua_block {
+        $stream_init_by_lua_block
+    }
     init_worker_by_lua_block {
         apisix.stream_init_worker()
     }

t/core/etcd-mtls.t

@@ -0,0 +1,239 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX;
+
+my $out = eval { `resty -e "local s=ngx.socket.tcp();print(s.tlshandshake)"` };
+
+if ($out !~ m/function:/) {
+    plan(skip_all => "tlshandshake not patched");
+} else {
+    plan('no_plan');
+}
+
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!$block->no_error_log && !$block->error_log) {
+        $block->set_value("no_error_log", "[error]\n[alert]");
+    }
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: run etcd in init phase
+--- yaml_config
+etcd:
+  host:
+    - "https://127.0.0.1:22379"
+  prefix: "/apisix"
+  tls:
+    cert: t/certs/mtls_client.crt
+    key: t/certs/mtls_client.key
+    verify: false
+--- init_by_lua_block
+    local apisix = require("apisix")
+    apisix.http_init()
+    local etcd = require("apisix.core.etcd")
+    assert(etcd.set("/a", "ab"))
+
+    local res, err = etcd.get("/a")
+    if not res then
+        ngx.log(ngx.ERR, err)
+        return
+    end
+    ngx.log(ngx.WARN, res.body.node.value)
+
+    local res, err = etcd.delete("/a")
+    if not res then
+        ngx.log(ngx.ERR, err)
+        return
+    end
+    ngx.log(ngx.WARN, res.status)
+
+    local res, err = etcd.get("/a")
+    if not res then
+        ngx.log(ngx.ERR, err)
+        return
+    end
+    ngx.log(ngx.WARN, res.status)
+--- config
+    location /t {
+        return 200;
+    }
+--- request
+GET /t
+--- grep_error_log eval
+qr/init_by_lua:\d+: \S+/
+--- grep_error_log_out
+init_by_lua:12: ab
+init_by_lua:19: 200
+init_by_lua:26: 404
+
+
+
+=== TEST 2: run etcd in init phase (stream)
+--- yaml_config
+etcd:
+  host:
+    - "https://127.0.0.1:22379"
+  prefix: "/apisix"
+  tls:
+    cert: t/certs/mtls_client.crt
+    key: t/certs/mtls_client.key
+    verify: false
+--- stream_init_by_lua_block
+    apisix = require("apisix")
+    apisix.stream_init()
+    local etcd = require("apisix.core.etcd")
+    assert(etcd.set("/a", "ab"))
+
+    local res, err = etcd.get("/a")
+    if not res then
+        ngx.log(ngx.ERR, err)
+        return
+    end
+    ngx.log(ngx.WARN, res.body.node.value)
+
+    local res, err = etcd.delete("/a")
+    if not res then
+        ngx.log(ngx.ERR, err)
+        return
+    end
+    ngx.log(ngx.WARN, res.status)
+
+    local res, err = etcd.get("/a")
+    if not res then
+        ngx.log(ngx.ERR, err)
+        return
+    end
+    ngx.log(ngx.WARN, res.status)
+--- stream_server_config
+    content_by_lua_block {
+        ngx.say("ok")
+    }
+--- stream_enable
+--- grep_error_log eval
+qr/init_by_lua:\d+: \S+/
+--- grep_error_log_out
+init_by_lua:12: ab
+init_by_lua:19: 200
+init_by_lua:26: 404
+
+
+
+=== TEST 3: sync
+--- extra_yaml_config
+etcd:
+  host:
+    - "https://127.0.0.1:22379"
+  prefix: "/apisix"
+  tls:
+    cert: t/certs/mtls_client.crt
+    key: t/certs/mtls_client.key
+    verify: false
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin").test
+
+            local consumers, _ = core.config.new("/consumers", {
+                automatic = true,
+                item_schema = core.schema.consumer,
+            })
+
+            ngx.sleep(0.6)
+            local idx = consumers.prev_index
+
+            local code, body = t('/apisix/admin/consumers',
+                ngx.HTTP_PUT,
+                [[{
+                    "username": "jobs",
+                    "plugins": {
+                        "basic-auth": {
+                            "username": "jobs",
+                            "password": "678901"
+                        }
+                    }
+                }]])
+
+            ngx.sleep(2)
+            local new_idx = consumers.prev_index
+            if new_idx > idx then
+                ngx.say("prev_index updated")
+            else
+                ngx.say("prev_index not update")
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+prev_index updated
+--- no_error_log
+[error]
+--- error_log
+waitdir key
+
+
+
+=== TEST 4: sync (stream)
+--- extra_yaml_config
+etcd:
+  host:
+    - "https://127.0.0.1:22379"
+  prefix: "/apisix"
+  tls:
+    cert: t/certs/mtls_client.crt
+    key: t/certs/mtls_client.key
+    verify: false
+--- stream_server_config
+    content_by_lua_block {
+        local core = require("apisix.core")
+
+        local sr, _ = core.config.new("/stream_routes", {
+            automatic = true,
+            item_schema = core.schema.stream_routes,
+        })
+
+        ngx.sleep(0.6)
+        local idx = sr.prev_index
+
+        assert(core.etcd.set("/stream_routes/1",
+            {
+                plugins = {
+                }
+            }))
+
+        ngx.sleep(2)
+        local new_idx = sr.prev_index
+        if new_idx > idx then
+            ngx.say("prev_index updated")
+        else
+            ngx.say("prev_index not update")
+        end
+        }
+--- stream_enable
+--- stream_response
+prev_index updated
+--- no_error_log
+[error]
+--- error_log
+waitdir key