feat: support gRPCS (#3411)

.gitignore

@@ -71,5 +71,6 @@ t/lib/dubbo-backend/dubbo-backend-provider/target/
 /conf/config-*.yaml
 !/conf/config-default.yaml
 /conf/debug-*.yaml
+/build-cache/
 # release tar package
 *.tgz

.travis/linux_openresty_common_runner.sh

@@ -74,20 +74,17 @@ do_install() {
     cp .travis/ASF* .travis/openwhisk-utilities/scancode/
 
     ls -l ./
-    if [ ! -f "build-cache/grpc_server_example" ]; then
-        wget https://github.com/iresty/grpc_server_example/releases/download/20200901/grpc_server_example-amd64.tar.gz
+    if [ ! -f "build-cache/grpc_server_example_20210122" ]; then
+        wget https://github.com/api7/grpc_server_example/releases/download/20210122/grpc_server_example-amd64.tar.gz
         tar -xvf grpc_server_example-amd64.tar.gz
         mv grpc_server_example build-cache/
-    fi
-
-    if [ ! -f "build-cache/proto/helloworld.proto" ]; then
-        if [ ! -f "grpc_server_example/main.go" ]; then
-            git clone https://github.com/iresty/grpc_server_example.git grpc_server_example
-        fi
 
-        cd grpc_server_example/
+        git clone --depth 1 https://github.com/api7/grpc_server_example.git grpc_server_example
+        pushd grpc_server_example/ || exit 1
         mv proto/ ../build-cache/
-        cd ..
+        popd || exit 1
+
+        touch build-cache/grpc_server_example_20210122
     fi
 
     if [ ! -f "build-cache/grpcurl" ]; then
@@ -101,7 +98,10 @@ script() {
     export_or_prefix
     openresty -V
 
-    ./build-cache/grpc_server_example &
+    ./build-cache/grpc_server_example \
+        -grpc-address :50051 -grpcs-address :50052 \
+        -crt ./t/certs/apisix.crt -key ./t/certs/apisix.key \
+        &
 
     ./bin/apisix help
     ./bin/apisix init

.travis/linux_tengine_runner.sh

@@ -234,12 +234,6 @@ do_install() {
     cp .travis/ASF* .travis/openwhisk-utilities/scancode/
 
     ls -l ./
-
-    if [ ! -f "build-cache/grpc_server_example" ]; then
-        wget https://github.com/iresty/grpc_server_example/releases/download/20200901/grpc_server_example-amd64.tar.gz
-        tar -xvf grpc_server_example-amd64.tar.gz
-        mv grpc_server_example build-cache/
-    fi
 }
 
 script() {

apisix/cli/ngx_tpl.lua

@@ -554,6 +554,52 @@ http {
             }
         }
 
+        {% if use_or_1_15 then %}
+        # hack for OpenResty before 1.17.8, which doesn't support variable inside grpc_pass
+        location @1_15_grpc_pass {
+            access_by_lua_block {
+                apisix.grpc_access_phase()
+            }
+
+            grpc_set_header   Content-Type application/grpc;
+            grpc_socket_keepalive on;
+            grpc_pass         grpc://apisix_backend;
+
+            header_filter_by_lua_block {
+                apisix.http_header_filter_phase()
+            }
+
+            body_filter_by_lua_block {
+                apisix.http_body_filter_phase()
+            }
+
+            log_by_lua_block {
+                apisix.http_log_phase()
+            }
+        }
+
+        location @1_15_grpcs_pass {
+            access_by_lua_block {
+                apisix.grpc_access_phase()
+            }
+
+            grpc_set_header   Content-Type application/grpc;
+            grpc_socket_keepalive on;
+            grpc_pass         grpcs://apisix_backend;
+
+            header_filter_by_lua_block {
+                apisix.http_header_filter_phase()
+            }
+
+            body_filter_by_lua_block {
+                apisix.http_body_filter_phase()
+            }
+
+            log_by_lua_block {
+                apisix.http_log_phase()
+            }
+        }
+        {% else %}
         location @grpc_pass {
 
             access_by_lua_block {
@@ -562,7 +608,7 @@ http {
 
             grpc_set_header   Content-Type application/grpc;
             grpc_socket_keepalive on;
-            grpc_pass         grpc://apisix_backend;
+            grpc_pass         $upstream_scheme://apisix_backend;
 
             header_filter_by_lua_block {
                 apisix.http_header_filter_phase()
@@ -576,6 +622,7 @@ http {
                 apisix.http_log_phase()
             }
         }
+        {% end %}
 
         {% if enabled_plugins["dubbo-proxy"] then %}
         location @dubbo_pass {

apisix/cli/ops.lua

@@ -184,9 +184,23 @@ Please modify "admin_key" in conf/config.yaml .
         util.die("ERROR: Admin API can only be used with etcd config_center.\n")
     end
 
-    local or_ver = util.execute_cmd("openresty -V 2>&1")
+    local or_ver = get_openresty_version()
+    if or_ver == nil then
+        util.die("can not find openresty\n")
+    end
+
+    local use_or_1_15 = true
+    local need_ver = "1.15.8"
+    if not check_version(or_ver, need_ver) then
+        util.die("openresty version must >=", need_ver, " current ", or_ver, "\n")
+    end
+    if check_version(or_ver, "1.17.8") then
+        use_or_1_15 = false
+    end
+
+    local or_info = util.execute_cmd("openresty -V 2>&1")
     local with_module_status = true
-    if or_ver and not or_ver:find("http_stub_status_module", 1, true) then
+    if or_info and not or_info:find("http_stub_status_module", 1, true) then
         stderr:write("'http_stub_status_module' module is missing in ",
                      "your openresty, please check it out. Without this ",
                      "module, there will be fewer monitoring indicators.\n")
@@ -245,6 +259,7 @@ Please modify "admin_key" in conf/config.yaml .
 
     -- Using template.render
     local sys_conf = {
+        use_or_1_15 = use_or_1_15,
         lua_path = env.pkg_path_org,
         lua_cpath = env.pkg_cpath_org,
         os_name = util.trim(util.execute_cmd("uname")),
@@ -365,16 +380,6 @@ Please modify "admin_key" in conf/config.yaml .
     if not ok then
         util.die("failed to update nginx.conf: ", err, "\n")
     end
-
-    local op_ver = get_openresty_version()
-    if op_ver == nil then
-        util.die("can not find openresty\n")
-    end
-
-    local need_ver = "1.15.8"
-    if not check_version(op_ver, need_ver) then
-        util.die("openresty version must >=", need_ver, " current ", op_ver, "\n")
-    end
 end
 
 

apisix/init.lua

@@ -28,6 +28,7 @@ local upstream_util = require("apisix.utils.upstream")
 local ctxdump       = require("resty.ctxdump")
 local ipmatcher     = require("resty.ipmatcher")
 local ngx           = ngx
+local ngx_version   = ngx.config.nginx_version
 local get_method    = ngx.req.get_method
 local ngx_exit      = ngx.exit
 local math          = math
@@ -511,6 +512,10 @@ function _M.http_access_phase()
         run_plugin("access", plugins, api_ctx)
     end
 
+    if route.value.service_protocol == "grpc" then
+        api_ctx.upstream_scheme = "grpc"
+    end
+
     local code, err = set_upstream(route, api_ctx)
     if code then
         core.log.error("failed to set upstream: ", err)
@@ -519,8 +524,13 @@ function _M.http_access_phase()
 
     set_upstream_host(api_ctx)
 
-    if route.value.service_protocol == "grpc" then
+    local up_scheme = api_ctx.upstream_scheme
+    if up_scheme == "grpcs" or up_scheme == "grpc" then
         ngx_var.ctx_ref = ctxdump.stash_ngx_ctx()
+        if ngx_version < 1017008 then
+            return ngx.exec("@1_15_" .. up_scheme .. "_pass")
+        end
+
         return ngx.exec("@grpc_pass")
     end
 

apisix/plugins/proxy-rewrite.lua

@@ -124,7 +124,6 @@ end
 
 do
     local upstream_vars = {
-        scheme     = "upstream_scheme",
         host       = "upstream_host",
         upgrade    = "upstream_upgrade",
         connection = "upstream_connection",
@@ -140,6 +139,9 @@ function _M.rewrite(conf, ctx)
             ctx.var[upstream_vars[name]] = conf[name]
         end
     end
+    if conf["scheme"] then
+        ctx.upstream_scheme = conf["scheme"]
+    end
 
     local upstream_uri = ctx.var.uri
     if conf.uri ~= nil then

apisix/schema_def.lua

@@ -341,6 +341,10 @@ local upstream_schema = {
             description = "the key of chash for dynamic load balancing",
             type = "string",
         },
+        scheme = {
+            default = "http",
+            enum = {"grpc", "grpcs", "http"}
+        },
         labels = {
             description = "key/value pairs to specify attributes",
             type = "object",

apisix/upstream.lua

@@ -22,6 +22,7 @@ local error = error
 local tostring = tostring
 local ipairs = ipairs
 local pairs = pairs
+local is_http = ngx.config.subsystem == "http"
 local upstreams
 local healthcheck
 
@@ -122,6 +123,17 @@ local function fetch_healthchecker(upstream)
 end
 
 
+local function set_upstream_scheme(ctx, upstream)
+    -- plugins like proxy-rewrite may already set ctx.upstream_scheme
+    if not ctx.upstream_scheme then
+        -- the old configuration doesn't have scheme field, so fallback to "http"
+        ctx.upstream_scheme = upstream.scheme or "http"
+    end
+
+    ctx.var["upstream_scheme"] = ctx.upstream_scheme
+end
+
+
 function _M.set_by_route(route, api_ctx)
     if api_ctx.upstream_conf then
         core.log.warn("upstream node has been specified, ",
@@ -175,11 +187,16 @@ function _M.set_by_route(route, api_ctx)
         return 502, "no valid upstream node"
     end
 
+    if not is_http then
+        return
+    end
+
     if nodes_count > 1 then
         local checker = fetch_healthchecker(up_conf)
         api_ctx.up_checker = checker
     end
 
+    set_upstream_scheme(api_ctx, up_conf)
     return
 end
 

doc/admin-api.md

@@ -73,7 +73,6 @@
 |upstream |False |Upstream|Enabled Upstream configuration, see [Upstream](architecture-design.md#upstream) for more||
 |upstream_id|False |Upstream|Enabled upstream id, see [Upstream](architecture-design.md#upstream) for more ||
 |service_id|False |Service|Binded Service configuration, see [Service](architecture-design.md#service) for more ||
-|service_protocol|False|Upstream protocol type|only `grpc`|Must set `grpc` if using `gRPC proxy` or `gRPC transcode`. |
 |labels   |False |Match Rules|Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}|
 |enable_websocket|False|Auxiliary| enable `websocket`(boolean), default `false`.||
 |status          |False|Auxiliary| enable this route, default `1`.|`1` to enable, `0` to disable|
@@ -525,6 +524,7 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
 |desc     |optional|upstream usage scenarios, and more.|
 |pass_host            |optional|`pass` pass the client request host, `node` not pass the client request host, using the upstream node host, `rewrite` rewrite host by the configured `upstream_host`.|
 |upstream_host    |optional|This option is only valid if the `pass_host` is `rewrite`.|
+|scheme|optional |The scheme used when talk with the upstream. The value is one of ['http', 'grpc', 'grpcs'], default to 'http'.|
 |labels|optional |Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}|
 |create_time|optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|
 |update_time|optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|

doc/grpc-proxy.md

@@ -22,20 +22,20 @@
 # grpc-proxy
 
 proxying gRPC traffic:
-gRPC client -> APISIX -> gRPC server
+gRPC client -> APISIX -> gRPC/gRPCS server
 
 ## Parameters
 
-* `service_protocol`:  the route's option `service_protocol` must be `grpc`
-* `uri`:   format likes /service/method, Example：/helloworld.Greeter/SayHello
+* `scheme`: the `scheme` of the route's upstream must be `grpc` or `grpcs`.
+* `uri`: format likes /service/method, Example：/helloworld.Greeter/SayHello
 
 ### Example
 
 #### create proxying gRPC route
 
 Here's an example, to proxying gRPC service by specified route:
 
-* attention: the route's option `service_protocol` must be `grpc`
+* attention: the `scheme` of the route's upstream must be `grpc` or `grpcs`.
 * attention: APISIX use TLS‑encrypted HTTP/2 to expose gRPC service, so need to [config SSL certificate](https.md)
 * the grpc server example：[grpc_server_example](https://github.com/iresty/grpc_server_example)
 
@@ -44,8 +44,8 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
 {
     "methods": ["POST", "GET"],
     "uri": "/helloworld.Greeter/SayHello",
-    "service_protocol": "grpc",
     "upstream": {
+        "scheme": "grpc",
         "type": "roundrobin",
         "nodes": {
             "127.0.0.1:50051": 1
@@ -66,3 +66,22 @@ $ grpcurl -insecure -import-path /pathtoprotos  -proto helloworld.proto  -d '{"n
 ```
 
 This means that the proxying is working.
+
+### gRPCS
+
+If your gRPC service encrypts with TLS by itself (so called `gPRCS`, gPRC + TLS), you need to change the `scheme` to `grpcs`. The example above runs gRPCS service on port 50052, to proxy gRPC request, we need to use the configuration below:
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "methods": ["POST", "GET"],
+    "uri": "/helloworld.Greeter/SayHello",
+    "upstream": {
+        "scheme": "grpcs",
+        "type": "roundrobin",
+        "nodes": {
+            "127.0.0.1:50052": 1
+        }
+    }
+}'
+```