diff --git a/.gitignore b/.gitignore index 548b6c239c0a..dd70e9b72e36 100644 --- a/.gitignore +++ b/.gitignore @@ -71,5 +71,6 @@ t/lib/dubbo-backend/dubbo-backend-provider/target/ /conf/config-*.yaml !/conf/config-default.yaml /conf/debug-*.yaml +/build-cache/ # release tar package *.tgz diff --git a/.travis/linux_openresty_common_runner.sh b/.travis/linux_openresty_common_runner.sh index 2229d6a1869e..04e2d3626612 100755 --- a/.travis/linux_openresty_common_runner.sh +++ b/.travis/linux_openresty_common_runner.sh @@ -74,20 +74,17 @@ do_install() { cp .travis/ASF* .travis/openwhisk-utilities/scancode/ ls -l ./ - if [ ! -f "build-cache/grpc_server_example" ]; then - wget https://github.com/iresty/grpc_server_example/releases/download/20200901/grpc_server_example-amd64.tar.gz + if [ ! -f "build-cache/grpc_server_example_20210122" ]; then + wget https://github.com/api7/grpc_server_example/releases/download/20210122/grpc_server_example-amd64.tar.gz tar -xvf grpc_server_example-amd64.tar.gz mv grpc_server_example build-cache/ - fi - - if [ ! -f "build-cache/proto/helloworld.proto" ]; then - if [ ! -f "grpc_server_example/main.go" ]; then - git clone https://github.com/iresty/grpc_server_example.git grpc_server_example - fi - cd grpc_server_example/ + git clone --depth 1 https://github.com/api7/grpc_server_example.git grpc_server_example + pushd grpc_server_example/ || exit 1 mv proto/ ../build-cache/ - cd .. + popd || exit 1 + + touch build-cache/grpc_server_example_20210122 fi if [ ! -f "build-cache/grpcurl" ]; then @@ -101,7 +98,10 @@ script() { export_or_prefix openresty -V - ./build-cache/grpc_server_example & + ./build-cache/grpc_server_example \ + -grpc-address :50051 -grpcs-address :50052 \ + -crt ./t/certs/apisix.crt -key ./t/certs/apisix.key \ + & ./bin/apisix help ./bin/apisix init diff --git a/.travis/linux_tengine_runner.sh b/.travis/linux_tengine_runner.sh index fc7c40dba4cd..ad8a5e31214b 100755 --- a/.travis/linux_tengine_runner.sh +++ b/.travis/linux_tengine_runner.sh @@ -234,12 +234,6 @@ do_install() { cp .travis/ASF* .travis/openwhisk-utilities/scancode/ ls -l ./ - - if [ ! -f "build-cache/grpc_server_example" ]; then - wget https://github.com/iresty/grpc_server_example/releases/download/20200901/grpc_server_example-amd64.tar.gz - tar -xvf grpc_server_example-amd64.tar.gz - mv grpc_server_example build-cache/ - fi } script() { diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index dab32d02083d..18f663277980 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -554,6 +554,52 @@ http { } } + {% if use_or_1_15 then %} + # hack for OpenResty before 1.17.8, which doesn't support variable inside grpc_pass + location @1_15_grpc_pass { + access_by_lua_block { + apisix.grpc_access_phase() + } + + grpc_set_header Content-Type application/grpc; + grpc_socket_keepalive on; + grpc_pass grpc://apisix_backend; + + header_filter_by_lua_block { + apisix.http_header_filter_phase() + } + + body_filter_by_lua_block { + apisix.http_body_filter_phase() + } + + log_by_lua_block { + apisix.http_log_phase() + } + } + + location @1_15_grpcs_pass { + access_by_lua_block { + apisix.grpc_access_phase() + } + + grpc_set_header Content-Type application/grpc; + grpc_socket_keepalive on; + grpc_pass grpcs://apisix_backend; + + header_filter_by_lua_block { + apisix.http_header_filter_phase() + } + + body_filter_by_lua_block { + apisix.http_body_filter_phase() + } + + log_by_lua_block { + apisix.http_log_phase() + } + } + {% else %} location @grpc_pass { access_by_lua_block { @@ -562,7 +608,7 @@ http { grpc_set_header Content-Type application/grpc; grpc_socket_keepalive on; - grpc_pass grpc://apisix_backend; + grpc_pass $upstream_scheme://apisix_backend; header_filter_by_lua_block { apisix.http_header_filter_phase() @@ -576,6 +622,7 @@ http { apisix.http_log_phase() } } + {% end %} {% if enabled_plugins["dubbo-proxy"] then %} location @dubbo_pass { diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index d6ebab18538b..45111f641410 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -184,9 +184,23 @@ Please modify "admin_key" in conf/config.yaml . util.die("ERROR: Admin API can only be used with etcd config_center.\n") end - local or_ver = util.execute_cmd("openresty -V 2>&1") + local or_ver = get_openresty_version() + if or_ver == nil then + util.die("can not find openresty\n") + end + + local use_or_1_15 = true + local need_ver = "1.15.8" + if not check_version(or_ver, need_ver) then + util.die("openresty version must >=", need_ver, " current ", or_ver, "\n") + end + if check_version(or_ver, "1.17.8") then + use_or_1_15 = false + end + + local or_info = util.execute_cmd("openresty -V 2>&1") local with_module_status = true - if or_ver and not or_ver:find("http_stub_status_module", 1, true) then + if or_info and not or_info:find("http_stub_status_module", 1, true) then stderr:write("'http_stub_status_module' module is missing in ", "your openresty, please check it out. Without this ", "module, there will be fewer monitoring indicators.\n") @@ -245,6 +259,7 @@ Please modify "admin_key" in conf/config.yaml . -- Using template.render local sys_conf = { + use_or_1_15 = use_or_1_15, lua_path = env.pkg_path_org, lua_cpath = env.pkg_cpath_org, os_name = util.trim(util.execute_cmd("uname")), @@ -365,16 +380,6 @@ Please modify "admin_key" in conf/config.yaml . if not ok then util.die("failed to update nginx.conf: ", err, "\n") end - - local op_ver = get_openresty_version() - if op_ver == nil then - util.die("can not find openresty\n") - end - - local need_ver = "1.15.8" - if not check_version(op_ver, need_ver) then - util.die("openresty version must >=", need_ver, " current ", op_ver, "\n") - end end diff --git a/apisix/init.lua b/apisix/init.lua index a33248f0d8ed..74e6ebac9a4a 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -28,6 +28,7 @@ local upstream_util = require("apisix.utils.upstream") local ctxdump = require("resty.ctxdump") local ipmatcher = require("resty.ipmatcher") local ngx = ngx +local ngx_version = ngx.config.nginx_version local get_method = ngx.req.get_method local ngx_exit = ngx.exit local math = math @@ -511,6 +512,10 @@ function _M.http_access_phase() run_plugin("access", plugins, api_ctx) end + if route.value.service_protocol == "grpc" then + api_ctx.upstream_scheme = "grpc" + end + local code, err = set_upstream(route, api_ctx) if code then core.log.error("failed to set upstream: ", err) @@ -519,8 +524,13 @@ function _M.http_access_phase() set_upstream_host(api_ctx) - if route.value.service_protocol == "grpc" then + local up_scheme = api_ctx.upstream_scheme + if up_scheme == "grpcs" or up_scheme == "grpc" then ngx_var.ctx_ref = ctxdump.stash_ngx_ctx() + if ngx_version < 1017008 then + return ngx.exec("@1_15_" .. up_scheme .. "_pass") + end + return ngx.exec("@grpc_pass") end diff --git a/apisix/plugins/proxy-rewrite.lua b/apisix/plugins/proxy-rewrite.lua index d2fc5cf6cb90..c7c37114990f 100644 --- a/apisix/plugins/proxy-rewrite.lua +++ b/apisix/plugins/proxy-rewrite.lua @@ -124,7 +124,6 @@ end do local upstream_vars = { - scheme = "upstream_scheme", host = "upstream_host", upgrade = "upstream_upgrade", connection = "upstream_connection", @@ -140,6 +139,9 @@ function _M.rewrite(conf, ctx) ctx.var[upstream_vars[name]] = conf[name] end end + if conf["scheme"] then + ctx.upstream_scheme = conf["scheme"] + end local upstream_uri = ctx.var.uri if conf.uri ~= nil then diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua index 991ab430f20c..78c1908e7319 100644 --- a/apisix/schema_def.lua +++ b/apisix/schema_def.lua @@ -341,6 +341,10 @@ local upstream_schema = { description = "the key of chash for dynamic load balancing", type = "string", }, + scheme = { + default = "http", + enum = {"grpc", "grpcs", "http"} + }, labels = { description = "key/value pairs to specify attributes", type = "object", diff --git a/apisix/upstream.lua b/apisix/upstream.lua index dd455d42da18..53c26252f959 100644 --- a/apisix/upstream.lua +++ b/apisix/upstream.lua @@ -22,6 +22,7 @@ local error = error local tostring = tostring local ipairs = ipairs local pairs = pairs +local is_http = ngx.config.subsystem == "http" local upstreams local healthcheck @@ -122,6 +123,17 @@ local function fetch_healthchecker(upstream) end +local function set_upstream_scheme(ctx, upstream) + -- plugins like proxy-rewrite may already set ctx.upstream_scheme + if not ctx.upstream_scheme then + -- the old configuration doesn't have scheme field, so fallback to "http" + ctx.upstream_scheme = upstream.scheme or "http" + end + + ctx.var["upstream_scheme"] = ctx.upstream_scheme +end + + function _M.set_by_route(route, api_ctx) if api_ctx.upstream_conf then core.log.warn("upstream node has been specified, ", @@ -175,11 +187,16 @@ function _M.set_by_route(route, api_ctx) return 502, "no valid upstream node" end + if not is_http then + return + end + if nodes_count > 1 then local checker = fetch_healthchecker(up_conf) api_ctx.up_checker = checker end + set_upstream_scheme(api_ctx, up_conf) return end diff --git a/doc/admin-api.md b/doc/admin-api.md index b2e668171f47..3744131ca81e 100644 --- a/doc/admin-api.md +++ b/doc/admin-api.md @@ -73,7 +73,6 @@ |upstream |False |Upstream|Enabled Upstream configuration, see [Upstream](architecture-design.md#upstream) for more|| |upstream_id|False |Upstream|Enabled upstream id, see [Upstream](architecture-design.md#upstream) for more || |service_id|False |Service|Binded Service configuration, see [Service](architecture-design.md#service) for more || -|service_protocol|False|Upstream protocol type|only `grpc`|Must set `grpc` if using `gRPC proxy` or `gRPC transcode`. | |labels |False |Match Rules|Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}| |enable_websocket|False|Auxiliary| enable `websocket`(boolean), default `false`.|| |status |False|Auxiliary| enable this route, default `1`.|`1` to enable, `0` to disable| @@ -525,6 +524,7 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst |desc |optional|upstream usage scenarios, and more.| |pass_host |optional|`pass` pass the client request host, `node` not pass the client request host, using the upstream node host, `rewrite` rewrite host by the configured `upstream_host`.| |upstream_host |optional|This option is only valid if the `pass_host` is `rewrite`.| +|scheme|optional |The scheme used when talk with the upstream. The value is one of ['http', 'grpc', 'grpcs'], default to 'http'.| |labels|optional |Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}| |create_time|optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing| |update_time|optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing| diff --git a/doc/grpc-proxy.md b/doc/grpc-proxy.md index 5553bf050571..ee1d57b4efc8 100644 --- a/doc/grpc-proxy.md +++ b/doc/grpc-proxy.md @@ -22,12 +22,12 @@ # grpc-proxy proxying gRPC traffic: -gRPC client -> APISIX -> gRPC server +gRPC client -> APISIX -> gRPC/gRPCS server ## Parameters -* `service_protocol`: the route's option `service_protocol` must be `grpc` -* `uri`: format likes /service/method, Example:/helloworld.Greeter/SayHello +* `scheme`: the `scheme` of the route's upstream must be `grpc` or `grpcs`. +* `uri`: format likes /service/method, Example:/helloworld.Greeter/SayHello ### Example @@ -35,7 +35,7 @@ gRPC client -> APISIX -> gRPC server Here's an example, to proxying gRPC service by specified route: -* attention: the route's option `service_protocol` must be `grpc` +* attention: the `scheme` of the route's upstream must be `grpc` or `grpcs`. * attention: APISIX use TLS‑encrypted HTTP/2 to expose gRPC service, so need to [config SSL certificate](https.md) * the grpc server example:[grpc_server_example](https://github.com/iresty/grpc_server_example) @@ -44,8 +44,8 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13 { "methods": ["POST", "GET"], "uri": "/helloworld.Greeter/SayHello", - "service_protocol": "grpc", "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -66,3 +66,22 @@ $ grpcurl -insecure -import-path /pathtoprotos -proto helloworld.proto -d '{"n ``` This means that the proxying is working. + +### gRPCS + +If your gRPC service encrypts with TLS by itself (so called `gPRCS`, gPRC + TLS), you need to change the `scheme` to `grpcs`. The example above runs gRPCS service on port 50052, to proxy gRPC request, we need to use the configuration below: + +```shell +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["POST", "GET"], + "uri": "/helloworld.Greeter/SayHello", + "upstream": { + "scheme": "grpcs", + "type": "roundrobin", + "nodes": { + "127.0.0.1:50052": 1 + } + } +}' +``` diff --git a/doc/plugins/grpc-transcode.md b/doc/plugins/grpc-transcode.md index f16f72a4340d..2eeecb0069a3 100644 --- a/doc/plugins/grpc-transcode.md +++ b/doc/plugins/grpc-transcode.md @@ -64,7 +64,7 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136 Here's an example, to enable the grpc-transcode plugin to specified route: -* attention: the route's option `service_protocol` must be `grpc` +* attention: the `scheme` in the route's upstream must be `grpc` * the grpc server example:[grpc_server_example](https://github.com/iresty/grpc_server_example) ```shell @@ -72,7 +72,6 @@ curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f { "methods": ["GET"], "uri": "/grpctest", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -81,6 +80,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -141,7 +141,6 @@ curl http://127.0.0.1:9080/apisix/admin/routes/23 -H 'X-API-KEY: edd1c9f034335f1 { "methods": ["GET"], "uri": "/zeebe/WorkflowInstanceCreate", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -151,6 +150,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/23 -H 'X-API-KEY: edd1c9f034335f1 } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:26500": 1 @@ -192,6 +192,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f "uri": "/grpctest", "plugins": {}, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 diff --git a/doc/zh-cn/admin-api.md b/doc/zh-cn/admin-api.md index 9fdcb8e30169..55be467cbad6 100644 --- a/doc/zh-cn/admin-api.md +++ b/doc/zh-cn/admin-api.md @@ -64,7 +64,6 @@ |upstream |`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 Upstream 配置,详见 [Upstream](architecture-design.md#upstream)|| |upstream_id|`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 upstream id,详见 [Upstream](architecture-design.md#upstream)|| |service_id|`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Service|绑定的 Service 配置,详见 [Service](architecture-design.md#service)|| -|service_protocol|可选|上游协议类型|只可以是 "grpc"|启用 `gRPC proxy` 或 `gRPC transcode` 插件时,必须用 "grpc"| |name |可选 |辅助 |标识路由名称|route-xxxx| |desc |可选 |辅助 |标识描述、使用场景等。|客户 xxxx| |host |可选 |匹配规则|当前请求域名,比如 `foo.com`;也支持泛域名,比如 `*.foo.com`。|"foo.com"| @@ -534,6 +533,7 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外,还支持对上 |desc |可选 |辅助|上游服务描述、使用场景等。|| |pass_host |可选|枚举|`pass` 透传客户端请求的 host, `node` 不透传客户端请求的 host, 使用 upstream node 配置的 host, `rewrite` 使用 `upstream_host` 配置的值重写 host 。|| |upstream_host |可选|辅助|只在 `pass_host` 配置为 `rewrite` 时有效。|| +|scheme|可选 |辅助|跟上游通信时使用的 scheme。需要是 ['http', 'grpc', 'grpcs'] 其中的一个,默认是 'http'。| |labels |可选 |匹配规则|标识附加属性的键值对|{"version":"v2","build":"16","env":"production"}| |create_time|可选|辅助|单位为秒的 epoch 时间戳,如果不指定则自动创建|1602883670| |update_time|可选|辅助|单位为秒的 epoch 时间戳,如果不指定则自动创建|1602883670| diff --git a/doc/zh-cn/grpc-proxy.md b/doc/zh-cn/grpc-proxy.md index 3f76fd8bec65..85b262f417fe 100644 --- a/doc/zh-cn/grpc-proxy.md +++ b/doc/zh-cn/grpc-proxy.md @@ -25,8 +25,8 @@ ## 参数 -* `service_protocol`: 这个 Route 的属性 `service_protocol` 必须设置为 `grpc` -* `uri`: 格式为 /service/method 如:/helloworld.Greeter/SayHello +* `scheme`: Route 对应的 Upstream 的 `scheme` 必须设置为 `grpc` 或者 `grpcs` +* `uri`: 格式为 /service/method 如:/helloworld.Greeter/SayHello ## 示例 @@ -34,17 +34,17 @@ 在指定 Route 中,代理 gRPC 服务接口: -* 注意: 这个 Route 的属性 `service_protocol` 必须设置为 `grpc`; +* 注意:这个 Route 对应的 Upstream 的 `scheme` 必须设置为 `grpc` 或者 `grpcs`。 * 注意: APISIX 使用 TLS 加密的 HTTP/2 暴露 gRPC 服务, 所以需要先 [配置 SSL 证书](https.md); -* 下面例子所代理的 gRPC 服务可供参考:[grpc_server_example](https://github.com/iresty/grpc_server_example)。 +* 下面例子所代理的 gRPC 服务可供参考:[grpc_server_example](https://github.com/api7/grpc_server_example)。 ```shell curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' { "methods": ["POST", "GET"], "uri": "/helloworld.Greeter/SayHello", - "service_protocol": "grpc", "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -66,3 +66,22 @@ grpcurl -insecure -import-path /pathtoprotos -proto helloworld.proto \ ``` 这表示已成功代理。 + +### gRPCS + +如果你的 gRPC 服务使用了自己的 TLS 加密,即所谓的 `gPRCS` (gRPC + TLS),那么需要修改 scheme 为 `grpcs`。继续上面的例子,50052 端口上跑的是 gPRCS 的服务,这时候应该这么配置: + +```shell +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["POST", "GET"], + "uri": "/helloworld.Greeter/SayHello", + "upstream": { + "scheme": "grpcs", + "type": "roundrobin", + "nodes": { + "127.0.0.1:50052": 1 + } + } +}' +``` diff --git a/doc/zh-cn/plugins/grpc-transcode.md b/doc/zh-cn/plugins/grpc-transcode.md index 90ab3938494a..52983508edb7 100644 --- a/doc/zh-cn/plugins/grpc-transcode.md +++ b/doc/zh-cn/plugins/grpc-transcode.md @@ -66,7 +66,7 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136 在指定 route 中,代理 grpc 服务接口: -* 注意: 这个 route 的属性`service_protocol` 必须设置为 `grpc` +* 注意: 这个 route 对应的 upstream 的属性 `scheme` 必须设置为 `grpc` * 代理 grpc 服务例子可参考:[grpc_server_example](https://github.com/iresty/grpc_server_example) ```shell @@ -74,7 +74,6 @@ curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f { "methods": ["GET"], "uri": "/grpctest", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -83,6 +82,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -140,7 +140,6 @@ curl http://127.0.0.1:9080/apisix/admin/routes/23 -H 'X-API-KEY: edd1c9f034335f1 { "methods": ["GET"], "uri": "/zeebe/WorkflowInstanceCreate", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -150,6 +149,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/23 -H 'X-API-KEY: edd1c9f034335f1 } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:26500": 1 @@ -190,6 +190,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f "uri": "/grpctest", "plugins": {}, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 diff --git a/t/APISIX.pm b/t/APISIX.pm index 220745e778de..9dcea3aaf04f 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -175,6 +175,79 @@ _EOC_ _EOC_ } +my $grpc_location = <<_EOC_; + location \@grpc_pass { + access_by_lua_block { + apisix.grpc_access_phase() + } + + grpc_set_header Content-Type application/grpc; + grpc_socket_keepalive on; + grpc_pass \$upstream_scheme://apisix_backend; + + header_filter_by_lua_block { + apisix.http_header_filter_phase() + } + + body_filter_by_lua_block { + apisix.http_body_filter_phase() + } + + log_by_lua_block { + apisix.http_log_phase() + } + } +_EOC_ + +if ($version =~ m/\/1.15.8/) { + $grpc_location = <<_EOC_; + # hack for OpenResty before 1.17.8, which doesn't support variable inside grpc_pass + location \@1_15_grpc_pass { + access_by_lua_block { + apisix.grpc_access_phase() + } + + grpc_set_header Content-Type application/grpc; + grpc_socket_keepalive on; + grpc_pass grpc://apisix_backend; + + header_filter_by_lua_block { + apisix.http_header_filter_phase() + } + + body_filter_by_lua_block { + apisix.http_body_filter_phase() + } + + log_by_lua_block { + apisix.http_log_phase() + } + } + + location \@1_15_grpcs_pass { + access_by_lua_block { + apisix.grpc_access_phase() + } + + grpc_set_header Content-Type application/grpc; + grpc_socket_keepalive on; + grpc_pass grpcs://apisix_backend; + + header_filter_by_lua_block { + apisix.http_header_filter_phase() + } + + body_filter_by_lua_block { + apisix.http_body_filter_phase() + } + + log_by_lua_block { + apisix.http_log_phase() + } + } +_EOC_ +} + add_block_preprocessor(sub { my ($block) = @_; @@ -488,28 +561,7 @@ _EOC_ } } - location \@grpc_pass { - access_by_lua_block { - apisix.grpc_access_phase() - } - - grpc_set_header Content-Type application/grpc; - grpc_socket_keepalive on; - grpc_pass grpc://apisix_backend; - - header_filter_by_lua_block { - apisix.http_header_filter_phase() - } - - body_filter_by_lua_block { - apisix.http_body_filter_phase() - } - - log_by_lua_block { - apisix.http_log_phase() - } - } - + $grpc_location $dubbo_location location = /proxy_mirror { diff --git a/t/admin/routes2.t b/t/admin/routes2.t index 269671e332a9..cde747ef628a 100644 --- a/t/admin/routes2.t +++ b/t/admin/routes2.t @@ -203,7 +203,7 @@ GET /t --- request GET /t --- response_body -{"action":"create","node":{"value":{"methods":["GET"],"priority":0,"status":1,"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"},"uri":"/not_unwanted_data_post"}}} +{"action":"create","node":{"value":{"methods":["GET"],"priority":0,"status":1,"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"},"uri":"/not_unwanted_data_post"}}} --- no_error_log [error] @@ -245,7 +245,7 @@ GET /t --- request GET /t --- response_body -{"action":"set","node":{"key":"/apisix/routes/1","value":{"id":1,"methods":["GET"],"priority":0,"status":1,"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"},"uri":"/index.html"}}} +{"action":"set","node":{"key":"/apisix/routes/1","value":{"id":1,"methods":["GET"],"priority":0,"status":1,"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"},"uri":"/index.html"}}} --- no_error_log [error] @@ -286,7 +286,7 @@ GET /t --- request GET /t --- response_body -{"action":"compareAndSwap","node":{"key":"/apisix/routes/1","value":{"id":"1","methods":["GET"],"priority":0,"status":1,"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"},"uri":"/index"}}} +{"action":"compareAndSwap","node":{"key":"/apisix/routes/1","value":{"id":"1","methods":["GET"],"priority":0,"status":1,"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"},"uri":"/index"}}} --- no_error_log [error] @@ -317,7 +317,7 @@ GET /t --- request GET /t --- response_body -{"action":"get","count":"1","node":{"key":"/apisix/routes/1","value":{"id":"1","methods":["GET"],"priority":0,"status":1,"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"},"uri":"/index"}}} +{"action":"get","count":"1","node":{"key":"/apisix/routes/1","value":{"id":"1","methods":["GET"],"priority":0,"status":1,"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"},"uri":"/index"}}} --- no_error_log [error] diff --git a/t/admin/services2.t b/t/admin/services2.t index 6af6a77462bb..d9bebdf4cbb5 100644 --- a/t/admin/services2.t +++ b/t/admin/services2.t @@ -70,7 +70,7 @@ __DATA__ } } --- response_body -{"action":"create","node":{"value":{"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}}} +{"action":"create","node":{"value":{"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}}} @@ -105,7 +105,7 @@ __DATA__ } } --- response_body -{"action":"set","node":{"key":"/apisix/services/1","value":{"id":"1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}}} +{"action":"set","node":{"key":"/apisix/services/1","value":{"id":"1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}}} @@ -140,7 +140,7 @@ __DATA__ } } --- response_body -{"action":"compareAndSwap","node":{"key":"/apisix/services/1","value":{"id":"1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}}} +{"action":"compareAndSwap","node":{"key":"/apisix/services/1","value":{"id":"1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}}} @@ -167,7 +167,7 @@ __DATA__ } } --- response_body -{"action":"get","count":"1","node":{"key":"/apisix/services/1","value":{"id":"1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}}} +{"action":"get","count":"1","node":{"key":"/apisix/services/1","value":{"id":"1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}}} diff --git a/t/admin/stream-routes.t b/t/admin/stream-routes.t index c4787d00d9a0..7ad1da0b47a3 100644 --- a/t/admin/stream-routes.t +++ b/t/admin/stream-routes.t @@ -431,7 +431,7 @@ GET /t } } --- response_body -{"action":"create","node":{"value":{"remote_addr":"127.0.0.1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}}} +{"action":"create","node":{"value":{"remote_addr":"127.0.0.1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}}} --- request GET /t --- no_error_log @@ -471,7 +471,7 @@ GET /t } } --- response_body -{"action":"set","node":{"key":"/apisix/stream_routes/1","value":{"id":"1","remote_addr":"127.0.0.1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}}} +{"action":"set","node":{"key":"/apisix/stream_routes/1","value":{"id":"1","remote_addr":"127.0.0.1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}}} --- request GET /t --- no_error_log @@ -502,7 +502,7 @@ GET /t } } --- response_body -{"action":"get","count":"1","node":{"key":"/apisix/stream_routes/1","value":{"id":"1","remote_addr":"127.0.0.1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}}} +{"action":"get","count":"1","node":{"key":"/apisix/stream_routes/1","value":{"id":"1","remote_addr":"127.0.0.1","upstream":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}}} --- request GET /t --- no_error_log diff --git a/t/admin/upstream2.t b/t/admin/upstream2.t index 3ea83b571321..0a641d4f50aa 100644 --- a/t/admin/upstream2.t +++ b/t/admin/upstream2.t @@ -68,7 +68,7 @@ __DATA__ } } --- response_body -{"action":"create","node":{"value":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}} +{"action":"create","node":{"value":{"hash_on":"vars","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}} @@ -101,7 +101,7 @@ __DATA__ } } --- response_body -{"action":"set","node":{"key":"/apisix/upstreams/unwanted","value":{"hash_on":"vars","id":"unwanted","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}} +{"action":"set","node":{"key":"/apisix/upstreams/unwanted","value":{"hash_on":"vars","id":"unwanted","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}} @@ -134,7 +134,7 @@ __DATA__ } } --- response_body -{"action":"compareAndSwap","node":{"key":"/apisix/upstreams/unwanted","value":{"hash_on":"vars","id":"unwanted","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}} +{"action":"compareAndSwap","node":{"key":"/apisix/upstreams/unwanted","value":{"hash_on":"vars","id":"unwanted","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}} @@ -161,7 +161,7 @@ __DATA__ } } --- response_body -{"action":"get","count":"1","node":{"key":"/apisix/upstreams/unwanted","value":{"hash_on":"vars","id":"unwanted","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","type":"roundrobin"}}} +{"action":"get","count":"1","node":{"key":"/apisix/upstreams/unwanted","value":{"hash_on":"vars","id":"unwanted","nodes":{"127.0.0.1:8080":1},"pass_host":"pass","scheme":"http","type":"roundrobin"}}} diff --git a/t/grpc-proxy-test.sh b/t/grpc-proxy-test.sh index 5ca323400eb3..6e80d171ba08 100755 --- a/t/grpc-proxy-test.sh +++ b/t/grpc-proxy-test.sh @@ -18,6 +18,8 @@ set -ex +echo "127.0.0.1 test.com" | sudo tee -a /etc/hosts + #set ssl curl http://127.0.0.1:9080/apisix/admin/ssl/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' { @@ -28,6 +30,22 @@ curl http://127.0.0.1:9080/apisix/admin/ssl/1 -H 'X-API-KEY: edd1c9f034335f136f #test grpc proxy curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["POST"], + "uri": "/helloworld.Greeter/SayHello", + "upstream": { + "scheme": "grpc", + "type": "roundrobin", + "nodes": { + "127.0.0.1:50051": 1 + } + } +}' + +./build-cache/grpcurl -insecure -import-path ./build-cache/proto -proto helloworld.proto -d '{"name":"apisix"}' test.com:9443 helloworld.Greeter.SayHello | grep 'Hello apisix' + +# the old way +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' { "methods": ["POST"], "uri": "/helloworld.Greeter/SayHello", @@ -40,10 +58,23 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f1 } }' -echo "127.0.0.1 test.com" | sudo tee -a /etc/hosts +./build-cache/grpcurl -insecure -import-path ./build-cache/proto -proto helloworld.proto -d '{"name":"apisix"}' test.com:9443 helloworld.Greeter.SayHello | grep 'Hello apisix' -./build-cache/grpcurl -insecure -import-path ./build-cache/proto -proto helloworld.proto -d '{"name":"apisix"}' test.com:9443 helloworld.Greeter.SayHello +#test grpcs proxy +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["POST"], + "uri": "/helloworld.Greeter/SayHello", + "upstream": { + "scheme": "grpcs", + "type": "roundrobin", + "nodes": { + "127.0.0.1:50052": 1 + } + } +}' +./build-cache/grpcurl -insecure -import-path ./build-cache/proto -proto helloworld.proto -d '{"name":"apisix"}' test.com:9443 helloworld.Greeter.SayHello | grep 'Hello apisix' #delete test data curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X DELETE diff --git a/t/node/grpc-proxy.t b/t/node/grpc-proxy.t index 02ee65841816..2f868c680e27 100644 --- a/t/node/grpc-proxy.t +++ b/t/node/grpc-proxy.t @@ -51,13 +51,13 @@ __DATA__ upstreams: - id: 1 type: roundrobin + scheme: grpc nodes: "127.0.0.1:9088": 1 routes: - id: 1 methods: - POST - service_protocol: grpc uri: "/hello" upstream_id: 1 #END @@ -76,6 +76,58 @@ consumers: key-auth: key: user-key #END +routes: + - id: 1 + methods: + - POST + uri: "/hello" + plugins: + key-auth: + consumer-restriction: + whitelist: + - jack + upstream: + scheme: grpc + type: roundrobin + nodes: + "127.0.0.1:9088": 1 +#END +--- more_headers +apikey: user-key +--- error_code: 502 + + + +=== TEST 3: with upstream_id (old way) +--- apisix_yaml +upstreams: + - id: 1 + type: roundrobin + nodes: + "127.0.0.1:9088": 1 +routes: + - id: 1 + methods: + - POST + service_protocol: grpc + uri: "/hello" + upstream_id: 1 +#END +--- error_code: 502 +--- error_log +proxy request to 127.0.0.1:9088 + + + +=== TEST 4: with consummer (old way) +--- apisix_yaml +consumers: + - username: jack + id: jack + plugins: + key-auth: + key: user-key +#END routes: - id: 1 methods: diff --git a/t/plugin/grpc-transcode.t b/t/plugin/grpc-transcode.t index d87f07b5f8d5..2a1afa20e4e5 100644 --- a/t/plugin/grpc-transcode.t +++ b/t/plugin/grpc-transcode.t @@ -144,7 +144,6 @@ passed [[{ "methods": ["GET", "POST"], "uri": "/grpctest", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -153,6 +152,7 @@ passed } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -210,7 +210,7 @@ qr/\{"message":"Hello world"\}/ -=== TEST 8: wrong service protocol +=== TEST 8: wrong upstream scheme --- config location /t { content_by_lua_block { @@ -220,7 +220,6 @@ qr/\{"message":"Hello world"\}/ [[{ "methods": ["GET"], "uri": "/grpctest", - "service_protocol": "asf", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -229,6 +228,7 @@ qr/\{"message":"Hello world"\}/ } }, "upstream": { + "scheme": "asf", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -261,7 +261,6 @@ GET /t [[{ "methods": ["GET"], "uri": "/grpctest", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -270,6 +269,7 @@ GET /t } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:1970": 1 @@ -361,7 +361,6 @@ passed [[{ "methods": ["GET"], "uri": "/grpc_plus", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -371,6 +370,7 @@ passed } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -424,7 +424,6 @@ qr/\{"result":"#2251799813685261"\}/ [[{ "methods": ["GET"], "uri": "/grpc_deadline", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -434,6 +433,7 @@ qr/\{"result":"#2251799813685261"\}/ } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -476,7 +476,6 @@ qr/\{"message":"Hello apisix"\}/ [[{ "methods": ["GET"], "uri": "/grpc_delay", - "service_protocol": "grpc", "plugins": { "grpc-transcode": { "proto_id": "1", @@ -486,6 +485,7 @@ qr/\{"message":"Hello apisix"\}/ } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1 @@ -607,6 +607,7 @@ passed } }, "upstream": { + "scheme": "grpc", "type": "roundrobin", "nodes": { "127.0.0.1:50051": 1