diff --git a/apisix/plugins/grpc-transcode/proto.lua b/apisix/plugins/grpc-transcode/proto.lua index de19be26bb97..e997b55ef877 100644 --- a/apisix/plugins/grpc-transcode/proto.lua +++ b/apisix/plugins/grpc-transcode/proto.lua @@ -14,25 +14,23 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- -local core = require("apisix.core") -local config_util = require("apisix.core.config_util") -local pb = require("pb") -local protoc = require("protoc") -local pcall = pcall -local ipairs = ipairs -local protos +local core = require("apisix.core") +local config_util = require("apisix.core.config_util") +local pb = require("pb") +local protoc = require("protoc") +local pcall = pcall +local ipairs = ipairs +local decode_base64 = ngx.decode_base64 +local protos local lrucache_proto = core.lrucache.new({ ttl = 300, count = 100 }) local proto_fake_file = "filename for loaded" -local function compile_proto(content) - -- clear pb state - pb.state(nil) - +local function compile_proto_text(content) protoc.reload() local _p = protoc.new() -- the loaded proto won't appears in _p.loaded without a file name after lua-protobuf=0.3.2, @@ -50,8 +48,6 @@ local function compile_proto(content) end local compiled = _p.loaded - -- fetch pb state - compiled.pb_state = pb.state(nil) local index = {} for _, s in ipairs(compiled[proto_fake_file].service or {}) do @@ -69,6 +65,54 @@ local function compile_proto(content) end +local function compile_proto_bin(content) + content = decode_base64(content) + if not content then + return nil + end + + -- pb.load doesn't return err + local ok = pb.load(content) + if not ok then + return nil + end + + local index = {} + for name, _, methods in pb.services() do + local method_index = {} + for _, m in ipairs(methods) do + method_index[m.name] = m + end + -- remove the prefix '.' + index[name:sub(2)] = method_index + end + + local compiled = {} + compiled[proto_fake_file] = {} + compiled[proto_fake_file].index = index + + return compiled +end + + +local function compile_proto(content) + -- clear pb state + pb.state(nil) + + local compiled, err = compile_proto_text(content) + if not compiled then + compiled = compile_proto_bin(content) + if not compiled then + return nil, err + end + end + + -- fetch pb state + compiled.pb_state = pb.state(nil) + return compiled +end + + local _M = { version = 0.1, compile_proto = compile_proto, diff --git a/apisix/plugins/grpc-transcode/util.lua b/apisix/plugins/grpc-transcode/util.lua index 18c9f7808a16..2a7bfb648e94 100644 --- a/apisix/plugins/grpc-transcode/util.lua +++ b/apisix/plugins/grpc-transcode/util.lua @@ -27,23 +27,26 @@ local type = type local _M = {version = 0.1} -function _M.find_method(protos, service, method) - local loaded = protos[proto_fake_file] - if not loaded or type(loaded) ~= "table" then +function _M.find_method(proto, service, method) + local loaded = proto[proto_fake_file] + if type(loaded) ~= "table" then + core.log.error("compiled proto not found") return nil end - if not loaded.index[service] or type(loaded.index[service]) ~= "table" then + if type(loaded.index[service]) ~= "table" then + core.log.error("compiled proto service not found") return nil end local res = loaded.index[service][method] if not res then + core.log.error("compiled proto method not found") return nil end -- restore pb state - pb.state(protos.pb_state) + pb.state(proto.pb_state) return res end diff --git a/docs/en/latest/plugins/grpc-transcode.md b/docs/en/latest/plugins/grpc-transcode.md index 2f576ef8d7f7..206169ecaa6f 100644 --- a/docs/en/latest/plugins/grpc-transcode.md +++ b/docs/en/latest/plugins/grpc-transcode.md @@ -29,7 +29,7 @@ HTTP(s) -> APISIX -> gRPC server #### Attributes -* `content`: `.proto` file's content. +* `content`: `.proto` or `.pb` file's content. #### Add a proto @@ -52,6 +52,67 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136 }' ``` +If your `.proto` file contains imports, or you want to combine multiple `.proto` files into a proto, +you can use `.pb` file to create the proto. + +Assumed we have a `.proto` called `proto/helloworld.proto`, which imports another proto file: + +```proto +syntax = "proto3"; + +package helloworld; +import "proto/import.proto"; +... +``` + +First of all, let's create a `.pb` file from `.proto` files: + +```shell +protoc --include_imports --descriptor_set_out=proto.pb proto/helloworld.proto +``` + +The output binary file `proto.pb` will contain both `helloworld.proto` and `import.proto`. + +Then we can submit the content of `proto.pb` as the `content` field of the proto. + +As the content is binary, we need to encode it in base64 first. Here we use a Python script to do it: + +```python +#!/usr/bin/env python +# coding: utf-8 +# save this file as upload_pb.py +import base64 +import sys +# sudo pip install requests +import requests + +if len(sys.argv) <= 1: + print("bad argument") + sys.exit(1) +with open(sys.argv[1], 'rb') as f: + content = base64.b64encode(f.read()) +id = sys.argv[2] +api_key = "edd1c9f034335f136f87ad84b625c8f1" # Change it + +reqParam = { + "content": content, +} +resp = requests.put("http://127.0.0.1:9080/apisix/admin/proto/" + id, json=reqParam, headers={ + "X-API-KEY": api_key, +}) +print(resp.status_code) +print(resp.text) +``` + +Create proto: + +```bash +chmod +x ./upload_pb.pb +./upload_pb.py proto.pb 1 +# 200 +# {"node":{"value":{"create_time":1643879753,"update_time":1643883085,"content":"CmgKEnByb3RvL2ltcG9ydC5wcm90bxIDcGtnIhoKBFVzZXISEgoEbmFtZRgBIAEoCVIEbmFtZSIeCghSZXNwb25zZRISCgRib2R5GAEgASgJUgRib2R5QglaBy4vcHJvdG9iBnByb3RvMwq9AQoPcHJvdG8vc3JjLnByb3RvEgpoZWxsb3dvcmxkGhJwcm90by9pbXBvcnQucHJvdG8iPAoHUmVxdWVzdBIdCgR1c2VyGAEgASgLMgkucGtnLlVzZXJSBHVzZXISEgoEYm9keRgCIAEoCVIEYm9keTI5CgpUZXN0SW1wb3J0EisKA1J1bhITLmhlbGxvd29ybGQuUmVxdWVzdBoNLnBrZy5SZXNwb25zZSIAQglaBy4vcHJvdG9iBnByb3RvMw=="},"key":"\/apisix\/proto\/1"},"action":"set"} +``` + ## Attribute List | Name | Type | Requirement | Default | Valid | Description | diff --git a/docs/zh/latest/plugins/grpc-transcode.md b/docs/zh/latest/plugins/grpc-transcode.md index d031943784d5..842072ee200c 100644 --- a/docs/zh/latest/plugins/grpc-transcode.md +++ b/docs/zh/latest/plugins/grpc-transcode.md @@ -27,7 +27,7 @@ HTTP(s) -> APISIX -> gRPC server ### 参数 -* `content`: `.proto` 文件的内容 +* `content`: `.proto` 或 `.pb` 文件的内容 ### 添加proto @@ -50,6 +50,67 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136 }' ``` +如果你的 `.proto` 文件包含 import,或者你想把多个 `.proto` 文件合并成一个 proto。 +你可以使用 `.pb` 文件来创建 proto。 + +假设我们有一个 `.proto` 叫 `proto/helloworld.proto`,它导入了另一个 proto 文件: + +```proto +syntax = "proto3"; + +package helloworld; +import "proto/import.proto"; +... +``` + +首先,让我们从 `.proto`文件创建一个`.pb`文件。 + +```shell +protoc --include_imports --descriptor_set_out=proto.pb proto/helloworld.proto +``` + +输出的二进制文件 `proto.pb` 将同时包含 `helloworld.proto` 和 `import.proto`。 + +然后我们可以将 `proto.pb` 的内容作为 proto 的 `content` 字段提交。 + +由于内容是二进制的,我们需要先对其进行 base64 编码。这里我们用一个 Python 脚本来做。 + +```python +#!/usr/bin/env python +# coding: utf-8 +# save this file as upload_pb.py +import base64 +import sys +# sudo pip install requests +import requests + +if len(sys.argv) <= 1: + print("bad argument") + sys.exit(1) +with open(sys.argv[1], 'rb') as f: + content = base64.b64encode(f.read()) +id = sys.argv[2] +api_key = "edd1c9f034335f136f87ad84b625c8f1" # Change it + +reqParam = { + "content": content, +} +resp = requests.put("http://127.0.0.1:9080/apisix/admin/proto/" + id, json=reqParam, headers={ + "X-API-KEY": api_key, +}) +print(resp.status_code) +print(resp.text) +``` + +创建proto: + +```bash +chmod +x ./upload_pb.pb +./upload_pb.py proto.pb 1 +# 200 +# {"node":{"value":{"create_time":1643879753,"update_time":1643883085,"content":"CmgKEnByb3RvL2ltcG9ydC5wcm90bxIDcGtnIhoKBFVzZXISEgoEbmFtZRgBIAEoCVIEbmFtZSIeCghSZXNwb25zZRISCgRib2R5GAEgASgJUgRib2R5QglaBy4vcHJvdG9iBnByb3RvMwq9AQoPcHJvdG8vc3JjLnByb3RvEgpoZWxsb3dvcmxkGhJwcm90by9pbXBvcnQucHJvdG8iPAoHUmVxdWVzdBIdCgR1c2VyGAEgASgLMgkucGtnLlVzZXJSBHVzZXISEgoEYm9keRgCIAEoCVIEYm9keTI5CgpUZXN0SW1wb3J0EisKA1J1bhITLmhlbGxvd29ybGQuUmVxdWVzdBoNLnBrZy5SZXNwb25zZSIAQglaBy4vcHJvdG9iBnByb3RvMw=="},"key":"\/apisix\/proto\/1"},"action":"set"} +``` + ## 参数列表 | 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index bfc07766e368..c4591d57fe18 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -46,7 +46,7 @@ dependencies = { "lua-resty-session = 2.24", "opentracing-openresty = 0.1", "lua-resty-radixtree = 2.8.1", - "lua-protobuf = 0.3.3", + "api7-lua-protobuf = 0.1.0", "lua-resty-openidc = 1.7.2-1", "luafilesystem = 1.7.0-2", "api7-lua-tinyyaml = 0.4.2", diff --git a/t/grpc_server_example b/t/grpc_server_example index f7ee318f701e..5e74be697f24 160000 --- a/t/grpc_server_example +++ b/t/grpc_server_example @@ -1 +1 @@ -Subproject commit f7ee318f701e04bf21bf000baab539f5a8bc7eaa +Subproject commit 5e74be697f24151648be1712fce0ab2fdd0ec964 diff --git a/t/plugin/grpc-transcode2.t b/t/plugin/grpc-transcode2.t index 7a48077fc933..93b84b2097b0 100644 --- a/t/plugin/grpc-transcode2.t +++ b/t/plugin/grpc-transcode2.t @@ -230,3 +230,155 @@ location /t { {"message":"Hello world, name: John"} --- error_log failed to encode request data to protobuf + + + +=== TEST 6: set binary rule +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + + local content = t.read_file("t/grpc_server_example/proto.pb") + local data = {content = ngx.encode_base64(content)} + local code, body = t.test('/apisix/admin/proto/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/grpctest", + "plugins": { + "grpc-transcode": { + "proto_id": "1", + "service": "helloworld.TestImport", + "method": "Run" + } + }, + "upstream": { + "scheme": "grpc", + "type": "roundrobin", + "nodes": { + "127.0.0.1:50051": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 7: hit route +--- request +POST /grpctest +{"body":"world","user":{"name":"Hello"}} +--- more_headers +Content-Type: application/json +--- response_body chomp +{"body":"Hello world"} + + + +=== TEST 8: service/method not found +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/service_not_found", + "plugins": { + "grpc-transcode": { + "proto_id": "1", + "service": "helloworld.TestImportx", + "method": "Run" + } + }, + "upstream": { + "scheme": "grpc", + "type": "roundrobin", + "nodes": { + "127.0.0.1:50051": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + [[{ + "uri": "/method_not_found", + "plugins": { + "grpc-transcode": { + "proto_id": "1", + "service": "helloworld.TestImport", + "method": "Runx" + } + }, + "upstream": { + "scheme": "grpc", + "type": "roundrobin", + "nodes": { + "127.0.0.1:50051": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 9: hit route +--- request +POST /service_not_found +{"body":"world","user":{"name":"Hello"}} +--- more_headers +Content-Type: application/json +--- error_log +Undefined service method +--- error_code: 503 + + + +=== TEST 10: hit route +--- request +POST /method_not_found +{"body":"world","user":{"name":"Hello"}} +--- more_headers +Content-Type: application/json +--- error_log +Undefined service method +--- error_code: 503