feat(grpc-transcode): support .pb file (#6264)

apisix/plugins/grpc-transcode/proto.lua

@@ -14,25 +14,23 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
-local core        = require("apisix.core")
-local config_util = require("apisix.core.config_util")
-local pb          = require("pb")
-local protoc      = require("protoc")
-local pcall       = pcall
-local ipairs      = ipairs
-local protos
+local core          = require("apisix.core")
+local config_util   = require("apisix.core.config_util")
+local pb            = require("pb")
+local protoc        = require("protoc")
+local pcall         = pcall
+local ipairs        = ipairs
+local decode_base64 = ngx.decode_base64
 
 
+local protos
 local lrucache_proto = core.lrucache.new({
     ttl = 300, count = 100
 })
 
 local proto_fake_file = "filename for loaded"
 
-local function compile_proto(content)
-    -- clear pb state
-    pb.state(nil)
-
+local function compile_proto_text(content)
     protoc.reload()
     local _p  = protoc.new()
     -- the loaded proto won't appears in _p.loaded without a file name after lua-protobuf=0.3.2,
@@ -50,8 +48,6 @@ local function compile_proto(content)
     end
 
     local compiled = _p.loaded
-    -- fetch pb state
-    compiled.pb_state = pb.state(nil)
 
     local index = {}
     for _, s in ipairs(compiled[proto_fake_file].service or {}) do
@@ -69,6 +65,54 @@ local function compile_proto(content)
 end
 
 
+local function compile_proto_bin(content)
+    content = decode_base64(content)
+    if not content then
+        return nil
+    end
+
+    -- pb.load doesn't return err
+    local ok = pb.load(content)
+    if not ok then
+        return nil
+    end
+
+    local index = {}
+    for name, _, methods in pb.services() do
+        local method_index = {}
+        for _, m in ipairs(methods) do
+            method_index[m.name] = m
+        end
+        -- remove the prefix '.'
+        index[name:sub(2)] = method_index
+    end
+
+    local compiled = {}
+    compiled[proto_fake_file] = {}
+    compiled[proto_fake_file].index = index
+
+    return compiled
+end
+
+
+local function compile_proto(content)
+    -- clear pb state
+    pb.state(nil)
+
+    local compiled, err = compile_proto_text(content)
+    if not compiled then
+        compiled = compile_proto_bin(content)
+        if not compiled then
+            return nil, err
+        end
+    end
+
+    -- fetch pb state
+    compiled.pb_state = pb.state(nil)
+    return compiled
+end
+
+
 local _M = {
     version = 0.1,
     compile_proto = compile_proto,

apisix/plugins/grpc-transcode/util.lua

@@ -27,23 +27,26 @@ local type              = type
 local _M = {version = 0.1}
 
 
-function _M.find_method(protos, service, method)
-    local loaded = protos[proto_fake_file]
-    if not loaded or type(loaded) ~= "table" then
+function _M.find_method(proto, service, method)
+    local loaded = proto[proto_fake_file]
+    if type(loaded) ~= "table" then
+        core.log.error("compiled proto not found")
         return nil
     end
 
-    if not loaded.index[service] or type(loaded.index[service]) ~= "table" then
+    if type(loaded.index[service]) ~= "table" then
+        core.log.error("compiled proto service not found")
         return nil
     end
 
     local res = loaded.index[service][method]
     if not res then
+        core.log.error("compiled proto method not found")
         return nil
     end
 
     -- restore pb state
-    pb.state(protos.pb_state)
+    pb.state(proto.pb_state)
     return res
 end
 

docs/en/latest/plugins/grpc-transcode.md

@@ -29,7 +29,7 @@ HTTP(s) -> APISIX -> gRPC server
 
 #### Attributes
 
-* `content`: `.proto` file's content.
+* `content`: `.proto` or `.pb` file's content.
 
 #### Add a proto
 
@@ -52,6 +52,67 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136
 }'
 ```
 
+If your `.proto` file contains imports, or you want to combine multiple `.proto` files into a proto,
+you can use `.pb` file to create the proto.
+
+Assumed we have a `.proto` called `proto/helloworld.proto`, which imports another proto file:
+
+```proto
+syntax = "proto3";
+
+package helloworld;
+import "proto/import.proto";
+...
+```
+
+First of all, let's create a `.pb` file from `.proto` files:
+
+```shell
+protoc --include_imports --descriptor_set_out=proto.pb proto/helloworld.proto
+```
+
+The output binary file `proto.pb` will contain both `helloworld.proto` and `import.proto`.
+
+Then we can submit the content of `proto.pb` as the `content` field of the proto.
+
+As the content is binary, we need to encode it in base64 first. Here we use a Python script to do it:
+
+```python
+#!/usr/bin/env python
+# coding: utf-8
+# save this file as upload_pb.py
+import base64
+import sys
+# sudo pip install requests
+import requests
+
+if len(sys.argv) <= 1:
+    print("bad argument")
+    sys.exit(1)
+with open(sys.argv[1], 'rb') as f:
+    content = base64.b64encode(f.read())
+id = sys.argv[2]
+api_key = "edd1c9f034335f136f87ad84b625c8f1" # Change it
+
+reqParam = {
+    "content": content,
+}
+resp = requests.put("http://127.0.0.1:9080/apisix/admin/proto/" + id, json=reqParam, headers={
+    "X-API-KEY": api_key,
+})
+print(resp.status_code)
+print(resp.text)
+```
+
+Create proto:
+
+```bash
+chmod +x ./upload_pb.pb
+./upload_pb.py proto.pb 1
+# 200
+# {"node":{"value":{"create_time":1643879753,"update_time":1643883085,"content":"CmgKEnByb3RvL2ltcG9ydC5wcm90bxIDcGtnIhoKBFVzZXISEgoEbmFtZRgBIAEoCVIEbmFtZSIeCghSZXNwb25zZRISCgRib2R5GAEgASgJUgRib2R5QglaBy4vcHJvdG9iBnByb3RvMwq9AQoPcHJvdG8vc3JjLnByb3RvEgpoZWxsb3dvcmxkGhJwcm90by9pbXBvcnQucHJvdG8iPAoHUmVxdWVzdBIdCgR1c2VyGAEgASgLMgkucGtnLlVzZXJSBHVzZXISEgoEYm9keRgCIAEoCVIEYm9keTI5CgpUZXN0SW1wb3J0EisKA1J1bhITLmhlbGxvd29ybGQuUmVxdWVzdBoNLnBrZy5SZXNwb25zZSIAQglaBy4vcHJvdG9iBnByb3RvMw=="},"key":"\/apisix\/proto\/1"},"action":"set"}
+```
+
 ## Attribute List
 
 | Name      | Type                                                                           | Requirement | Default | Valid | Description                      |

docs/zh/latest/plugins/grpc-transcode.md

@@ -27,7 +27,7 @@ HTTP(s) -> APISIX -> gRPC server
 
 ### 参数
 
-* `content`: `.proto` 文件的内容
+* `content`: `.proto` 或 `.pb` 文件的内容
 
 ### 添加proto
 
@@ -50,6 +50,67 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136
 }'
 ```
 
+如果你的 `.proto` 文件包含 import，或者你想把多个 `.proto` 文件合并成一个 proto。
+你可以使用 `.pb` 文件来创建 proto。
+
+假设我们有一个 `.proto` 叫 `proto/helloworld.proto`，它导入了另一个 proto 文件：
+
+```proto
+syntax = "proto3";
+
+package helloworld;
+import "proto/import.proto";
+...
+```
+
+首先，让我们从 `.proto`文件创建一个`.pb`文件。
+
+```shell
+protoc --include_imports --descriptor_set_out=proto.pb proto/helloworld.proto
+```
+
+输出的二进制文件 `proto.pb` 将同时包含 `helloworld.proto` 和 `import.proto`。
+
+然后我们可以将 `proto.pb` 的内容作为 proto 的 `content` 字段提交。
+
+由于内容是二进制的，我们需要先对其进行 base64 编码。这里我们用一个 Python 脚本来做。
+
+```python
+#!/usr/bin/env python
+# coding: utf-8
+# save this file as upload_pb.py
+import base64
+import sys
+# sudo pip install requests
+import requests
+
+if len(sys.argv) <= 1:
+    print("bad argument")
+    sys.exit(1)
+with open(sys.argv[1], 'rb') as f:
+    content = base64.b64encode(f.read())
+id = sys.argv[2]
+api_key = "edd1c9f034335f136f87ad84b625c8f1" # Change it
+
+reqParam = {
+    "content": content,
+}
+resp = requests.put("http://127.0.0.1:9080/apisix/admin/proto/" + id, json=reqParam, headers={
+    "X-API-KEY": api_key,
+})
+print(resp.status_code)
+print(resp.text)
+```
+
+创建proto：
+
+```bash
+chmod +x ./upload_pb.pb
+./upload_pb.py proto.pb 1
+# 200
+# {"node":{"value":{"create_time":1643879753,"update_time":1643883085,"content":"CmgKEnByb3RvL2ltcG9ydC5wcm90bxIDcGtnIhoKBFVzZXISEgoEbmFtZRgBIAEoCVIEbmFtZSIeCghSZXNwb25zZRISCgRib2R5GAEgASgJUgRib2R5QglaBy4vcHJvdG9iBnByb3RvMwq9AQoPcHJvdG8vc3JjLnByb3RvEgpoZWxsb3dvcmxkGhJwcm90by9pbXBvcnQucHJvdG8iPAoHUmVxdWVzdBIdCgR1c2VyGAEgASgLMgkucGtnLlVzZXJSBHVzZXISEgoEYm9keRgCIAEoCVIEYm9keTI5CgpUZXN0SW1wb3J0EisKA1J1bhITLmhlbGxvd29ybGQuUmVxdWVzdBoNLnBrZy5SZXNwb25zZSIAQglaBy4vcHJvdG9iBnByb3RvMw=="},"key":"\/apisix\/proto\/1"},"action":"set"}
+```
+
 ## 参数列表
 
 | 名称      | 类型                                                                       | 必选项 | 默认值 | 有效值 | 描述                       |

rockspec/apisix-master-0.rockspec

@@ -46,7 +46,7 @@ dependencies = {
     "lua-resty-session = 2.24",
     "opentracing-openresty = 0.1",
     "lua-resty-radixtree = 2.8.1",
-    "lua-protobuf = 0.3.3",
+    "api7-lua-protobuf = 0.1.0",
     "lua-resty-openidc = 1.7.2-1",
     "luafilesystem = 1.7.0-2",
     "api7-lua-tinyyaml = 0.4.2",