feat(wasm): support getting request body

.github/workflows/build.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2.1.5
         with:
-          go-version: "1.15"
+          go-version: "1.17"
 
       - name: Cache deps
         uses: actions/cache@v2.1.7
@@ -79,7 +79,7 @@ jobs:
         run: sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3 libpcre3-dev libldap2-dev
 
       - name: Build wasm code
-        if: startsWith(matrix.os_name, 'linux_openresty')
+        if: matrix.os_name == 'linux_openresty'
         run: |
           export TINYGO_VER=0.20.0
           wget https://github.com/tinygo-org/tinygo/releases/download/v${TINYGO_VER}/tinygo_${TINYGO_VER}_amd64.deb 2>/dev/null

apisix/cli/ngx_tpl.lua

@@ -587,6 +587,10 @@ http {
             set $ctx_ref                     '';
             set $from_error_page             '';
 
+            {% if wasm then %}
+            set $wasm_process_req_body       '';
+            {% end %}
+
             # http server location configuration snippet starts
             {% if http_server_location_configuration_snippet then %}
             {* http_server_location_configuration_snippet *}

apisix/wasm.lua

@@ -63,30 +63,48 @@ end
 
 
 local function http_request_wrapper(self, conf, ctx)
+    local name = self.name
     local plugin_ctx, err = fetch_plugin_ctx(conf, ctx, self.plugin)
     if not plugin_ctx then
-        core.log.error("failed to fetch wasm plugin ctx: ", err)
+        core.log.error(name, ": failed to fetch wasm plugin ctx: ", err)
         return 503
     end
 
     local ok, err = wasm.on_http_request_headers(plugin_ctx)
     if not ok then
-        core.log.error("failed to run wasm plugin: ", err)
+        core.log.error(name, ": failed to run wasm plugin: ", err)
         return 503
     end
+
+    -- $wasm_process_req_body is predefined in ngx_tpl.lua
+    local handle_body = ctx.var.wasm_process_req_body
+    if handle_body ~= '' then
+        local body, err = core.request.get_body()
+        if err ~= nil then
+            core.log.error(name, ": failed to get request body: ", err)
+            return 503
+        end
+
+        local ok, err = wasm.on_http_request_body(plugin_ctx, body, true)
+        if not ok then
+            core.log.error(name, ": failed to run wasm plugin: ", err)
+            return 503
+        end
+    end
 end
 
 
 local function header_filter_wrapper(self, conf, ctx)
+    local name = self.name
     local plugin_ctx, err = fetch_plugin_ctx(conf, ctx, self.plugin)
     if not plugin_ctx then
-        core.log.error("failed to fetch wasm plugin ctx: ", err)
+        core.log.error(name, ": failed to fetch wasm plugin ctx: ", err)
         return 503
     end
 
     local ok, err = wasm.on_http_response_headers(plugin_ctx)
     if not ok then
-        core.log.error("failed to run wasm plugin: ", err)
+        core.log.error(name, ": failed to run wasm plugin: ", err)
         return 503
     end
 end

ci/centos7-ci.sh

@@ -50,7 +50,7 @@ install_dependencies() {
     # add go1.15 binary to the path
     mkdir build-cache
     # centos-7 ci runs on a docker container with the centos image on top of ubuntu host. Go is required inside the container.
-    cd build-cache/ && wget https://golang.org/dl/go1.15.linux-amd64.tar.gz && tar -xf go1.15.linux-amd64.tar.gz
+    cd build-cache/ && wget https://golang.org/dl/go1.17.linux-amd64.tar.gz && tar -xf go1.17.linux-amd64.tar.gz
     export PATH=$PATH:$(pwd)/go/bin
     cd ..
     # install and start grpc_server_example

docs/en/latest/wasm.md

@@ -101,6 +101,7 @@ Here is the mapping between Proxy WASM callbacks and APISIX's phases:
 * `proxy_on_configure`: run once there is not PluginContext for the new configuration.
 For example, when the first request hits the route which has WASM plugin configured.
 * `proxy_on_http_request_headers`: run in the access/rewrite phase, depends on the configuration of `http_request_phase`.
+* `proxy_on_http_request_body`: run in the same phase of `proxy_on_http_request_headers`. To run this callback, we need to set property `wasm_process_req_body` to non-empty value in `proxy_on_http_request_headers`. See `t/wasm/request-body/main.go` as an example.
 * `proxy_on_http_response_headers`: run in the header_filter phase.
 
 ## Example

t/APISIX.pm

@@ -217,6 +217,13 @@ if ($version =~ m/\/apisix-nginx-module/) {
 _EOC_
 }
 
+my $a6_ngx_vars = "";
+if ($version =~ m/\/apisix-nginx-module/) {
+    $a6_ngx_vars = <<_EOC_;
+    set \$wasm_process_req_body       '';
+_EOC_
+}
+
 add_block_preprocessor(sub {
     my ($block) = @_;
     my $wait_etcd_sync = $block->wait_etcd_sync // 0.1;
@@ -714,6 +721,7 @@ _EOC_
             set \$upstream_cache_key             '';
             set \$upstream_cache_bypass          '';
             set \$upstream_no_cache              '';
+            $a6_ngx_vars
 
             proxy_cache                         \$upstream_cache_zone;
             proxy_cache_valid                   any 10s;

t/plugin/grpc-web/go.mod

@@ -4,10 +4,9 @@ go 1.16
 
 require (
 	github.com/golang/protobuf v1.4.3
-	github.com/satori/go.uuid v1.2.0 // indirect
-	golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
+	github.com/satori/go.uuid v1.2.0
+	golang.org/x/net v0.0.0-20200822124328-c89045814202
 	golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
-	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/grpc v1.43.0
 )

t/plugin/grpc-web/go.sum

@@ -64,8 +64,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -75,12 +73,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -117,6 +112,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

t/wasm/go.mod

@@ -1,9 +1,9 @@
 module github.com/api7/wasm-nginx-module
 
-go 1.15
+go 1.17
 
 require (
-	github.com/tetratelabs/proxy-wasm-go-sdk v0.14.1-0.20210819090022-1e4e69881a31
+	github.com/tetratelabs/proxy-wasm-go-sdk v0.16.0
 	github.com/valyala/fastjson v1.6.3
 )
 

t/wasm/go.sum

@@ -1,15 +1,16 @@
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/tetratelabs/proxy-wasm-go-sdk v0.14.1-0.20210819090022-1e4e69881a31 h1:V3GXN5nayOdIU3NypbxVegGFCVGm78qOA8Q7wkeudy8=
-github.com/tetratelabs/proxy-wasm-go-sdk v0.14.1-0.20210819090022-1e4e69881a31/go.mod h1:qZ+4i6e2wHlhnhgpH0VG4QFzqd2BEvQbQFU0npt2e2k=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.16.0 h1:6xhDLV4DD2+q3Rs4CDh7cqo69rQ50XgCusv/58D44o4=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.16.0/go.mod h1:8CxNZJ+9yDEvNnAog384fC8j1tKNF0tTZevGjOuY9ds=
 github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
 github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

t/wasm/request-body.t

@@ -0,0 +1,152 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX;
+
+my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx';
+my $version = eval { `$nginx_binary -V 2>&1` };
+
+if ($version !~ m/\/apisix-nginx-module/) {
+    plan(skip_all => "apisix-nginx-module not installed");
+} else {
+    plan('no_plan');
+}
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+        $block->set_value("no_error_log", "[error]");
+    }
+
+    if (!defined $block->request) {
+        $block->set_value("request", "GET /t");
+    }
+
+    my $extra_yaml_config = <<_EOC_;
+wasm:
+    plugins:
+        - name: wasm-request-body
+          priority: 7997
+          file: t/wasm/request-body/main.go.wasm
+_EOC_
+    $block->set_value("extra_yaml_config", $extra_yaml_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: sanity
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "uri": "/hello",
+                    "upstream": {
+                        "type": "roundrobin",
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        }
+                    },
+                    "plugins": {
+                        "wasm-request-body": {
+                            "conf": "{\"processReqBody\":true, \"start\":1, \"size\":3}"
+                        }
+                    }
+                }]]
+            )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say(body)
+                return
+            end
+
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 2: hit
+--- request
+POST /hello
+hello
+--- grep_error_log eval
+qr/request get body: \w+/
+--- grep_error_log_out
+request get body: ell
+
+
+
+=== TEST 3: no body
+--- request
+POST /hello
+--- error_log
+error status returned by host: not found
+
+
+
+=== TEST 4: do not process body
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "uri": "/hello",
+                    "upstream": {
+                        "type": "roundrobin",
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        }
+                    },
+                    "plugins": {
+                        "wasm-request-body": {
+                            "conf": "{\"processReqBody\":false, \"start\":1, \"size\":3}"
+                        }
+                    }
+                }]]
+            )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say(body)
+                return
+            end
+
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 5: hit
+--- request
+POST /hello
+hello
+--- grep_error_log eval
+qr/request get body: \w+/
+--- grep_error_log_out