json_body, form_body

.github/workflows/docker.yml

@@ -23,7 +23,7 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Test
-      run: go test -v -timeout 120s -race ./...
+      run: go test -v -timeout 300s -race ./...
 
     - name: Build and push edge docker image
       if: github.event_name == 'push'

.github/workflows/go.yml

@@ -30,7 +30,7 @@ jobs:
       run: go vet ./...
 
     - name: Test
-      run: go test -v -timeout 120s -race ./...
+      run: go test -v -timeout 300s -race ./...
 
     - name: Build binary
       run: go build -v -o couper .

Makefile

@@ -2,18 +2,18 @@ image:
 	docker build -t avenga/couper:latest .
 
 test:
-	go test -v -short -race -timeout 120s ./...
+	go test -v -short -race -timeout 300s ./...
 
 test-docker:
-	docker run --rm -v $(CURDIR):/go/app -w /go/app  golang sh -c "go test -short -count 10 -v -timeout 120s -race ./..."
+	docker run --rm -v $(CURDIR):/go/app -w /go/app  golang sh -c "go test -short -count 10 -v -timeout 300s -race ./..."
 
 test-coverage:
-	go test -short -timeout 120s -covermode=count -coverprofile=ac.coverage ./accesscontrol
-	go test -short -timeout 120s -covermode=count -coverprofile=eval.coverage ./eval
-	go test -short -timeout 120s -covermode=count -coverprofile=config.coverage ./config
-	go test -short -timeout 120s -covermode=count -coverprofile=handler.coverage ./handler
-	go test -short -timeout 120s -covermode=count -coverprofile=server.coverage ./server
-	go test -short -timeout 120s -covermode=count -coverprofile=main.coverage ./
+	go test -short -timeout 300s -covermode=count -coverprofile=ac.coverage ./accesscontrol
+	go test -short -timeout 300s -covermode=count -coverprofile=eval.coverage ./eval
+	go test -short -timeout 300s -covermode=count -coverprofile=config.coverage ./config
+	go test -short -timeout 300s -covermode=count -coverprofile=handler.coverage ./handler
+	go test -short -timeout 300s -covermode=count -coverprofile=server.coverage ./server
+	go test -short -timeout 300s -covermode=count -coverprofile=main.coverage ./
 	$(MAKE) test-coverage-show
 
 test-coverage-show:

config/backend.go

@@ -59,6 +59,10 @@ func (b Backend) Schema(inline bool) *hcl.BodySchema {
 }
 
 func newBackendSchema(schema *hcl.BodySchema, body hcl.Body) *hcl.BodySchema {
+	if body == nil {
+		return schema
+	}
+
 	for i, block := range schema.Blocks {
 		// Inline backend block MAY have no label.
 		if block.Type == "backend" && len(block.LabelNames) > 0 {

config/configload/load.go

@@ -313,6 +313,22 @@ func refineEndpoints(definedBackends Backends, endpoints config.Endpoints) error
 			if diags.HasErrors() {
 				return diags
 			}
+
+			_, existsBody := content.Attributes["body"]
+			_, existsFormBody := content.Attributes["form_body"]
+			_, existsJsonBody := content.Attributes["json_body"]
+			if existsBody && existsFormBody || existsBody && existsJsonBody || existsFormBody && existsJsonBody {
+				rangeAttr := "body"
+				if !existsBody {
+					rangeAttr = "form_body"
+				}
+				return hcl.Diagnostics{&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "request can only have one of body, form_body or json_body attributes",
+					Subject:  &content.Attributes[rangeAttr].Range,
+				}}
+			}
+
 			renameAttribute(content, "headers", "set_request_headers")
 			renameAttribute(content, "query_params", "set_query_params")
 
@@ -327,6 +343,19 @@ func refineEndpoints(definedBackends Backends, endpoints config.Endpoints) error
 			endpoint.Requests = append(endpoint.Requests, reqConfig)
 		}
 
+		if endpoint.Response != nil {
+			content, _, _ := endpoint.Response.HCLBody().PartialContent(config.ResponseInlineSchema)
+			_, existsBody := content.Attributes["body"]
+			_, existsJsonBody := content.Attributes["json_body"]
+			if existsBody && existsJsonBody {
+				return hcl.Diagnostics{&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "response can only have one of body or json_body attributes",
+					Subject:  &content.Attributes["body"].Range,
+				}}
+			}
+		}
+
 		names := map[string]struct{}{}
 		unique := map[string]struct{}{}
 		itemRange := endpoint.Remain.MissingItemRange()

config/request.go

@@ -40,6 +40,8 @@ func (r Request) Schema(inline bool) *hcl.BodySchema {
 	type Inline struct {
 		Backend     *Backend             `hcl:"backend,block"`
 		Body        string               `hcl:"body,optional"`
+		FormBody    string               `hcl:"form_body,optional"`
+		JsonBody    string               `hcl:"json_body,optional"`
 		Headers     map[string]string    `hcl:"headers,optional"`
 		Method      string               `hcl:"method,optional"`
 		QueryParams map[string]cty.Value `hcl:"query_params,optional"`

config/response.go

@@ -31,9 +31,10 @@ func (r Response) Schema(inline bool) *hcl.BodySchema {
 	}
 
 	type Inline struct {
-		Body    string            `hcl:"body,optional"`
-		Headers map[string]string `hcl:"headers,optional"`
-		Status  int               `hcl:"status,optional"`
+		Body     string            `hcl:"body,optional"`
+		JsonBody string            `hcl:"json_body,optional"`
+		Headers  map[string]string `hcl:"headers,optional"`
+		Status   int               `hcl:"status,optional"`
 	}
 
 	schema, _ := gohcl.ImpliedBodySchema(&Inline{})

docs/README.md

@@ -205,7 +205,7 @@ since these references get evaluated at start.
 | `cookies.<name>`          | Value from `Cookie` request header for requested key (&#9888; last wins!) |
 | `query.<name>`            | Query parameter values (&#9888; last wins!) |
 | `path_params.<name>`      | Value from a named path parameter defined within an endpoint path label |
-| `post.<name>`             | Post form parameter |
+| `form_body.<name>`        | Parameter in a `application/x-www-form-urlencoded` body |
 | `json_body.<name>`        | Access json decoded object properties. Media type must be `application/json`. |
 | `ctx.<name>.<property_name>` | Request context containing claims from JWT used for [Access Control](#access-control) or information from a SAML assertion, `<name>` being the [JWT Block's](#jwt-block) or [SAML Block's](#saml-block) label and `property_name` being the claim's or assertion information's name |
 
@@ -219,7 +219,7 @@ since these references get evaluated at start.
 | `headers.<name>`          | HTTP request header value for requested lower-case key |
 | `cookies.<name>`          | Value from `Cookie` request header for requested key (&#9888; last wins!) |
 | `query.<name>`            | Query parameter values (&#9888; last wins!) |
-| `post.<name>`             | Post form parameter |
+| `form_body.<name>`        | Parameter in a `application/x-www-form-urlencoded` body |
 | `ctx.<name>.<property_name>` | Request context containing claims from JWT used for [Access Control](#access-control) or information from a SAML assertion, `<name>` being the [JWT Block's](#jwt-block) or [SAML Block's](#saml-block) label and `property_name` being the claim's or assertion information's name |
 | `url`                     | Backend origin URL |
 
@@ -437,7 +437,7 @@ produce an explicit or implicit client response.
 | [Request Block(s)](#request-block) |  |
 | [Response Block](#response-block)  |  |
 | **Attributes**                     | **Description** |
-| `request_body_limit`               | <ul><li>Optional.</li><li>Configures the maximum buffer size while accessing `req.post` or `req.json_body` content.</li><li>Valid units are: `KiB, MiB, GiB`.</li><li>Default limit is `64MiB`.</li></ul> |
+| `request_body_limit`               | <ul><li>Optional.</li><li>Configures the maximum buffer size while accessing `req.form_body` or `req.json_body` content.</li><li>Valid units are: `KiB, MiB, GiB`.</li><li>Default limit is `64MiB`.</li></ul> |
 | `path`                             | <ul><li>Optional.</li><li>Changeable part of the upstream URL.</li><li>Changes the path suffix of the outgoing request.</li></ul> |
 | `access_control`                   | <ul><li>Optional.</li><li>Sets predefined [Access Control](#access-control) for current `Endpoint Block` context.</li><li>*Example:* `access_control = ["foo"]`</li></ul> |
 | [Modifier](#modifier)              | <ul><li>Optional.</li><li>All [Modifier](#modifier).</li></ul> |
@@ -474,7 +474,9 @@ The `request` block creates and executes a request to a backend service.
 | **Attributes**                                      | **Description** |
 | [Backend Block Reference](#backend-block-reference) | <ul><li>&#9888; Mandatory if no [Backend Block](#backend-block) is defined.</li><li>References or refines a [Backend Block](#backend-block).</li></ul> |
 | `url`                                               | <ul><li>Optional.</li><li>If defined, the host part of the URL must be the same as the `origin` attribute of the used [Backend Block](#backend-block) or [Backend Block Reference](#backend-block-reference) (if defined).</li></ul> |
-| `body`                                              | <ul><li>String.</li><li>Optional.</li></ul> |
+| `body`                                              | <ul><li>String.</li><li>Optional. Creates implicit default `Content-Type: text/plain` header field.</li></ul> |
+| `json_body`                                         | <ul><li>null, Boolean, Number, String, Object, or Tuple.</li><li>Optional. Creates implicit default `Content-Type: application/json` header field.</li></ul> |
+| `form_body`                                         | <ul><li>Object.</li><li>Optional. Creates implicit default `Content-Type: application/x-www-form-urlencoded` header field.</li></ul> |
 | `method`                                            | <ul><li>String.</li><li>Optional.</li><li>Default `GET`.</li></ul> |
 | `headers`                                           | <ul><li>Optional.</li><li>Same as `set_request_headers` in [Request Header](#request-header).</li></ul> |
 | `query_params`                                      | <ul><li>Optional.</li><li>Same as `set_query_params` in [Query Parameter](#query-parameter).</li></ul> |
@@ -488,7 +490,8 @@ The `response` block creates and sends a client response.
 | *context*      | [Endpoint Block](#endpoint-block). |
 | *label*        | Not implemented. |
 | **Attributes** | **Description** |
-| `body`         | <ul><li>String.</li><li>Optional.</li></ul> |
+| `body`         | <ul><li>String.</li><li>Optional. Creates implicit default `Content-Type: text/plain` header field.</li></ul> |
+| `json_body`    | <ul><li>null, Boolean, Number, String, Object, or Tuple.</li><li>Optional. Creates implicit default `Content-Type: application/json` header field.</li></ul> |
 | `status`       | <ul><li>HTTP status code.</li><li>Optional.</li><li>Default `200`.</li></ul> |
 | `headers`      | <ul><li>Optional.</li><li>Same as `set_response_headers` in [Request Header](#response-header).</li></ul> |
 

eval/buffer.go

@@ -30,7 +30,7 @@ func (i BufferOption) GoString() string {
 	return strings.Join(result, "|")
 }
 
-// MustBuffer determines if any of the hcl.bodies makes use of 'post' or 'json_body'.
+// MustBuffer determines if any of the hcl.bodies makes use of 'form_body' or 'json_body'.
 func MustBuffer(body hcl.Body) BufferOption {
 	result := BufferNone
 
@@ -52,17 +52,17 @@ func MustBuffer(body hcl.Body) BufferOption {
 			nameField := reflect.ValueOf(traversal[1]).FieldByName("Name")
 			name := nameField.String()
 			switch name {
+			case FormBody:
+				if rootName == ClientRequest {
+					result |= BufferRequest
+				}
 			case JsonBody:
 				switch rootName {
 				case ClientRequest:
 					result |= BufferRequest
 				case BackendResponse:
 					result |= BufferResponse
 				}
-			case Post:
-				if rootName == ClientRequest {
-					result |= BufferRequest
-				}
 			}
 		}
 	}

eval/context.go

@@ -104,12 +104,12 @@ func (c *Context) WithClientRequest(req *http.Request) *Context {
 	}
 
 	ctx.eval.Variables[ClientRequest] = cty.ObjectVal(ctxMap.Merge(ContextMap{
+		FormBody:  seetie.ValuesMapToValue(parseForm(req).PostForm),
 		ID:        cty.StringVal(id),
 		JsonBody:  seetie.MapToValue(parseReqJSON(req)),
 		Method:    cty.StringVal(req.Method),
 		Path:      cty.StringVal(req.URL.Path),
 		PathParam: seetie.MapToValue(pathParams),
-		Post:      seetie.ValuesMapToValue(parseForm(req).PostForm),
 		Query:     seetie.ValuesMapToValue(req.URL.Query()),
 		URL:       cty.StringVal(newRawURL(req.URL).String()),
 	}.Merge(newVariable(ctx.inner, req.Cookies(), req.Header))))
@@ -141,11 +141,11 @@ func (c *Context) WithBeresps(beresps ...*http.Response) *Context {
 			name = n
 		}
 		bereqs[name] = cty.ObjectVal(ContextMap{
-			Method: cty.StringVal(bereq.Method),
-			Path:   cty.StringVal(bereq.URL.Path),
-			Post:   seetie.ValuesMapToValue(parseForm(bereq).PostForm),
-			Query:  seetie.ValuesMapToValue(bereq.URL.Query()),
-			URL:    cty.StringVal(newRawURL(bereq.URL).String()),
+			FormBody: seetie.ValuesMapToValue(parseForm(bereq).PostForm),
+			Method:   cty.StringVal(bereq.Method),
+			Path:     cty.StringVal(bereq.URL.Path),
+			Query:    seetie.ValuesMapToValue(bereq.URL.Query()),
+			URL:      cty.StringVal(newRawURL(bereq.URL).String()),
 		}.Merge(newVariable(ctx.inner, bereq.Cookies(), bereq.Header)))
 
 		var jsonBody map[string]interface{}
@@ -214,12 +214,11 @@ const defaultMaxMemory = 32 << 20 // 32 MB
 // As Proxy we should not consume the request body.
 // Rewind body via GetBody method.
 func parseForm(r *http.Request) *http.Request {
-	if r.GetBody == nil {
+	if r.GetBody == nil || r.Form != nil {
 		return r
 	}
 	switch r.Method {
 	case http.MethodPut, http.MethodPatch, http.MethodPost:
-		r.Body, _ = r.GetBody() // rewind
 		_ = r.ParseMultipartForm(defaultMaxMemory)
 		r.Body, _ = r.GetBody() // reset
 	}
@@ -253,9 +252,8 @@ func parseReqJSON(req *http.Request) map[string]interface{} {
 		return nil
 	}
 
-	req.Body, _ = req.GetBody() // rewind
-	result := parseJSON(req.Body)
-	req.Body, _ = req.GetBody() // reset
+	body, _ := req.GetBody()
+	result := parseJSON(body)
 	return result
 }
 

eval/context_test.go

@@ -44,7 +44,7 @@ func TestNewHTTPContext(t *testing.T) {
 		want      http.Header
 	}{
 		{"Variables / POST", http.MethodPost, http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, bytes.NewBufferString(`user=hans`), "", baseCtx, `
-					post = req.post.user[0]
+					post = req.form_body.user[0]
 					method = req.method
 		`, http.Header{"post": {"hans"}, "method": {http.MethodPost}}},
 		{"Variables / Query", http.MethodGet, http.Header{"User-Agent": {"test/v1"}}, nil, "?name=peter", baseCtx, `

eval/http.go

@@ -3,13 +3,16 @@ package eval
 import (
 	"bytes"
 	"context"
+	er "errors"
 	"io"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function/stdlib"
 
 	"github.com/avenga/couper/config/meta"
 	"github.com/avenga/couper/config/request"
@@ -63,6 +66,12 @@ func SetGetBody(req *http.Request, bodyLimit int64) error {
 		req.GetBody = func() (io.ReadCloser, error) {
 			return io.NopCloser(bytes.NewBuffer(bodyBytes)), nil
 		}
+		// reset body initially, additional body reads which are not depending on http.Request
+		// internals like form parsing should just call GetBody() and use the returned reader.
+		req.Body, _ = req.GetBody()
+		// parsing form data now since they read/write request attributes which could be
+		// difficult with multiple routines later on.
+		parseForm(req)
 	}
 
 	return nil
@@ -265,6 +274,56 @@ func GetAttribute(ctx *hcl.EvalContext, content *hcl.BodyContent, name string) (
 	return seetie.ValueToString(val), nil
 }
 
+func GetBody(ctx *hcl.EvalContext, content *hcl.BodyContent) (string, string, error) {
+	attr, ok := content.Attributes["json_body"]
+	if ok {
+		val, err := attr.Expr.Value(ctx)
+		if err != nil {
+			return "", "", err
+		}
+
+		val, err1 := stdlib.JSONEncodeFunc.Call([]cty.Value{val})
+		if err1 != nil {
+			return "", "", err1
+		}
+
+		return val.AsString(), "application/json", nil
+	}
+
+	attr, ok = content.Attributes["form_body"]
+	if ok {
+		val, err := attr.Expr.Value(ctx)
+		if err != nil {
+			return "", "", err
+		}
+
+		if valType := val.Type(); !(valType.IsObjectType() || valType.IsMapType()) {
+			return "", "", er.New("value of form_body must be object")
+		}
+
+		data := url.Values{}
+		for k, v := range val.AsValueMap() {
+			for _, sv := range seetie.ValueToStringSlice(v) {
+				data.Add(k, sv)
+			}
+		}
+
+		return data.Encode(), "application/x-www-form-urlencoded", nil
+	}
+
+	attr, ok = content.Attributes["body"]
+	if ok {
+		val, err := attr.Expr.Value(ctx)
+		if err != nil {
+			return "", "", err
+		}
+
+		return seetie.ValueToString(val), "text/plain", nil
+	}
+
+	return "", "", nil
+}
+
 func SetHeader(val cty.Value, headerCtx http.Header) {
 	expMap := seetie.ValueToMap(val)
 	for key, v := range expMap {

eval/variables.go

@@ -11,14 +11,14 @@ const (
 	Cookies          = "cookies"
 	Endpoint         = "endpoint"
 	Environment      = "env"
+	FormBody         = "form_body"
 	Headers          = "headers"
 	HttpStatus       = "status"
 	ID               = "id"
 	JsonBody         = "json_body"
 	Method           = "method"
 	Path             = "path"
 	PathParam        = "path_params"
-	Post             = "post"
 	Query            = "query"
 	URL              = "url"
 )