From fab339ffbc5997bdb301e3a7c253e53404a7dd24 Mon Sep 17 00:00:00 2001 From: Marcel Ludwig Date: Thu, 1 Apr 2021 11:28:45 +0200 Subject: [PATCH 1/4] Fix json type assumption Use a more generic approach and convert to explicit cty types later on --- eval/context.go | 44 ++++++++++++++++------- server/http_integration_test.go | 62 +++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/eval/context.go b/eval/context.go index 4eba81de7..318c2956b 100644 --- a/eval/context.go +++ b/eval/context.go @@ -106,7 +106,7 @@ func (c *Context) WithClientRequest(req *http.Request) *Context { ctx.eval.Variables[ClientRequest] = cty.ObjectVal(ctxMap.Merge(ContextMap{ FormBody: seetie.ValuesMapToValue(parseForm(req).PostForm), ID: cty.StringVal(id), - JsonBody: seetie.MapToValue(parseReqJSON(req)), + JsonBody: parseReqJSON(req), Method: cty.StringVal(req.Method), Path: cty.StringVal(req.URL.Path), PathParam: seetie.MapToValue(pathParams), @@ -148,13 +148,13 @@ func (c *Context) WithBeresps(beresps ...*http.Response) *Context { URL: cty.StringVal(newRawURL(bereq.URL).String()), }.Merge(newVariable(ctx.inner, bereq.Cookies(), bereq.Header))) - var jsonBody map[string]interface{} + var jsonBody cty.Value if (ctx.bufferOption & BufferResponse) == BufferResponse { jsonBody = parseRespJSON(beresp) } resps[name] = cty.ObjectVal(ContextMap{ HttpStatus: cty.StringVal(strconv.Itoa(beresp.StatusCode)), - JsonBody: seetie.MapToValue(jsonBody), + JsonBody: jsonBody, }.Merge(newVariable(ctx.inner, beresp.Cookies(), beresp.Header))) } @@ -230,43 +230,61 @@ func isJSONMediaType(contentType string) bool { return m == "application/json" } -func parseJSON(r io.Reader) map[string]interface{} { +func parseJSON(r io.Reader) interface{} { if r == nil { return nil } - var result map[string]interface{} + b, err := ioutil.ReadAll(r) if err != nil { return nil } + + var result interface{} + _ = json.Unmarshal(b, &result) return result } -func parseReqJSON(req *http.Request) map[string]interface{} { +func parseReqJSON(req *http.Request) cty.Value { if req.GetBody == nil { - return nil + return cty.EmptyObjectVal } if !isJSONMediaType(req.Header.Get("Content-Type")) { - return nil + return cty.EmptyObjectVal } body, _ := req.GetBody() result := parseJSON(body) - return result + + return jsonToValue(result) } -func parseRespJSON(beresp *http.Response) map[string]interface{} { +func parseRespJSON(beresp *http.Response) cty.Value { if !isJSONMediaType(beresp.Header.Get("Content-Type")) { - return nil + return cty.EmptyObjectVal } buf := &bytes.Buffer{} - io.Copy(buf, beresp.Body) // TODO: err handling + _, err := io.Copy(buf, beresp.Body) + if err != nil { + return cty.EmptyObjectVal + } + // reset beresp.Body = NewReadCloser(bytes.NewBuffer(buf.Bytes()), beresp.Body) - return parseJSON(buf) + return jsonToValue(parseJSON(buf)) +} + +func jsonToValue(result interface{}) cty.Value { + switch result.(type) { + case map[string]interface{}: + return seetie.MapToValue(result.(map[string]interface{})) + case []interface{}: + return seetie.ListToValue(result.([]interface{})) + } + return cty.EmptyObjectVal } func newRawURL(u *url.URL) *url.URL { diff --git a/server/http_integration_test.go b/server/http_integration_test.go index 12ab7d6a9..9cb230bf7 100644 --- a/server/http_integration_test.go +++ b/server/http_integration_test.go @@ -1705,6 +1705,68 @@ func TestHTTPServer_Endpoint_Response_JSONBody_Evaluation(t *testing.T) { } } +func TestHTTPServer_Endpoint_Response_JSONBody_Array_Evaluation(t *testing.T) { + client := newClient() + + confPath := path.Join("testdata/integration/endpoint_eval/15_couper.hcl") + shutdown, _ := newCouper(confPath, test.New(t)) + defer shutdown() + + helper := test.New(t) + + content := `[1, 2, {"data": true}]` + + req, err := http.NewRequest(http.MethodGet, "http://example.com:8080/req?foo=bar", strings.NewReader(content)) + helper.Must(err) + req.Header.Set("User-Agent", "") + req.Header.Set("Content-Type", "application/json") + + res, err := client.Do(req) + helper.Must(err) + + resBytes, err := ioutil.ReadAll(res.Body) + helper.Must(err) + + _ = res.Body.Close() + + type Expectation struct { + JSONBody interface{} `json:"json_body"` + Headers test.Header `json:"headers"` + Method string `json:"method"` + Query url.Values `json:"query"` + Url string `json:"url"` + } + + var jsonResult Expectation + err = json.Unmarshal(resBytes, &jsonResult) + if err != nil { + t.Errorf("unmarshal json: %v: got:\n%s", err, string(resBytes)) + } + + exp := Expectation{ + Method: http.MethodGet, + JSONBody: []interface{}{ + 1, + 2, + map[string]interface{}{ + "data": true, + }, + }, + Headers: map[string]string{ + "content-length": strconv.Itoa(len(content)), + "content-type": "application/json", + }, + Query: map[string][]string{ + "foo": {"bar"}, + }, + Url: "/req", + } + + if fmt.Sprint(jsonResult) != fmt.Sprint(exp) { + t.Errorf("\nwant:\t%#v\ngot:\t%#v\npayload: %s", exp, jsonResult, string(resBytes)) + } +} + func TestHTTPServer_Endpoint_Evaluation_Inheritance(t *testing.T) { client := newClient() From f552d6e8c0ad7e97b81e71a3589e312dfa80b570 Mon Sep 17 00:00:00 2001 From: Marcel Ludwig Date: Thu, 1 Apr 2021 11:47:35 +0200 Subject: [PATCH 2/4] Use cty json to unmarshal all types --- eval/context.go | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/eval/context.go b/eval/context.go index 318c2956b..fe35f002f 100644 --- a/eval/context.go +++ b/eval/context.go @@ -3,7 +3,6 @@ package eval import ( "bytes" "context" - "encoding/json" "io" "io/ioutil" "mime" @@ -18,6 +17,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" "github.com/zclconf/go-cty/cty/function/stdlib" + ctyjson "github.com/zclconf/go-cty/cty/json" "github.com/avenga/couper/config/jwt" "github.com/avenga/couper/config/request" @@ -230,20 +230,26 @@ func isJSONMediaType(contentType string) bool { return m == "application/json" } -func parseJSON(r io.Reader) interface{} { +func parseJSON(r io.Reader) cty.Value { if r == nil { - return nil + return cty.NilVal } b, err := ioutil.ReadAll(r) if err != nil { - return nil + return cty.NilVal } - var result interface{} + impliedType, err := ctyjson.ImpliedType(b) + if err != nil { + return cty.NilVal + } - _ = json.Unmarshal(b, &result) - return result + val, err := ctyjson.Unmarshal(b, impliedType) + if err != nil { + return cty.NilVal + } + return val } func parseReqJSON(req *http.Request) cty.Value { @@ -256,9 +262,7 @@ func parseReqJSON(req *http.Request) cty.Value { } body, _ := req.GetBody() - result := parseJSON(body) - - return jsonToValue(result) + return parseJSON(body) } func parseRespJSON(beresp *http.Response) cty.Value { @@ -274,17 +278,7 @@ func parseRespJSON(beresp *http.Response) cty.Value { // reset beresp.Body = NewReadCloser(bytes.NewBuffer(buf.Bytes()), beresp.Body) - return jsonToValue(parseJSON(buf)) -} - -func jsonToValue(result interface{}) cty.Value { - switch result.(type) { - case map[string]interface{}: - return seetie.MapToValue(result.(map[string]interface{})) - case []interface{}: - return seetie.ListToValue(result.([]interface{})) - } - return cty.EmptyObjectVal + return parseJSON(buf) } func newRawURL(u *url.URL) *url.URL { From 750f5efe81cd5382fc6f8276924c9f9b9310cd93 Mon Sep 17 00:00:00 2001 From: Marcel Ludwig Date: Thu, 1 Apr 2021 12:16:16 +0200 Subject: [PATCH 3/4] Fixup handle null values --- eval/context_test.go | 4 ++-- internal/seetie/convert.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eval/context_test.go b/eval/context_test.go index e5e931723..33c143f3a 100644 --- a/eval/context_test.go +++ b/eval/context_test.go @@ -79,7 +79,7 @@ func TestNewHTTPContext(t *testing.T) { {"Variables / GET /w json body & null value", http.MethodGet, http.Header{"Content-Type": {"application/json"}}, bytes.NewBufferString(`{"json": null}`), "", baseCtx, ` method = req.method title = req.json_body.json - `, http.Header{"method": {http.MethodGet}, "title": {""}}}, + `, http.Header{"method": {http.MethodGet}, "title": nil}}, } for _, tt := range tests { @@ -116,7 +116,7 @@ func TestNewHTTPContext(t *testing.T) { result := seetie.ValueToStringSlice(cv) if !reflect.DeepEqual(v, result) { - t.Errorf("Expected %q, got: %#v", v, cv) + t.Errorf("Expected %q, got: %#v, type: %#v", v, result, cv) } } }) diff --git a/internal/seetie/convert.go b/internal/seetie/convert.go index fb36b3acf..73c873ad3 100644 --- a/internal/seetie/convert.go +++ b/internal/seetie/convert.go @@ -166,7 +166,7 @@ func CookiesToMapValue(cookies []*http.Cookie) cty.Value { func ValueToStringSlice(src cty.Value) []string { var l []string - if !src.IsKnown() { + if !src.IsKnown() || src.IsNull() { return l } From 51d74eecfc13f4cece762dd3230042872a60e82c Mon Sep 17 00:00:00 2001 From: Johannes Koch Date: Thu, 1 Apr 2021 13:49:04 +0200 Subject: [PATCH 4/4] uncommented waiting tests; changed one test to test empty array --- server/http_integration_test.go | 106 +++++++++++++++----------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/server/http_integration_test.go b/server/http_integration_test.go index 9cb230bf7..6bb25cbcd 100644 --- a/server/http_integration_test.go +++ b/server/http_integration_test.go @@ -1277,80 +1277,76 @@ func TestHTTPServer_request_bodies(t *testing.T) { }, }, }, - /* - { - "/request/json_body/dyn", - "true", - "application/json", - expectation{ - Body: "true", // currently: "{}" - Args: url.Values{}, - Method: "POST", - Headers: http.Header{ - "Content-Length": []string{"4"}, // currently: "2" - "Content-Type": []string{"application/json"}, - }, + { + "/request/json_body/dyn", + "true", + "application/json", + expectation{ + Body: "true", + Args: url.Values{}, + Method: "POST", + Headers: http.Header{ + "Content-Length": []string{"4"}, + "Content-Type": []string{"application/json"}, }, }, - { - "/request/json_body/dyn", - "1.23", - "application/json", - expectation{ - Body: "1.23", // currently: "{}" - Args: url.Values{}, - Method: "POST", - Headers: http.Header{ - "Content-Length": []string{"4"}, // currently: "2" - "Content-Type": []string{"application/json"}, - }, + }, + { + "/request/json_body/dyn", + "1.23", + "application/json", + expectation{ + Body: "1.23", + Args: url.Values{}, + Method: "POST", + Headers: http.Header{ + "Content-Length": []string{"4"}, + "Content-Type": []string{"application/json"}, }, }, - { - "/request/json_body/dyn", - "\"ab\"", - "application/json", - expectation{ - Body: "\"ab\"", // currently: "{}" - Args: url.Values{}, - Method: "POST", - Headers: http.Header{ - "Content-Length": []string{"4"}, // currently: "2" - "Content-Type": []string{"application/json"}, - }, + }, + { + "/request/json_body/dyn", + "\"ab\"", + "application/json", + expectation{ + Body: "\"ab\"", + Args: url.Values{}, + Method: "POST", + Headers: http.Header{ + "Content-Length": []string{"4"}, + "Content-Type": []string{"application/json"}, }, }, - */ + }, { "/request/json_body/dyn", - "{\"a\":3}", + "{\"a\":3,\"b\":[]}", "application/json", expectation{ - Body: "{\"a\":3}", + Body: "{\"a\":3,\"b\":[]}", Args: url.Values{}, Method: "POST", Headers: http.Header{ - "Content-Length": []string{"7"}, + "Content-Length": []string{"14"}, "Content-Type": []string{"application/json"}, }, }, }, - /* - { - "/request/json_body/dyn", - "[0,1]", - "application/json", - expectation{ - Body: "[0,1]", // currently: "{}" - Args: url.Values{}, - Method: "POST", - Headers: http.Header{ - "Content-Length": []string{"5"}, // currently: "2" - "Content-Type": []string{"application/json"}, - }, + { + "/request/json_body/dyn", + "[0,1]", + "application/json", + expectation{ + Body: "[0,1]", + Args: url.Values{}, + Method: "POST", + Headers: http.Header{ + "Content-Length": []string{"5"}, + "Content-Type": []string{"application/json"}, }, }, - */ + }, { "/request/form_body", "",