From b2926406fa025a7254b648f36db9d365f598a201 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Wed, 20 Jul 2022 16:08:07 +0930 Subject: [PATCH 1/6] x-pack/filebeat/input/httpjson: add transaction tracer --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 61 +++++++++++++++++++ go.mod | 1 + go.sum | 2 + .../docs/inputs/input-httpjson.asciidoc | 37 +++++++++++ .../filebeat/input/httpjson/config_request.go | 16 +++++ x-pack/filebeat/input/httpjson/input.go | 4 +- x-pack/filebeat/input/httpjson/request.go | 44 +++++++++++++ 8 files changed, 164 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7c5e88ee37ba..1725c78b4de9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -118,6 +118,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Optimize grok patterns in system.auth module pipeline. {pull}32360[32360] - Checkpoint module: add authentication operation outcome enrichment. {issue}32230[32230] {pull}32431[32431] - add documentation for decode_xml_wineventlog processor field mappings. {pull}32456[32456] +- httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 1e8922847604..b7122f652d11 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21901,6 +21901,36 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/gokrb5.v7@v7.5. limitations under the License. +-------------------------------------------------------------------------------- +Dependency : gopkg.in/natefinch/lumberjack.v2 +Version: v2.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/natefinch/lumberjack.v2@v2.0.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 Nate Finch + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + -------------------------------------------------------------------------------- Dependency : gopkg.in/yaml.v2 Version: v2.4.0 @@ -26087,6 +26117,37 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/trac limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/BurntSushi/toml +Version: v0.3.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/toml@v0.3.1/COPYING: + +The MIT License (MIT) + +Copyright (c) 2013 TOML authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/OneOfOne/xxhash Version: v1.2.2 diff --git a/go.mod b/go.mod index ef0ef2fed138..f8784a9391c0 100644 --- a/go.mod +++ b/go.mod @@ -212,6 +212,7 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.0.0 go.elastic.co/apm/v2 v2.0.0 go.mongodb.org/mongo-driver v1.5.1 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) require ( diff --git a/go.sum b/go.sum index e6107647f2d1..b433ca1f4858 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,7 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -2502,6 +2503,7 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index e181c8abeb26..5c172a306587 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -546,6 +546,43 @@ filebeat.inputs: value: '[[now (parseDuration "-1h")]]' ---- +[float] +==== `request.tracer.filename` + +It is possible to log httpjson requests and responses to a local file-system for debugging configurations. +This option is enabled by setting the `request.tracer.filename` value. Additional options are available to +tune log rotation behavior. + +Enabling this option compromises security and should only be used for debugging. + +[float] +==== `request.tracer.maxsize` + +This value sets the maximum size, in megabytes, the log file will reach before it is rotated. By default +logs are allowed to reach 1MB before rotation. + +[float] +==== `request.tracer.maxage` + +This specifies the number days to retain rotated log files. If it is not set, log files are retained +indefinitely. + +[float] +==== `request.tracer.maxbackups` + +The number of old logs to retain. If it is not set all old logs are retained subject to the `request.tracer.maxage` +setting. + +[float] +==== `request.tracer.localtime` + +Whether to use the host's local time rather that UTC for timestamping rotated log file names. + +[float] +==== `request.tracer.compress` + +This determines whether rotated logs should be gzip compressed. + [float] ==== `response.decode_as` diff --git a/x-pack/filebeat/input/httpjson/config_request.go b/x-pack/filebeat/input/httpjson/config_request.go index 2e2b2402f9f0..dcfda22ee1da 100644 --- a/x-pack/filebeat/input/httpjson/config_request.go +++ b/x-pack/filebeat/input/httpjson/config_request.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "gopkg.in/natefinch/lumberjack.v2" + "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/transport/httpcommon" ) @@ -100,6 +102,8 @@ type requestConfig struct { Transforms transformsConfig `config:"transforms"` Transport httpcommon.HTTPTransportSettings `config:",inline"` + + Tracer *lumberjack.Logger `config:"tracer"` } func (c *requestConfig) Validate() error { @@ -124,5 +128,17 @@ func (c *requestConfig) Validate() error { } } + if c.Tracer != nil { + if c.Tracer.Filename == "" { + return errors.New("request tracer must have a filename if used") + } + if c.Tracer.MaxSize == 0 { + // By default Lumberjack caps file sizes at 100MB which + // is excessive for a debugging logger, so default to 1MB + // which is the minimum. + c.Tracer.MaxSize = 1 + } + } + return nil } diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 84241a800a31..3b0d3a39e780 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -179,10 +179,10 @@ func newHTTPClient(ctx context.Context, config config, log *logp.Logger) (*httpC if err != nil { return nil, err } - return &httpClient{client: authClient, limiter: limiter}, nil + return &httpClient{client: authClient, limiter: limiter, tracer: newTracer(config.Request.Tracer)}, nil } - return &httpClient{client: client.StandardClient(), limiter: limiter}, nil + return &httpClient{client: client.StandardClient(), limiter: limiter, tracer: newTracer(config.Request.Tracer)}, nil } func checkRedirect(config *requestConfig, log *logp.Logger) func(*http.Request, []*http.Request) error { diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 17bece981b7e..9ae72d75393a 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -10,11 +10,15 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" + "net/http/httputil" "net/url" "strings" + "time" "github.com/PaesslerAG/jsonpath" + "gopkg.in/natefinch/lumberjack.v2" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/elastic-agent-libs/logp" @@ -32,6 +36,7 @@ func registerRequestTransforms() { type httpClient struct { client *http.Client limiter *rateLimiter + tracer *tracer } func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Response, error) { @@ -42,6 +47,7 @@ func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Respon return nil, fmt.Errorf("failed to execute http client.Do: %w", err) } defer resp.Body.Close() + c.tracer.trace(req, resp) // Read the whole resp.Body so we can release the connection. // This implementation is inspired by httputil.DumpResponse @@ -57,6 +63,44 @@ func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Respon return resp, nil } +// tracer is a lightweight transaction tracer for debugging httpjson configurations. +// It is not intended to be used for machine ingestion, but rather as a human aid. +type tracer struct { + sess int64 // sess is the unix time of the start of a session. + txn int // txn is a session-unique transaction id. + log *log.Logger +} + +func newTracer(logger *lumberjack.Logger) *tracer { + if logger == nil { + return nil + } + return &tracer{sess: time.Now().UnixMilli(), log: log.New(logger, "", log.LstdFlags)} +} + +func (t *tracer) trace(req *http.Request, resp *http.Response) { + if t == nil { + return + } + t.txn++ + b, err := httputil.DumpRequestOut(req, true) + if err != nil { + t.log.Print(err) + } else { + t.log.Printf("request %d.%d:\n\t%s", t.sess, t.txn, bytes.ReplaceAll(b, []byte{'\n'}, []byte("\n\t"))) + } + if resp == nil { + t.log.Printf("no response %d.%d", t.sess, t.txn) + return + } + b, err = httputil.DumpResponse(resp, true) + if err != nil { + t.log.Printf("response %d.%d: %v", t.sess, t.txn, err) + } else { + t.log.Printf("response %d.%d:\n\t%s", t.sess, t.txn, bytes.ReplaceAll(b, []byte{'\n'}, []byte("\n\t"))) + } +} + func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, error) { req := transformable{} req.setURL(rf.url) From 5c91223f39de65a5567d3f21b6fea02bd271922c Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Sat, 23 Jul 2022 08:46:50 +0930 Subject: [PATCH 2/6] fix body handling --- x-pack/filebeat/input/httpjson/request.go | 28 ++++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 9ae72d75393a..09a9768ec019 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -40,14 +40,15 @@ type httpClient struct { } func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Response, error) { + c.tracer.txnStart(req) resp, err := c.limiter.execute(stdCtx, func() (*http.Response, error) { return c.client.Do(req) }) + c.tracer.txnDone(resp) if err != nil { return nil, fmt.Errorf("failed to execute http client.Do: %w", err) } defer resp.Body.Close() - c.tracer.trace(req, resp) // Read the whole resp.Body so we can release the connection. // This implementation is inspired by httputil.DumpResponse @@ -66,9 +67,10 @@ func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Respon // tracer is a lightweight transaction tracer for debugging httpjson configurations. // It is not intended to be used for machine ingestion, but rather as a human aid. type tracer struct { - sess int64 // sess is the unix time of the start of a session. - txn int // txn is a session-unique transaction id. - log *log.Logger + sess int64 // sess is the unix time of the start of a session. + txn int // txn is a session-unique transaction id. + inTxn bool + log *log.Logger } func newTracer(logger *lumberjack.Logger) *tracer { @@ -78,22 +80,36 @@ func newTracer(logger *lumberjack.Logger) *tracer { return &tracer{sess: time.Now().UnixMilli(), log: log.New(logger, "", log.LstdFlags)} } -func (t *tracer) trace(req *http.Request, resp *http.Response) { +func (t *tracer) txnStart(req *http.Request) { if t == nil { return } + if t.inTxn { + t.log.Printf("no response %d.%d", t.sess, t.txn) + } t.txn++ + t.inTxn = true b, err := httputil.DumpRequestOut(req, true) if err != nil { t.log.Print(err) } else { t.log.Printf("request %d.%d:\n\t%s", t.sess, t.txn, bytes.ReplaceAll(b, []byte{'\n'}, []byte("\n\t"))) } +} + +func (t *tracer) txnDone(resp *http.Response) { + if t == nil { + return + } + if !t.inTxn { + t.log.Printf("missing request %d.%d", t.sess, t.txn) + } + t.inTxn = false if resp == nil { t.log.Printf("no response %d.%d", t.sess, t.txn) return } - b, err = httputil.DumpResponse(resp, true) + b, err := httputil.DumpResponse(resp, true) if err != nil { t.log.Printf("response %d.%d: %v", t.sess, t.txn, err) } else { From 77bae95446c89e8658bbe826f07346c4f737f8e4 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 26 Jul 2022 16:04:04 +0930 Subject: [PATCH 3/6] get resp before it is clobbered --- x-pack/filebeat/input/httpjson/request.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 09a9768ec019..40502022c224 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -40,11 +40,12 @@ type httpClient struct { } func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Response, error) { - c.tracer.txnStart(req) resp, err := c.limiter.execute(stdCtx, func() (*http.Response, error) { - return c.client.Do(req) + c.tracer.txnStart(req) + resp, err := c.client.Do(req) + c.tracer.txnDone(resp) + return resp, err }) - c.tracer.txnDone(resp) if err != nil { return nil, fmt.Errorf("failed to execute http client.Do: %w", err) } From 6728231a6dfaf5c0f21dd66f9b70480432600042 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 26 Jul 2022 16:04:31 +0930 Subject: [PATCH 4/6] remove redundant err check --- x-pack/filebeat/input/httpjson/rate_limiter.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/rate_limiter.go b/x-pack/filebeat/input/httpjson/rate_limiter.go index dd8c2298086f..1dce2fe26486 100644 --- a/x-pack/filebeat/input/httpjson/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/rate_limiter.go @@ -41,10 +41,6 @@ func newRateLimiterFromConfig(config *rateLimitConfig, log *logp.Logger) *rateLi func (r *rateLimiter) execute(ctx context.Context, f func() (*http.Response, error)) (*http.Response, error) { for { resp, err := f() - if err != nil { - return nil, err - } - if err != nil { return nil, fmt.Errorf("failed to read http.response.body: %w", err) } From 4b72b04132c38ceba806bacd55c200d2106b12e6 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 28 Jul 2022 12:40:19 +0930 Subject: [PATCH 5/6] x-pack/filebeat/input/httpjson/internal/httplog: new package for HTTP req/resp logging Co-authored-by: Andrew Kroh --- NOTICE.txt | 422 +++++++++--------- go.mod | 2 +- x-pack/filebeat/input/httpjson/input.go | 19 +- .../httpjson/internal/httplog/roundtripper.go | 200 +++++++++ x-pack/filebeat/input/httpjson/request.go | 63 +-- 5 files changed, 430 insertions(+), 276 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go diff --git a/NOTICE.txt b/NOTICE.txt index b7122f652d11..d495b49206b2 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -20244,6 +20244,217 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.0.0/LICENS limitations under the License. +-------------------------------------------------------------------------------- +Dependency : go.elastic.co/ecszap +Version: v1.0.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + -------------------------------------------------------------------------------- Dependency : go.elastic.co/go-licence-detector Version: v0.5.0 @@ -44519,217 +44730,6 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : go.elastic.co/ecszap -Version: v1.0.1 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2020 Elastic and contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -------------------------------------------------------------------------------- Dependency : go.elastic.co/fastjson Version: v1.1.0 diff --git a/go.mod b/go.mod index f8784a9391c0..9ddf3b12573f 100644 --- a/go.mod +++ b/go.mod @@ -159,7 +159,7 @@ require ( github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 github.com/xdg/scram v1.0.3 - go.elastic.co/ecszap v1.0.1 // indirect + go.elastic.co/ecszap v1.0.1 go.elastic.co/go-licence-detector v0.5.0 go.etcd.io/bbolt v1.3.6 go.uber.org/atomic v1.9.0 diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 3b0d3a39e780..87ea2a2257dc 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -14,13 +14,16 @@ import ( "time" retryablehttp "github.com/hashicorp/go-retryablehttp" + "go.elastic.co/ecszap" "go.uber.org/zap" + "go.uber.org/zap/zapcore" v2 "github.com/elastic/beats/v7/filebeat/input/v2" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/libbeat/version" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/internal/httplog" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/transport/httpcommon" @@ -160,6 +163,18 @@ func newHTTPClient(ctx context.Context, config config, log *logp.Logger) (*httpC return nil, err } + if config.Request.Tracer != nil { + w := zapcore.AddSync(config.Request.Tracer) + core := ecszap.NewCore( + ecszap.NewDefaultEncoderConfig(), + w, + zap.DebugLevel, + ) + traceLogger := zap.New(core) + + netHTTPClient.Transport = httplog.NewLoggingRoundTripper(netHTTPClient.Transport, traceLogger) + } + netHTTPClient.CheckRedirect = checkRedirect(config.Request, log) client := &retryablehttp.Client{ @@ -179,10 +194,10 @@ func newHTTPClient(ctx context.Context, config config, log *logp.Logger) (*httpC if err != nil { return nil, err } - return &httpClient{client: authClient, limiter: limiter, tracer: newTracer(config.Request.Tracer)}, nil + return &httpClient{client: authClient, limiter: limiter}, nil } - return &httpClient{client: client.StandardClient(), limiter: limiter, tracer: newTracer(config.Request.Tracer)}, nil + return &httpClient{client: client.StandardClient(), limiter: limiter}, nil } func checkRedirect(config *requestConfig, log *logp.Logger) func(*http.Request, []*http.Request) error { diff --git a/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go b/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go new file mode 100644 index 000000000000..bd6394b3ebc7 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go @@ -0,0 +1,200 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Package httplog provides http request and response transaction logging. +package httplog + +import ( + "bytes" + "encoding/base32" + "encoding/binary" + "errors" + "fmt" + "io" + "net/http" + "net/http/httputil" + "strconv" + "time" + + "go.uber.org/atomic" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var _ http.RoundTripper = (*LoggingRoundTripper)(nil) + +// TraceIDKey is key used to add a trace.id value to the context of HTTP +// requests. The value will be logged by LoggingRoundTripper. +const TraceIDKey = contextKey("trace.id") + +type contextKey string + +// NewLoggingRoundTripper returns a LoggingRoundTripper that logs requests and +// responses to the provided logger. +func NewLoggingRoundTripper(next http.RoundTripper, logger *zap.Logger) *LoggingRoundTripper { + return &LoggingRoundTripper{ + transport: next, + logger: logger, + txBaseID: newID(), + txIDCounter: atomic.NewUint64(0), + } +} + +// LoggingRoundTripper is an http.RoundTripper that logs requests and responses. +type LoggingRoundTripper struct { + transport http.RoundTripper + logger *zap.Logger // Destination logger. + txBaseID string // Random value to make transaction IDs unique. + txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. +} + +// RoundTrip implements the http.RoundTripper interface, logging +// the request and response to the underlying logger. +// +// Fields logged in requests: +// url.original +// url.scheme +// url.path +// url.domain +// url.port +// url.query +// http.request +// user_agent.original +// http.request.body.content +// http.request.body.bytes +// http.request.mime_type +// event.original (the full request and body from httputil.DumpRequestOut) +// +// Fields logged in responses: +// http.response.status_code +// http.response.body.content +// http.response.body.bytes +// http.response.mime_type +// event.original (the full response and body from httputil.DumpResponse) +// +func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Create a child logger for this request. + log := rt.logger.With( + zap.String("transaction.id", rt.nextTxID()), + ) + + if v := req.Context().Value(TraceIDKey); v != nil { + if traceID, ok := v.(string); ok { + log = log.With(zap.String("trace.id", traceID)) + } + } + + reqParts := []zapcore.Field{ + zap.String("url.original", req.URL.String()), + zap.String("url.scheme", req.URL.Scheme), + zap.String("url.path", req.URL.Path), + zap.String("url.domain", req.URL.Hostname()), + zap.String("url.port", req.URL.Port()), + zap.String("url.query", req.URL.RawQuery), + zap.String("http.request.method", req.Method), + zap.String("user_agent.original", req.Header.Get("User-Agent")), + } + var ( + body []byte + err error + errors []error + ) + req.Body, body, err = copyBody(req.Body) + if err != nil { + errors = append(errors, fmt.Errorf("failed to read request body: %w", err)) + } else { + reqParts = append(reqParts, + zap.ByteString("http.request.body.content", body), + zap.Int("http.request.body.bytes", len(body)), + zap.String("http.request.mime_type", req.Header.Get("Content-Type")), + ) + } + message, err := httputil.DumpRequestOut(req, true) + if err != nil { + errors = append(errors, fmt.Errorf("failed to dump request: %w", err)) + } else { + reqParts = append(reqParts, zap.ByteString("event.original", message)) + } + if errors != nil { + reqParts = append(reqParts, zap.Errors("error.message", errors)) + } + log.Debug("HTTP request", reqParts...) + + resp, err := rt.transport.RoundTrip(req) + if err != nil { + log.Debug("HTTP response error", zap.NamedError("error.message", err)) + return resp, err + } + if resp == nil { + log.Debug("HTTP response error", noResponse) + return resp, err + } + respParts := append(reqParts[:0], + zap.Int("http.response.status_code", resp.StatusCode), + ) + errors = errors[:0] + resp.Body, body, err = copyBody(resp.Body) + if err != nil { + errors = append(errors, fmt.Errorf("failed to read response body: %w", err)) + } else { + respParts = append(respParts, + zap.ByteString("http.response.body.content", body), + zap.Int("http.response.body.bytes", len(body)), + zap.String("http.response.mime_type", resp.Header.Get("Content-Type")), + ) + } + message, err = httputil.DumpResponse(resp, true) + if err != nil { + errors = append(errors, fmt.Errorf("failed to dump response: %w", err)) + } else { + respParts = append(respParts, zap.ByteString("event.original", message)) + } + if errors != nil { + respParts = append(respParts, zap.Errors("error.message", errors)) + } + log.Debug("HTTP response", respParts...) + + return resp, err +} + +// nextTxID returns the next transaction.id value. It increments the internal +// request counter. +func (rt *LoggingRoundTripper) nextTxID() string { + count := rt.txIDCounter.Inc() + return rt.txBaseID + "-" + strconv.FormatUint(count, 10) +} + +var noResponse = zap.NamedError("error.message", errors.New("unexpected nil response")) + +// newID returns an ID derived from the current time. +func newID() string { + var data [8]byte + binary.LittleEndian.PutUint64(data[:], uint64(time.Now().UnixNano())) + return base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(data[:]) +} + +// copyBody is derived from drainBody in net/http/httputil/dump.go +// +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// copyBody reads all of b to memory and then returns a +// ReadCloser yielding the same bytes, and the bytes themselves. +// +// It returns an error if the initial slurp of all bytes fails. +func copyBody(b io.ReadCloser) (r io.ReadCloser, body []byte, err error) { + if b == nil || b == http.NoBody { + // No copying needed. Preserve the magic sentinel meaning of NoBody. + return http.NoBody, nil, nil + } + var buf bytes.Buffer + if _, err = buf.ReadFrom(b); err != nil { + return nil, buf.Bytes(), err + } + if err = b.Close(); err != nil { + return nil, buf.Bytes(), err + } + return io.NopCloser(&buf), buf.Bytes(), nil +} diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 40502022c224..17bece981b7e 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -10,15 +10,11 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" - "net/http/httputil" "net/url" "strings" - "time" "github.com/PaesslerAG/jsonpath" - "gopkg.in/natefinch/lumberjack.v2" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/elastic-agent-libs/logp" @@ -36,15 +32,11 @@ func registerRequestTransforms() { type httpClient struct { client *http.Client limiter *rateLimiter - tracer *tracer } func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Response, error) { resp, err := c.limiter.execute(stdCtx, func() (*http.Response, error) { - c.tracer.txnStart(req) - resp, err := c.client.Do(req) - c.tracer.txnDone(resp) - return resp, err + return c.client.Do(req) }) if err != nil { return nil, fmt.Errorf("failed to execute http client.Do: %w", err) @@ -65,59 +57,6 @@ func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Respon return resp, nil } -// tracer is a lightweight transaction tracer for debugging httpjson configurations. -// It is not intended to be used for machine ingestion, but rather as a human aid. -type tracer struct { - sess int64 // sess is the unix time of the start of a session. - txn int // txn is a session-unique transaction id. - inTxn bool - log *log.Logger -} - -func newTracer(logger *lumberjack.Logger) *tracer { - if logger == nil { - return nil - } - return &tracer{sess: time.Now().UnixMilli(), log: log.New(logger, "", log.LstdFlags)} -} - -func (t *tracer) txnStart(req *http.Request) { - if t == nil { - return - } - if t.inTxn { - t.log.Printf("no response %d.%d", t.sess, t.txn) - } - t.txn++ - t.inTxn = true - b, err := httputil.DumpRequestOut(req, true) - if err != nil { - t.log.Print(err) - } else { - t.log.Printf("request %d.%d:\n\t%s", t.sess, t.txn, bytes.ReplaceAll(b, []byte{'\n'}, []byte("\n\t"))) - } -} - -func (t *tracer) txnDone(resp *http.Response) { - if t == nil { - return - } - if !t.inTxn { - t.log.Printf("missing request %d.%d", t.sess, t.txn) - } - t.inTxn = false - if resp == nil { - t.log.Printf("no response %d.%d", t.sess, t.txn) - return - } - b, err := httputil.DumpResponse(resp, true) - if err != nil { - t.log.Printf("response %d.%d: %v", t.sess, t.txn, err) - } else { - t.log.Printf("response %d.%d:\n\t%s", t.sess, t.txn, bytes.ReplaceAll(b, []byte{'\n'}, []byte("\n\t"))) - } -} - func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, error) { req := transformable{} req.setURL(rf.url) From 6cc0263116d8e666bf718f38e31ccfd573922840 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 28 Jul 2022 21:30:44 +0930 Subject: [PATCH 6/6] address pr comments --- .../httpjson/internal/httplog/roundtripper.go | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go b/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go index bd6394b3ebc7..319ec8b65eb6 100644 --- a/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go @@ -96,13 +96,13 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err zap.String("user_agent.original", req.Header.Get("User-Agent")), } var ( - body []byte - err error - errors []error + body []byte + err error + errorsMessages []string ) req.Body, body, err = copyBody(req.Body) if err != nil { - errors = append(errors, fmt.Errorf("failed to read request body: %w", err)) + errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read request body: %s", err)) } else { reqParts = append(reqParts, zap.ByteString("http.request.body.content", body), @@ -112,12 +112,16 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err } message, err := httputil.DumpRequestOut(req, true) if err != nil { - errors = append(errors, fmt.Errorf("failed to dump request: %w", err)) + errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump request: %s", err)) } else { reqParts = append(reqParts, zap.ByteString("event.original", message)) } - if errors != nil { - reqParts = append(reqParts, zap.Errors("error.message", errors)) + switch len(errorsMessages) { + case 0: + case 1: + reqParts = append(reqParts, zap.String("error.message", errorsMessages[0])) + default: + reqParts = append(reqParts, zap.Strings("error.message", errorsMessages)) } log.Debug("HTTP request", reqParts...) @@ -133,10 +137,10 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err respParts := append(reqParts[:0], zap.Int("http.response.status_code", resp.StatusCode), ) - errors = errors[:0] + errorsMessages = errorsMessages[:0] resp.Body, body, err = copyBody(resp.Body) if err != nil { - errors = append(errors, fmt.Errorf("failed to read response body: %w", err)) + errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read response body: %s", err)) } else { respParts = append(respParts, zap.ByteString("http.response.body.content", body), @@ -146,12 +150,16 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err } message, err = httputil.DumpResponse(resp, true) if err != nil { - errors = append(errors, fmt.Errorf("failed to dump response: %w", err)) + errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump response: %s", err)) } else { respParts = append(respParts, zap.ByteString("event.original", message)) } - if errors != nil { - respParts = append(respParts, zap.Errors("error.message", errors)) + switch len(errorsMessages) { + case 0: + case 1: + respParts = append(reqParts, zap.String("error.message", errorsMessages[0])) + default: + respParts = append(reqParts, zap.Strings("error.message", errorsMessages)) } log.Debug("HTTP response", respParts...)