From ba19cc23a9e6b760b731d8212b6e0e23e424f59e Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 25 Aug 2022 15:09:21 +0900 Subject: [PATCH 1/2] allow for json/ndjson content type with charset (#32767) * allow for json/ndjson content type with charset * add '; charset=UTF-8' in integration tests * changelog * assessing CI * revert correct behaviour (cherry picked from commit b40349ce5ac44a49c65e268419dc17336df1509d) # Conflicts: # x-pack/filebeat/input/awss3/input_integration_test.go --- CHANGELOG.next.asciidoc | 6 +++++ .../input/awss3/input_integration_test.go | 24 +++++++++++++++---- x-pack/filebeat/input/awss3/s3_objects.go | 2 +- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e8945a00594..ffeb6ae017a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -38,6 +38,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* +- Fix counter for number of events published in `httpjson` input. {pull}31993[31993] +- Fix handling of Checkpoint event for R81. {issue}32380[32380] {pull}32458[32458] +- Fix a hang on `apt-get update` stage in packaging. {pull}32580[32580] +- gcp-pubsub input: Restart Pub/Sub client on all errors. {issue}32550[32550] {pull}32712[32712] +- Fix not parsing as json when `json` and `ndjson` content types have charset information in `aws-s3` input {pull}32767[32767] +- Update `cloud.region` parsing in cloudtrail fileset. {pull}32763[32763] - Fix file.path field in cloudtrail fileset to use json.digestS3Object. {pull}32759[32759] *Heartbeat* diff --git a/x-pack/filebeat/input/awss3/input_integration_test.go b/x-pack/filebeat/input/awss3/input_integration_test.go index 32b61bfcd53..a171a2eca62 100644 --- a/x-pack/filebeat/input/awss3/input_integration_test.go +++ b/x-pack/filebeat/input/awss3/input_integration_test.go @@ -16,6 +16,11 @@ import ( "io/ioutil" "os" "path/filepath" +<<<<<<< HEAD +======= + "runtime" + "strings" +>>>>>>> b40349ce5a (allow for json/ndjson content type with charset (#32767)) "testing" "time" @@ -86,7 +91,6 @@ file_selectors: - regex: 'events-array.json$' expand_event_list_from_field: Events - content_type: application/json include_s3_metadata: - last-modified - x-amz-version-id @@ -95,7 +99,6 @@ file_selectors: - Content-Type - regex: '\.(?:nd)?json(\.gz)?$' - content_type: application/json - regex: 'multiline.txt$' parsers: @@ -115,7 +118,6 @@ file_selectors: - regex: 'events-array.json$' expand_event_list_from_field: Events - content_type: application/json include_s3_metadata: - last-modified - x-amz-version-id @@ -124,7 +126,6 @@ file_selectors: - Content-Type - regex: '\.(?:nd)?json(\.gz)?$' - content_type: application/json - regex: 'multiline.txt$' parsers: @@ -324,11 +325,26 @@ func uploadS3TestFiles(t *testing.T, region, bucket string, filenames ...string) t.Fatalf("Failed to open file %q, %v", filename, err) } + contentType := "" + if strings.HasSuffix(filename, "ndjson") || strings.HasSuffix(filename, "ndjson.gz") { + contentType = contentTypeNDJSON + "; charset=UTF-8" + } else if strings.HasSuffix(filename, "json") || strings.HasSuffix(filename, "json.gz") { + contentType = contentTypeJSON + "; charset=UTF-8" + } + // Upload the file to S3. +<<<<<<< HEAD result, err := uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(bucket), Key: aws.String(filepath.Base(filename)), Body: bytes.NewReader(data), +======= + result, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(filepath.Base(filename)), + Body: bytes.NewReader(data), + ContentType: aws.String(contentType), +>>>>>>> b40349ce5a (allow for json/ndjson content type with charset (#32767)) }) if err != nil { t.Fatalf("Failed to upload file %q: %v", filename, err) diff --git a/x-pack/filebeat/input/awss3/s3_objects.go b/x-pack/filebeat/input/awss3/s3_objects.go index 9626a84a4ad..ba34d8e516e 100644 --- a/x-pack/filebeat/input/awss3/s3_objects.go +++ b/x-pack/filebeat/input/awss3/s3_objects.go @@ -151,7 +151,7 @@ func (p *s3ObjectProcessor) ProcessS3Object() error { // Process object content stream. switch { - case contentType == contentTypeJSON || contentType == contentTypeNDJSON: + case strings.HasPrefix(contentType, contentTypeJSON) || strings.HasPrefix(contentType, contentTypeNDJSON): err = p.readJSON(reader) default: err = p.readFile(reader) From 44b65a85b88a7f519e02cf8a328b1d1eb1ca8775 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 25 Aug 2022 15:29:36 +0900 Subject: [PATCH 2/2] fix backport conflicts --- CHANGELOG.next.asciidoc | 7 +------ x-pack/filebeat/input/awss3/input_integration_test.go | 11 ----------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ffeb6ae017a..e7994d33768 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -38,13 +38,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* -- Fix counter for number of events published in `httpjson` input. {pull}31993[31993] -- Fix handling of Checkpoint event for R81. {issue}32380[32380] {pull}32458[32458] -- Fix a hang on `apt-get update` stage in packaging. {pull}32580[32580] -- gcp-pubsub input: Restart Pub/Sub client on all errors. {issue}32550[32550] {pull}32712[32712] -- Fix not parsing as json when `json` and `ndjson` content types have charset information in `aws-s3` input {pull}32767[32767] -- Update `cloud.region` parsing in cloudtrail fileset. {pull}32763[32763] - Fix file.path field in cloudtrail fileset to use json.digestS3Object. {pull}32759[32759] +- Fix not parsing as json when `json` and `ndjson` content types have charset information in `aws-s3` input {pull}32767[32767] *Heartbeat* diff --git a/x-pack/filebeat/input/awss3/input_integration_test.go b/x-pack/filebeat/input/awss3/input_integration_test.go index a171a2eca62..e56da8f3745 100644 --- a/x-pack/filebeat/input/awss3/input_integration_test.go +++ b/x-pack/filebeat/input/awss3/input_integration_test.go @@ -16,11 +16,7 @@ import ( "io/ioutil" "os" "path/filepath" -<<<<<<< HEAD -======= - "runtime" "strings" ->>>>>>> b40349ce5a (allow for json/ndjson content type with charset (#32767)) "testing" "time" @@ -333,18 +329,11 @@ func uploadS3TestFiles(t *testing.T, region, bucket string, filenames ...string) } // Upload the file to S3. -<<<<<<< HEAD result, err := uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(bucket), - Key: aws.String(filepath.Base(filename)), - Body: bytes.NewReader(data), -======= - result, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(filepath.Base(filename)), Body: bytes.NewReader(data), ContentType: aws.String(contentType), ->>>>>>> b40349ce5a (allow for json/ndjson content type with charset (#32767)) }) if err != nil { t.Fatalf("Failed to upload file %q: %v", filename, err)