From 9f12d841f34b9a985518b0c61b7135900c00a203 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Sat, 12 Nov 2022 01:54:30 -0500 Subject: [PATCH 01/22] Filebeat - Add parse_aws_vpc_flow_log processor This is a processor for parsing AWS VPC flow logs. It requires a user specified log format. It can populate the original flow log fields, ECS fields, or both. Usage: processors: - parse_aws_vpc_flow_log: format: version account-id interface-id srcaddr dstaddr srcport dstport protocol packets bytes start end action log-status Benchmark: goos: darwin goarch: arm64 (Apple M1 Max) pkg: github.com/elastic/beats/v7/x-pack/filebeat/processors/aws_vpcflow BenchmarkProcessorRun/v5-mode-original-10 2694968 2212 ns/op 2836 B/op 31 allocs/op BenchmarkProcessorRun/v5-mode-ecs_and_original-10 1812913 3318 ns/op 2972 B/op 36 allocs/op --- auditbeat/docs/index.asciidoc | 1 + heartbeat/docs/index.asciidoc | 1 + libbeat/docs/processors-list.asciidoc | 6 + metricbeat/docs/index.asciidoc | 1 + packetbeat/docs/index.asciidoc | 1 + winlogbeat/docs/index.asciidoc | 1 + x-pack/filebeat/include/list.go | 1 + .../filebeat/processors/aws_vpcflow/config.go | 99 +++++ .../processors/aws_vpcflow/config_test.go | 75 ++++ .../docs/parse_aws_vpc_flow_log.asciidoc | 211 +++++++++++ .../processors/aws_vpcflow/mapping.go | 265 +++++++++++++ .../aws_vpcflow/parse_aws_vpc_flow_log.go | 177 +++++++++ .../parse_aws_vpc_flow_log_test.go | 290 ++++++++++++++ .../testdata/aws-vpc-flow-logs.yml | 75 ++++ .../testdata/custom-nat-gateway.golden.yml | 116 ++++++ .../testdata/default-v2-format.golden.yml | 196 ++++++++++ .../service-name-path-direction.golden.yml | 100 +++++ .../testdata/tcp-flag-sequence.golden.yml | 357 ++++++++++++++++++ .../testdata/transit-gateway.golden.yml | 152 ++++++++ .../testdata/v5-fields-ecs.golden.yml | 53 +++ .../v5-fields-ecs_and_original.golden.yml | 68 ++++ .../testdata/v5-fields-original.golden.yml | 34 ++ .../filebeat/processors/aws_vpcflow/types.go | 80 ++++ .../processors/aws_vpcflow/word_iterator.go | 78 ++++ .../aws_vpcflow/word_iterator_test.go | 39 ++ x-pack/functionbeat/docs/index.asciidoc | 1 + 26 files changed, 2478 insertions(+) create mode 100644 x-pack/filebeat/processors/aws_vpcflow/config.go create mode 100644 x-pack/filebeat/processors/aws_vpcflow/config_test.go create mode 100644 x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc create mode 100644 x-pack/filebeat/processors/aws_vpcflow/mapping.go create mode 100644 x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go create mode 100644 x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/aws-vpc-flow-logs.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/types.go create mode 100644 x-pack/filebeat/processors/aws_vpcflow/word_iterator.go create mode 100644 x-pack/filebeat/processors/aws_vpcflow/word_iterator_test.go diff --git a/auditbeat/docs/index.asciidoc b/auditbeat/docs/index.asciidoc index 56f3d7cc7560..6afd1de2b6a5 100644 --- a/auditbeat/docs/index.asciidoc +++ b/auditbeat/docs/index.asciidoc @@ -22,6 +22,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :linux_os: :no_decode_cef_processor: :no_decode_csv_fields_processor: +:no_parse_aws_vpc_flow_log_processor: :no_script_processor: :no_timestamp_processor: diff --git a/heartbeat/docs/index.asciidoc b/heartbeat/docs/index.asciidoc index 144e71918b81..d323e65baf2a 100644 --- a/heartbeat/docs/index.asciidoc +++ b/heartbeat/docs/index.asciidoc @@ -24,6 +24,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :no_dashboards: :no_decode_cef_processor: :no_decode_csv_fields_processor: +:no_parse_aws_vpc_flow_log_processor: :no_timestamp_processor: include::{libbeat-dir}/shared-beats-attributes.asciidoc[] diff --git a/libbeat/docs/processors-list.asciidoc b/libbeat/docs/processors-list.asciidoc index c09ad5ac3a35..19d352816506 100644 --- a/libbeat/docs/processors-list.asciidoc +++ b/libbeat/docs/processors-list.asciidoc @@ -104,6 +104,9 @@ endif::[] ifndef::no_rename_processor[] * <> endif::[] +ifndef::no_parse_aws_vpc_flow_log_processor[] +* <> +endif::[] ifndef::no_script_processor[] * <> endif::[] @@ -231,6 +234,9 @@ endif::[] ifndef::no_rename_processor[] include::{libbeat-processors-dir}/actions/docs/rename.asciidoc[] endif::[] +ifndef::no_parse_aws_vpc_flow_log_processor[] +include::{x-filebeat-processors-dir}/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc[] +endif::[] ifndef::no_script_processor[] include::{libbeat-processors-dir}/script/docs/script.asciidoc[] endif::[] diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 7dedaf6bda55..0b7937787f2f 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -28,6 +28,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :win_os: :no_decode_cef_processor: :no_decode_csv_fields_processor: +:no_parse_aws_vpc_flow_log_processor: :no_timestamp_processor: :kubernetes_default_indexers: {docdir}/kubernetes-default-indexers-matchers.asciidoc diff --git a/packetbeat/docs/index.asciidoc b/packetbeat/docs/index.asciidoc index 7ff9a667c6d2..0f66d2dde5d0 100644 --- a/packetbeat/docs/index.asciidoc +++ b/packetbeat/docs/index.asciidoc @@ -24,6 +24,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :win_os: :no_decode_cef_processor: :no_decode_csv_fields_processor: +:no_parse_aws_vpc_flow_log_processor: :no_script_processor: :no_timestamp_processor: diff --git a/winlogbeat/docs/index.asciidoc b/winlogbeat/docs/index.asciidoc index 97c1023d4c26..80ed3ef23451 100644 --- a/winlogbeat/docs/index.asciidoc +++ b/winlogbeat/docs/index.asciidoc @@ -20,6 +20,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :win_only: :no_decode_cef_processor: :no_decode_csv_fields_processor: +:no_parse_aws_vpc_flow_log_processor: :include_translate_sid_processor: :export_pipeline: diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index c1382ae36275..f7c35308ed32 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -65,5 +65,6 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zoom" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zscaler" _ "github.com/elastic/beats/v7/x-pack/filebeat/processors/add_nomad_metadata" + _ "github.com/elastic/beats/v7/x-pack/filebeat/processors/aws_vpcflow" _ "github.com/elastic/beats/v7/x-pack/filebeat/processors/decode_cef" ) diff --git a/x-pack/filebeat/processors/aws_vpcflow/config.go b/x-pack/filebeat/processors/aws_vpcflow/config.go new file mode 100644 index 000000000000..967aba127da4 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/config.go @@ -0,0 +1,99 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws_vpcflow + +import ( + "errors" + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +// mode represents the processing mode (original, ecs, ecs_and_original). +type mode uint8 + +const ( + originalMode mode = iota // originalMode generates the fields specified in the format string. + ecsMode // ecsMode maps the original fields to ECS and removes the original field if it was mapped. + ecsAndOriginalMode // ecsAndOriginalMode maps the original fields to ECS and retains all the original fields. +) + +var modeStrings = map[mode]string{ + originalMode: "original", + ecsMode: "ecs", + ecsAndOriginalMode: "ecs_and_original", +} + +func (m *mode) Unpack(s string) error { + for modeConst, modeStr := range modeStrings { + if strings.EqualFold(modeStr, s) { + *m = modeConst + return nil + } + } + return fmt.Errorf("invalid mode type <%v> for "+procName, s) +} + +func (m *mode) UnmarshalYAML(value *yaml.Node) error { + return m.Unpack(value.Value) +} + +func (m *mode) String() string { + if s, found := modeStrings[*m]; found { + return s + } + return "unknown mode" +} + +// config contains the configuration options for the processor. +type config struct { + Format string `config:"format" validate:"required"` // VPC flow log format. + Mode mode `config:"mode"` // Mode controls what fields are generated. + Field string `config:"field"` // Source field containing the VPC flow log message. + TargetField string `config:"target_field"` // Target field for the VPC flow log object. This applies only to the original VPC flow log fields. ECS fields are written to the standard location. + IgnoreMissing bool `config:"ignore_missing"` // Ignore missing source field. + IgnoreFailure bool `config:"ignore_failure"` // Ignore failures while parsing and transforming the flow log message. + ID string `config:"id"` // Instance ID for debugging purposes. +} + +// Validate validates the config settings. It returns an error if the format +// string is invalid. +func (c *config) Validate() error { + _, err := parseFormat(c.Format) + return err +} + +func defaultConfig() config { + return config{ + Mode: ecsMode, + Field: "message", + TargetField: "aws.vpcflow", + } +} + +// parseFormat parses VPC flow log format string and returns an ordered list of +// the expected fields. +func parseFormat(format string) ([]vpcFlowField, error) { + tokens := strings.Fields(format) + if len(tokens) == 0 { + return nil, errors.New("format must contain at lease one field") + } + + fields := make([]vpcFlowField, 0, len(tokens)) + for _, token := range tokens { + // Elastic uses underscores in field names rather than dashes. + underscoreToken := strings.ReplaceAll(token, "-", "_") + + field, found := nameToFieldMap[underscoreToken] + if !found { + return nil, fmt.Errorf("unknown field %q", token) + } + + fields = append(fields, field) + } + + return fields, nil +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/config_test.go b/x-pack/filebeat/processors/aws_vpcflow/config_test.go new file mode 100644 index 000000000000..5a8a99c7fe62 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/config_test.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws_vpcflow + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/require" + + conf "github.com/elastic/elastic-agent-libs/config" +) + +func TestConfigUnpack(t *testing.T) { + testCases := []struct { + yamlConfig string + error bool + }{ + { + yamlConfig: ` +--- +mode: ecs_and_original +id: us-east-vpcflow +format: instance-id interface-id srcaddr dstaddr pkt-srcaddr pkt-dstaddr +`, + }, + { + yamlConfig: ` +--- +mode: original +format: version interface-id account-id vpc-id subnet-id instance-id srcaddr dstaddr srcport dstport protocol tcp-flags type pkt-srcaddr pkt-dstaddr action log-status +`, + }, + { + yamlConfig: ` +--- +mode: ecs +format: version srcaddr dstaddr srcport dstport protocol start end type packets bytes account-id vpc-id subnet-id instance-id interface-id region az-id sublocation-type sublocation-id action tcp-flags pkt-srcaddr pkt-dstaddr pkt-src-aws-service pkt-dst-aws-service traffic-path flow-direction log-status +`, + }, + { + yamlConfig: ` +--- +mode: ecs +format: version srcaddr dstaddr srcport dstport protocol start end type packets bytes account-id vpc-id subnet-id instance-id interface-id region az-id sublocation-type sublocation-id action tcp-flags pkt-srcaddr pkt-dstaddr pkt-src-aws-service pkt-dst-aws-service traffic-path flow-direction log-status +`, + }, + { + error: true, + yamlConfig: ` +--- +mode: invalid +format: version +`, + }, + } + + for i, tc := range testCases { + tc := tc + t.Run(strconv.Itoa(i), func(t *testing.T) { + rawConfig := conf.MustNewConfigFrom(tc.yamlConfig) + + c := defaultConfig() + err := rawConfig.Unpack(&c) + if tc.error { + t.Log("Error:", err) + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc b/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc new file mode 100644 index 000000000000..6df0f25044fd --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc @@ -0,0 +1,211 @@ +[[processor-parse-aws-vpc-flow-log]] +[role="xpack"] +=== Parse AWS VPC Flow Log + +++++ +parse_aws_vpc_flow_log +++++ + +The `parse_aws_vpc_flow_log` processor decodes AWS VPC Flow log messages. + +Below is an example configuration that decodes the `message` field using the +default version 2 VPC flow log format. + +[source,yaml] +---- +processors: + - parse_aws_vpc_flow_log: + format: version account-id interface-id srcaddr dstaddr srcport dstport protocol packets bytes start end action log-status + field: message +---- + +The `parse_aws_vpc_flow_log` processor has the following configuration settings. + +.Parse AWS VPC Flow Log options +[options="header"] +|====== +| Name | Required | Default | Description | +| `field` | no | `message` | Source field containing the VPC flow log message. | +| `target_field` | no | `aws.vpcflow` | Target field for the VPC flow log object. This applies only to the original VPC flow log fields. ECS fields are written to the standard location. | +| `format` | yes | | VPC flow log format. This supports VPC flow log fields from versions 2 through 5. | +| `mode` | no | `ecs` | Mode controls what fields are generated. The available options are `original`, `ecs`, and `ecs_and_original`. `original` generates the fields specified in the format string. `ecs` maps the original fields to ECS and removes the original fields that are mapped to ECS. `ecs_and_original` maps the original fields to ECS and retains all the original fields. | +| `ignore_missing` | no | false | Ignore missing source field. | +| `ignore_failure` | no | false | Ignore failures while parsing and transforming the flow log message. | +| `id` | no | | Instance ID for debugging purposes. | +|====== + +[float] +=== Modes + +[float] +==== Original + +This mode returns the same fields found in the `format` string. It will drop any +fields whose value a dash (`-`). It converts the strings into the appropriate +data types. These are the known field names and their data types. + +NOTE: The AWS VPC flow field names use underscores instead of dashes within +Filebeat. You may configure the `format` using field names that contain either. + +[options="header"] +|====== +| VPC Flow Log Field | Data Type | +| account_id | string | +| action | string | +| az_id | string | +| bytes | long | +| dstaddr | ip | +| dstport | integer | +| end | timestamp | +| flow_direction | string | +| instance_id | string | +| interface_id | string | +| log_status | string | +| packets | long | +| pkt_dst_aws_service | string | +| pkt_dstaddr | ip | +| pkt_src_aws_service | string | +| pkt_srcaddr | ip | +| protocol | integer | +| region | string | +| srcaddr | ip | +| srcport | integer | +| start | timestamp | +| sublocation_id | string | +| sublocation_type | string | +| subnet_id | string | +| tcp_flags | integer | +| tcp_flags_array* | integer | +| traffic_path | integer | +| type | string | +| version | integer | +| vpc_id | string | +|====== + +[float] +==== ECS + +This mode maps the original VPC flow log fields into their associated Elastic +Common Schema (ECS) fields. It removes the original fields that were mapped to +ECS to reduced duplication. These are the field associations. There may be some +transformations applied to derive the ECS field. + +[options="header"] +|====== +| VPC Flow Log Field | ECS Field | +| account_id | cloud.account.id | +| action | event.outcome | +| az_id | cloud.availability_zone | +| bytes | network.bytes | +| bytes | source.bytes | +| dstaddr | destination.address | +| dstaddr | destination.ip | +| dstport | destination.port | +| end | @timestamp | +| end | event.end | +| flow_direction | network.direction | +| instance_id | cloud.instance.id | +| packets | network.packets | +| packets | source.packets | +| protocol | network.iana_number | +| protocol | network.protocol | +| region | cloud.region | +| srcaddr | network.type | +| srcaddr | source.address | +| srcaddr | source.ip | +| srcport | source.port | +| start | event.start | +|====== + +[float] +==== ECS and Original + +This mode maps the fields into ECS and retains all the original fields. Below +is an example document produced using `ecs_and_orignal` mode. + +[source,json] +---- +{ + "@timestamp": "2021-03-26T03:29:09Z", + "aws": { + "vpcflow": { + "account_id": "64111117617", + "action": "REJECT", + "az_id": "use1-az5", + "bytes": 1, + "dstaddr": "10.200.0.0", + "dstport": 33004, + "end": "2021-03-26T03:29:09Z", + "flow_direction": "ingress", + "instance_id": "i-0axxxxxx1ad77", + "interface_id": "eni-069xxxxxb7a490", + "log_status": "OK", + "packets": 52, + "pkt_dst_aws_service": "CLOUDFRONT", + "pkt_dstaddr": "10.200.0.80", + "pkt_src_aws_service": "AMAZON", + "pkt_srcaddr": "89.160.20.156", + "protocol": 17, + "region": "us-east-1", + "srcaddr": "89.160.20.156", + "srcport": 50041, + "start": "2021-03-26T03:28:12Z", + "sublocation_id": "fake-id", + "sublocation_type": "wavelength", + "subnet_id": "subnet-02d645xxxxxxxdbc0", + "tcp_flags": 1, + "tcp_flags_array": [ + "fin" + ], + "traffic_path": 1, + "type": "IPv4", + "version": 5, + "vpc_id": "vpc-09676f97xxxxxb8a7" + } + }, + "cloud": { + "account": { + "id": "64111117617" + }, + "availability_zone": "use1-az5", + "instance": { + "id": "i-0axxxxxx1ad77" + }, + "region": "us-east-1" + }, + "destination": { + "address": "10.200.0.0", + "ip": "10.200.0.0", + "port": 33004 + }, + "event": { + "end": "2021-03-26T03:29:09Z", + "outcome": "failure", + "start": "2021-03-26T03:28:12Z" + }, + "message": "5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1", + "network": { + "bytes": 1, + "direction": "ingress", + "iana_number": "17", + "packets": 52, + "protocol": "udp", + "type": "ipv4" + }, + "related": { + "ip": [ + "89.160.20.156", + "10.200.0.0", + "10.200.0.80" + ] + }, + "source": { + "address": "89.160.20.156", + "bytes": 1, + "ip": "89.160.20.156", + "packets": 52, + "port": 50041 + } +} +---- + diff --git a/x-pack/filebeat/processors/aws_vpcflow/mapping.go b/x-pack/filebeat/processors/aws_vpcflow/mapping.go new file mode 100644 index 000000000000..a3d69f4f467a --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/mapping.go @@ -0,0 +1,265 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws_vpcflow + +import ( + "math/bits" + "strconv" + "strings" + + "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/elastic/beats/v7/libbeat/beat" +) + +type vpcFlowField struct { + Name string // Name of the VPC flow field that is added to our events. + Type dataType // Data type to convert the string into. + Enrich func(originalFields mapstr.M, value interface{}) // Optional enrichment function to add new derived fields into the 'target_field' namespace. + ECSMappings []ecsFieldMapping // List of ECS fields to create or derive from this field. +} + +type ecsFieldMapping struct { + Target string // ECS field target. + Transform func(targetField string, value interface{}, event *beat.Event) // Optional transform to modify the value. If omitted the value is copied. +} + +var nameToFieldMap map[string]vpcFlowField + +func init() { + nameToFieldMap = make(map[string]vpcFlowField, len(fields)) + for _, field := range fields { + nameToFieldMap[field.Name] = field + } +} + +var fields = []vpcFlowField{ + { + Name: "version", + Type: integerType, + }, + { + Name: "account_id", + Type: stringType, + ECSMappings: []ecsFieldMapping{ + {Target: "cloud.account.id"}, + }, + }, + { + Name: "interface_id", + Type: stringType, + }, + { + Name: "srcaddr", + Type: ipType, + ECSMappings: []ecsFieldMapping{ + {Target: "source.address"}, + {Target: "source.ip"}, + { + Target: "network.type", + Transform: func(targetField string, value interface{}, event *beat.Event) { + if ip := value.(string); strings.Contains(ip, ".") { + event.PutValue(targetField, "ipv4") //nolint:errcheck // This can only fail if 'network' is not an object. + } else { + event.PutValue(targetField, "ipv6") //nolint:errcheck // This can only fail if 'network' is not an object. + } + }, + }, + }, + }, + { + Name: "dstaddr", + Type: ipType, + ECSMappings: []ecsFieldMapping{ + {Target: "destination.address"}, + {Target: "destination.ip"}, + }, + }, + { + Name: "srcport", + Type: integerType, + ECSMappings: []ecsFieldMapping{ + {Target: "source.port"}, + }, + }, + { + Name: "dstport", + Type: integerType, + ECSMappings: []ecsFieldMapping{ + {Target: "destination.port"}, + }, + }, + { + Name: "protocol", + Type: integerType, + ECSMappings: []ecsFieldMapping{ + { + Target: "network.iana_number", + Transform: func(targetField string, value interface{}, event *beat.Event) { + protocol := value.(int32) + event.PutValue(targetField, strconv.Itoa(int(protocol))) //nolint:errcheck // This can only fail if 'network' is not an object. + }, + }, + { + Target: "network.protocol", + Transform: func(targetField string, value interface{}, event *beat.Event) { + var name string + switch protocol := value.(int32); protocol { + case 0: + name = "hopopt" + case 1: + name = "icmp" + case 2: + name = "igmp" + case 6: + name = "tcp" + case 8: + name = "egp" + case 17: + name = "udp" + case 47: + name = "gre" + case 50: + name = "esp" + case 58: + name = "ipv6-icmp" + case 112: + name = "vrrp" + case 132: + name = "sctp" + } + + if name != "" { + event.PutValue(targetField, name) //nolint:errcheck // This can only fail if 'network' is not an object. + } + }, + }, + }, + }, + { + Name: "packets", + Type: longType, + ECSMappings: []ecsFieldMapping{ + {Target: "source.packets"}, + {Target: "network.packets"}, + }, + }, + { + Name: "bytes", + Type: longType, + ECSMappings: []ecsFieldMapping{ + {Target: "source.bytes"}, + {Target: "network.bytes"}, + }, + }, + { + Name: "start", + Type: timestampType, + ECSMappings: []ecsFieldMapping{ + {Target: "event.start"}, + }, + }, + { + Name: "end", + Type: timestampType, + ECSMappings: []ecsFieldMapping{ + {Target: "event.end"}, + {Target: "@timestamp"}, + }, + }, + { + Name: "action", + Type: stringType, + ECSMappings: []ecsFieldMapping{ + { + Target: "event.outcome", + Transform: func(targetField string, value interface{}, event *beat.Event) { + var outcome string + + switch s := value.(string); s { + case "ACCEPT": + outcome = "success" + case "REJECT": + outcome = "failure" + } + + if outcome != "" { + event.PutValue(targetField, outcome) //nolint:errcheck // This can only fail if 'event' is not an object. + } + }, + }, + }, + }, + {Name: "log_status", Type: stringType}, + {Name: "vpc_id", Type: stringType}, + {Name: "subnet_id", Type: stringType}, + { + Name: "instance_id", + Type: stringType, + ECSMappings: []ecsFieldMapping{ + {Target: "cloud.instance.id"}, + }, + }, + { + Name: "tcp_flags", + Type: integerType, + Enrich: func(originalFields mapstr.M, value interface{}) { + flag := value.(int32) + flags := make([]string, 0, bits.OnesCount8(uint8(flag))) + if flag&0x01 != 0 { + flags = append(flags, "fin") + } + if flag&0x02 != 0 { + flags = append(flags, "syn") + } + if flag&0x04 != 0 { + flags = append(flags, "rst") + } + if flag&0x08 != 0 { + flags = append(flags, "psh") + } + if flag&0x10 != 0 { + flags = append(flags, "ack") + } + if flag&0x20 != 0 { + flags = append(flags, "urg") + } + + if len(flags) > 0 { + originalFields["tcp_flags_array"] = flags + } + }, + }, + {Name: "type", Type: stringType}, + // TODO: Could these be used in some way to set source.nat.* and destination.nat.*. + {Name: "pkt_srcaddr", Type: ipType}, + {Name: "pkt_dstaddr", Type: ipType}, + { + Name: "region", + Type: stringType, + ECSMappings: []ecsFieldMapping{ + {Target: "cloud.region"}, + }, + }, + { + Name: "az_id", + Type: stringType, + ECSMappings: []ecsFieldMapping{ + {Target: "cloud.availability_zone"}, + }, + }, + {Name: "sublocation_type", Type: stringType}, + {Name: "sublocation_id", Type: stringType}, + {Name: "pkt_src_aws_service", Type: stringType}, + {Name: "pkt_dst_aws_service", Type: stringType}, + { + Name: "flow_direction", + Type: stringType, + ECSMappings: []ecsFieldMapping{ + {Target: "network.direction"}, + }, + }, + {Name: "traffic_path", Type: integerType}, +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go new file mode 100644 index 000000000000..fa065e1fd0ea --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go @@ -0,0 +1,177 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws_vpcflow + +import ( + "encoding/json" + "errors" + "fmt" + + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/processors" + jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" +) + +const ( + procName = "parse_aws_vpc_flow_log" + logName = "processor." + procName +) + +func init() { + processors.RegisterPlugin(procName, New) + jsprocessor.RegisterPlugin("ParseAWSVPCFlowLog", New) +} + +var ( + errValueNotString = errors.New("field must be a string") + errInvalidFormat = errors.New("log did not match the specified format") +) + +type processor struct { + config + fields []vpcFlowField + log *logp.Logger + expectedIPCount int +} + +// New constructs a new processor built from ucfg config. +func New(cfg *conf.C) (processors.Processor, error) { + c := defaultConfig() + if err := cfg.Unpack(&c); err != nil { + return nil, fmt.Errorf("fail to unpack the "+procName+" processor configuration: %w", err) + } + + return newParseAWSVPCFlowLog(c) +} + +func newParseAWSVPCFlowLog(c config) (*processor, error) { + log := logp.NewLogger(logName) + if c.ID != "" { + log = log.With("instance_id", c.ID) + } + + fields, err := parseFormat(c.Format) + if err != nil { + return nil, fmt.Errorf("failed to parse vpc flow log format: %w", err) + } + + var ipCount int + for _, f := range fields { + if f.Type == ipType { + ipCount++ + } + } + + return &processor{config: c, fields: fields, expectedIPCount: ipCount, log: log}, nil +} + +func (p *processor) String() string { + json, _ := json.Marshal(p.config) + return procName + "=" + string(json) +} + +func (p *processor) Run(event *beat.Event) (*beat.Event, error) { + err := p.run(event) + if err == nil || p.IgnoreFailure || (p.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound)) { + return event, nil + } + + return event, err +} + +func (p *processor) run(event *beat.Event) error { + v, err := event.GetValue(p.Field) + if err != nil { + return err + } + + strValue, ok := v.(string) + if !ok { + return errValueNotString + } + + itr := newWordIterator(strValue) + if itr.Count() != len(p.fields) { + return errInvalidFormat + } + + var relatedIPs []string + if p.Mode > originalMode { + relatedIPs = make([]string, 0, p.expectedIPCount) + } + + originalFields := make(mapstr.M, len(p.fields)) + + for itr.Next() { + // Read one word. + value := itr.Word() + if value == "-" { + continue + } + field := p.fields[itr.WordIndex()] + + // Convert the string the expected type. + v, err := toType(value, field.Type) + if err != nil { + return fmt.Errorf("failed to parse <%v>: %w", field.Name, err) + } + + // Add to the 'original' fields when we are in original mode + // or ecs_and_original mode. Or if there are no ECS mappings then + // retain the original field. + if p.Mode != ecsMode || len(field.ECSMappings) == 0 { + originalFields[field.Name] = v + + if field.Enrich != nil { + field.Enrich(originalFields, v) + } + } + + // Apply ECS transforms when in ecs or ecs_and_original modes. + if p.Mode > originalMode { + for _, mapping := range field.ECSMappings { + if mapping.Transform == nil { + if _, err = event.PutValue(mapping.Target, v); err != nil { + return err + } + } else { + mapping.Transform(mapping.Target, v, event) + } + } + + if field.Type == ipType { + relatedIPs = appendUnique(relatedIPs, v.(string)) + } + } + } + + if _, err = event.PutValue(p.TargetField, originalFields); err != nil { + return err + } + + if len(relatedIPs) > 0 { + if _, err = event.PutValue("related.ip", relatedIPs); err != nil { + return err + } + } + + return nil +} + +// appendUnique appends a value to the slice if the given value does not already +// exist in the slice. It determines if item is in the slice by iterating over +// all elements in the slice and checking equality. +func appendUnique(s []string, item string) []string { + for _, existing := range s { + if item == existing { + return s + } + } + return append(s, item) +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go new file mode 100644 index 000000000000..1356ae270bb2 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -0,0 +1,290 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws_vpcflow + +import ( + "flag" + "os" + "path/filepath" + "testing" + "time" + + "gopkg.in/yaml.v3" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/elastic/beats/v7/libbeat/beat" +) + +var updateGolden = flag.Bool("update", false, "Update golden test data.") + +const ( + formatV5 = `version account-id interface-id srcaddr dstaddr srcport dstport protocol packets bytes start end action log-status vpc-id subnet-id instance-id tcp-flags type pkt-srcaddr pkt-dstaddr region az-id sublocation-type sublocation-id pkt-src-aws-service pkt-dst-aws-service flow-direction traffic-path` + formatV5Sample = `5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1` +) + +func TestProcessorRun(t *testing.T) { + t.Run("v5-mode-ecs_and_original", func(t *testing.T) { + c := defaultConfig() + c.Format = formatV5 + c.Mode = ecsAndOriginalMode + + p, err := newParseAWSVPCFlowLog(c) + require.NoError(t, err) + + evt := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: map[string]interface{}{ + "message": formatV5Sample, + }, + } + + out, err := p.Run(&evt) + require.NoError(t, err) + + start := time.Unix(1616729292, 0).UTC() + end := time.Unix(1616729349, 0).UTC() + expected := mapstr.M{ + "aws": mapstr.M{ + "vpcflow": mapstr.M{ + "account_id": "64111117617", + "action": "REJECT", + "az_id": "use1-az5", + "bytes": int64(1), + "dstaddr": "10.200.0.0", + "dstport": int32(33004), + "end": end, + "flow_direction": "ingress", + "instance_id": "i-0axxxxxx1ad77", + "interface_id": "eni-069xxxxxb7a490", + "log_status": "OK", + "packets": int64(52), + "pkt_dst_aws_service": "CLOUDFRONT", + "pkt_dstaddr": "10.200.0.80", + "pkt_src_aws_service": "AMAZON", + "pkt_srcaddr": "89.160.20.156", + "protocol": int32(17), + "region": "us-east-1", + "srcaddr": "89.160.20.156", + "srcport": int32(50041), + "start": start, + "sublocation_id": "fake-id", + "sublocation_type": "wavelength", + "subnet_id": "subnet-02d645xxxxxxxdbc0", + "tcp_flags": int32(1), + "tcp_flags_array": []string{ + "fin", + }, + "traffic_path": int32(1), + "type": "IPv4", + "version": int32(5), + "vpc_id": "vpc-09676f97xxxxxb8a7", + }, + }, + "cloud": mapstr.M{ + "account": mapstr.M{ + "id": "64111117617", + }, + "availability_zone": "use1-az5", + "instance": mapstr.M{ + "id": "i-0axxxxxx1ad77", + }, + "region": "us-east-1", + }, + "destination": mapstr.M{ + "address": "10.200.0.0", + "ip": "10.200.0.0", + "port": int32(33004), + }, + "event": mapstr.M{ + "end": end, + "outcome": "failure", + "start": start, + }, + "message": formatV5Sample, + "network": mapstr.M{ + "bytes": int64(1), + "direction": "ingress", + "iana_number": "17", + "packets": int64(52), + "protocol": "udp", + "type": "ipv4", + }, + "related": mapstr.M{ + "ip": []string{"89.160.20.156", "10.200.0.0", "10.200.0.80"}, + }, + "source": mapstr.M{ + "address": "89.160.20.156", + "bytes": int64(1), + "ip": "89.160.20.156", + "packets": int64(52), + "port": int32(50041), + }, + } + + assert.Equal(t, end, out.Timestamp) + if diff := cmp.Diff(expected, out.Fields); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestGoldenFile(t *testing.T) { + testCases := readGoldenTestCase(t) + + if *updateGolden { + // Delete existing golden files. + goldens, _ := filepath.Glob("testdata/*.golden.yml") + for _, golden := range goldens { + os.Remove(golden) + } + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.Name, func(t *testing.T) { + c := defaultConfig() + c.Format = tc.Format + if tc.Mode != nil { + c.Mode = *tc.Mode + } + + p, err := newParseAWSVPCFlowLog(c) + require.NoError(t, err) + + observed := make([]mapstr.M, 0, len(tc.Samples)) + for _, sample := range tc.Samples { + evt := &beat.Event{Fields: mapstr.M{"message": sample}} + out, err := p.Run(evt) + require.NoError(t, err) + + if !out.Timestamp.IsZero() { + out.Fields["@timestamp"] = out.Timestamp + } + observed = append(observed, out.Fields) + } + + goldenFile := filepath.Join("testdata", tc.Name+".golden.yml") + if *updateGolden { + writeGolden(t, goldenFile, observed) + } else { + expectedYAML := readGolden(t, goldenFile) + + observedYAML, err := yaml.Marshal(observed) + require.NoError(t, err) + + assert.YAMLEq(t, expectedYAML, string(observedYAML)) + } + }) + } +} + +type GoldenTestCase struct { + Name string `yaml:"-"` // Name of test. + Mode *mode // Processing mode (what fields to generate). + Format string // Flow log format. + Samples []string // List of sample logs to parse. +} + +func readGoldenTestCase(t testing.TB) []GoldenTestCase { + t.Helper() + + f, err := os.Open("testdata/aws-vpc-flow-logs.yml") + if err != nil { + t.Fatal(err) + } + + dec := yaml.NewDecoder(f) + dec.KnownFields(true) + + var testCases map[string]GoldenTestCase + if err = dec.Decode(&testCases); err != nil { + t.Fatal(err) + } + + testCasesList := make([]GoldenTestCase, 0, len(testCases)) + for k, v := range testCases { + v.Name = k + testCasesList = append(testCasesList, v) + } + + return testCasesList +} + +func writeGolden(t testing.TB, path string, events []mapstr.M) { + t.Helper() + + f, err := os.Create(path) + require.NoError(t, err) + defer f.Close() + + enc := yaml.NewEncoder(f) + enc.SetIndent(2) + if err = enc.Encode(events); err != nil { + t.Fatal() + } +} + +func readGolden(t testing.TB, path string) string { + t.Helper() + + yamlData, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + return string(yamlData) +} + +func BenchmarkProcessorRun(b *testing.B) { + b.Run("v5-mode-original", func(b *testing.B) { + c := defaultConfig() + c.Format = formatV5 + c.Mode = originalMode + + p, err := newParseAWSVPCFlowLog(c) + require.NoError(b, err) + + evt := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: map[string]interface{}{ + "message": formatV5Sample, + }, + } + + for i := 0; i < b.N; i++ { + if _, err = p.Run(&evt); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("v5-mode-ecs_and_original", func(b *testing.B) { + c := defaultConfig() + c.Format = formatV5 + c.Mode = ecsAndOriginalMode + + p, err := newParseAWSVPCFlowLog(c) + require.NoError(b, err) + + evt := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: map[string]interface{}{ + "message": formatV5Sample, + }, + } + + for i := 0; i < b.N; i++ { + if _, err = p.Run(&evt); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/aws-vpc-flow-logs.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/aws-vpc-flow-logs.yml new file mode 100644 index 000000000000..d9d21d98c04b --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/aws-vpc-flow-logs.yml @@ -0,0 +1,75 @@ +--- + +default-v2-format: + format: version account-id interface-id srcaddr dstaddr srcport dstport protocol packets bytes start end action log-status + samples: + # Accepted and rejected traffic + # https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html#flow-log-example-accepted-rejected + - 2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK + - 2 123456789010 eni-1235b8ca123456789 172.31.9.69 172.31.9.12 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK + # No data and skipped records + # https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html#flow-log-example-no-data + - 2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA + - 2 123456789010 eni-11111111aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA + # Security group and network ACL rules + # https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html#flow-log-example-security-groups + - 2 123456789010 eni-1235b8ca123456789 203.0.113.12 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK + - 2 123456789010 eni-1235b8ca123456789 172.31.16.139 203.0.113.12 0 0 1 4 336 1432917094 1432917142 REJECT OK + # IPV6 Traffic + # https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html#flow-log-example-ipv6 + - 2 123456789010 eni-1235b8ca123456789 2001:db8:1234:a100:8d6e:3477:df66:f105 2001:db8:1234:a102:3304:8879:34cf:4071 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK + +# https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html#flow-log-example-tcp-flag +tcp-flag-sequence: + format: version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status + samples: + - 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK + - 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43416 10.0.0.62 52.213.180.42 6 376 7 1566848875 1566848933 ACCEPT 18 OK + - 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 100701 70 1566848875 1566848933 ACCEPT 2 OK + - 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 632 12 1566848875 1566848933 ACCEPT 18 OK + - 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 63388 1219 1566848933 1566849113 ACCEPT 1 OK + - 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 23294588 15774 1566848933 1566849113 ACCEPT 1 OK + - 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43638 5001 52.213.180.42 10.0.0.62 6 1260 17 1566933133 1566933193 ACCEPT 3 OK + - 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43638 10.0.0.62 52.213.180.42 6 967 14 1566933133 1566933193 ACCEPT 19 OK + +# https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html#flow-log-example-nat +custom-nat-gateway: + format: instance-id interface-id srcaddr dstaddr pkt-srcaddr pkt-dstaddr + samples: + - '- eni-1235b8ca123456789 10.0.1.5 10.0.0.220 10.0.1.5 203.0.113.5' + - '- eni-1235b8ca123456789 10.0.0.220 203.0.113.5 10.0.0.220 203.0.113.5' + - '- eni-1235b8ca123456789 203.0.113.5 10.0.0.220 203.0.113.5 10.0.0.220' + - '- eni-1235b8ca123456789 10.0.0.220 10.0.1.5 203.0.113.5 10.0.1.5' + - i-01234567890123456 eni-1111aaaa2222bbbb3 10.0.1.5 203.0.113.5 10.0.1.5 203.0.113.5 #Traffic from the source instance to host on the internet + - i-01234567890123456 eni-1111aaaa2222bbbb3 203.0.113.5 10.0.1.5 203.0.113.5 10.0.1.5 #Response traffic from host on the internet to the source instance + +# https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html#flow-log-example-tgw +transit-gateway: + format: version interface-id account-id vpc-id subnet-id instance-id srcaddr dstaddr srcport dstport protocol tcp-flags type pkt-srcaddr pkt-dstaddr action log-status + samples: + - 3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.20.33.164 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK + - 3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.40.2.236 10.20.33.164 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK + - 3 eni-11111111111111111 123456789010 vpc-abcdefab012345678 subnet-11111111aaaaaaaaa - 10.40.1.175 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK + - 3 eni-22222222222222222 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb - 10.40.2.236 10.40.2.31 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK + +# https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html#flow-log-example-traffic-path +service-name-path-direction: + format: version srcaddr dstaddr srcport dstport protocol start end type packets bytes account-id vpc-id subnet-id instance-id interface-id region az-id sublocation-type sublocation-id action tcp-flags pkt-srcaddr pkt-dstaddr pkt-src-aws-service pkt-dst-aws-service traffic-path flow-direction log-status + samples: + - 5 52.95.128.179 10.0.0.71 80 34210 6 1616729292 1616729349 IPv4 14 15044 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 19 52.95.128.179 10.0.0.71 S3 - - ingress OK + - 5 10.0.0.71 52.95.128.179 34210 80 6 1616729292 1616729349 IPv4 7 471 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 3 10.0.0.71 52.95.128.179 - S3 8 egress OK + +# Full list of all fields in v5 in order. +v5-fields-original: &v5-fields + mode: original + format: version account-id interface-id srcaddr dstaddr srcport dstport protocol packets bytes start end action log-status vpc-id subnet-id instance-id tcp-flags type pkt-srcaddr pkt-dstaddr region az-id sublocation-type sublocation-id pkt-src-aws-service pkt-dst-aws-service flow-direction traffic-path + samples: + - 5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1 + +v5-fields-ecs: + <<: *v5-fields + mode: ecs + +v5-fields-ecs_and_original: + <<: *v5-fields + mode: ecs_and_original diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.yml new file mode 100644 index 000000000000..d2a28dd8aa58 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.yml @@ -0,0 +1,116 @@ +- aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + pkt_dstaddr: 203.0.113.5 + pkt_srcaddr: 10.0.1.5 + destination: + address: 10.0.0.220 + ip: 10.0.0.220 + message: '- eni-1235b8ca123456789 10.0.1.5 10.0.0.220 10.0.1.5 203.0.113.5' + network: + type: ipv4 + related: + ip: + - 10.0.1.5 + - 10.0.0.220 + - 203.0.113.5 + source: + address: 10.0.1.5 + ip: 10.0.1.5 +- aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + pkt_dstaddr: 203.0.113.5 + pkt_srcaddr: 10.0.0.220 + destination: + address: 203.0.113.5 + ip: 203.0.113.5 + message: '- eni-1235b8ca123456789 10.0.0.220 203.0.113.5 10.0.0.220 203.0.113.5' + network: + type: ipv4 + related: + ip: + - 10.0.0.220 + - 203.0.113.5 + source: + address: 10.0.0.220 + ip: 10.0.0.220 +- aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + pkt_dstaddr: 10.0.0.220 + pkt_srcaddr: 203.0.113.5 + destination: + address: 10.0.0.220 + ip: 10.0.0.220 + message: '- eni-1235b8ca123456789 203.0.113.5 10.0.0.220 203.0.113.5 10.0.0.220' + network: + type: ipv4 + related: + ip: + - 203.0.113.5 + - 10.0.0.220 + source: + address: 203.0.113.5 + ip: 203.0.113.5 +- aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + pkt_dstaddr: 10.0.1.5 + pkt_srcaddr: 203.0.113.5 + destination: + address: 10.0.1.5 + ip: 10.0.1.5 + message: '- eni-1235b8ca123456789 10.0.0.220 10.0.1.5 203.0.113.5 10.0.1.5' + network: + type: ipv4 + related: + ip: + - 10.0.0.220 + - 10.0.1.5 + - 203.0.113.5 + source: + address: 10.0.0.220 + ip: 10.0.0.220 +- aws: + vpcflow: + interface_id: eni-1111aaaa2222bbbb3 + pkt_dstaddr: 203.0.113.5 + pkt_srcaddr: 10.0.1.5 + cloud: + instance: + id: i-01234567890123456 + destination: + address: 203.0.113.5 + ip: 203.0.113.5 + message: i-01234567890123456 eni-1111aaaa2222bbbb3 10.0.1.5 203.0.113.5 10.0.1.5 203.0.113.5 + network: + type: ipv4 + related: + ip: + - 10.0.1.5 + - 203.0.113.5 + source: + address: 10.0.1.5 + ip: 10.0.1.5 +- aws: + vpcflow: + interface_id: eni-1111aaaa2222bbbb3 + pkt_dstaddr: 10.0.1.5 + pkt_srcaddr: 203.0.113.5 + cloud: + instance: + id: i-01234567890123456 + destination: + address: 10.0.1.5 + ip: 10.0.1.5 + message: i-01234567890123456 eni-1111aaaa2222bbbb3 203.0.113.5 10.0.1.5 203.0.113.5 10.0.1.5 + network: + type: ipv4 + related: + ip: + - 203.0.113.5 + - 10.0.1.5 + source: + address: 203.0.113.5 + ip: 203.0.113.5 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.yml new file mode 100644 index 000000000000..5c2f91044ce4 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.yml @@ -0,0 +1,196 @@ +- '@timestamp': 2014-12-14T04:07:50Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + version: 2 + cloud: + account: + id: "123456789010" + destination: + address: 172.31.16.21 + ip: 172.31.16.21 + port: 22 + event: + end: 2014-12-14T04:07:50Z + outcome: success + start: 2014-12-14T04:06:50Z + message: 2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK + network: + bytes: 4249 + iana_number: "6" + packets: 20 + protocol: tcp + type: ipv4 + related: + ip: + - 172.31.16.139 + - 172.31.16.21 + source: + address: 172.31.16.139 + bytes: 4249 + ip: 172.31.16.139 + packets: 20 + port: 20641 +- '@timestamp': 2014-12-14T04:07:50Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + version: 2 + cloud: + account: + id: "123456789010" + destination: + address: 172.31.9.12 + ip: 172.31.9.12 + port: 3389 + event: + end: 2014-12-14T04:07:50Z + outcome: failure + start: 2014-12-14T04:06:50Z + message: 2 123456789010 eni-1235b8ca123456789 172.31.9.69 172.31.9.12 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK + network: + bytes: 4249 + iana_number: "6" + packets: 20 + protocol: tcp + type: ipv4 + related: + ip: + - 172.31.9.69 + - 172.31.9.12 + source: + address: 172.31.9.69 + bytes: 4249 + ip: 172.31.9.69 + packets: 20 + port: 49761 +- '@timestamp': 2015-05-10T18:02:14Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: NODATA + version: 2 + cloud: + account: + id: "123456789010" + event: + end: 2015-05-10T18:02:14Z + start: 2015-05-10T18:01:16Z + message: 2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA +- '@timestamp': 2015-05-10T18:02:14Z + aws: + vpcflow: + interface_id: eni-11111111aaaaaaaaa + log_status: SKIPDATA + version: 2 + cloud: + account: + id: "123456789010" + event: + end: 2015-05-10T18:02:14Z + start: 2015-05-10T18:01:16Z + message: 2 123456789010 eni-11111111aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA +- '@timestamp': 2015-05-29T16:32:22Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + version: 2 + cloud: + account: + id: "123456789010" + destination: + address: 172.31.16.139 + ip: 172.31.16.139 + port: 0 + event: + end: 2015-05-29T16:32:22Z + outcome: success + start: 2015-05-29T16:30:27Z + message: 2 123456789010 eni-1235b8ca123456789 203.0.113.12 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK + network: + bytes: 336 + iana_number: "1" + packets: 4 + protocol: icmp + type: ipv4 + related: + ip: + - 203.0.113.12 + - 172.31.16.139 + source: + address: 203.0.113.12 + bytes: 336 + ip: 203.0.113.12 + packets: 4 + port: 0 +- '@timestamp': 2015-05-29T16:32:22Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + version: 2 + cloud: + account: + id: "123456789010" + destination: + address: 203.0.113.12 + ip: 203.0.113.12 + port: 0 + event: + end: 2015-05-29T16:32:22Z + outcome: failure + start: 2015-05-29T16:31:34Z + message: 2 123456789010 eni-1235b8ca123456789 172.31.16.139 203.0.113.12 0 0 1 4 336 1432917094 1432917142 REJECT OK + network: + bytes: 336 + iana_number: "1" + packets: 4 + protocol: icmp + type: ipv4 + related: + ip: + - 172.31.16.139 + - 203.0.113.12 + source: + address: 172.31.16.139 + bytes: 336 + ip: 172.31.16.139 + packets: 4 + port: 0 +- '@timestamp': 2016-10-31T11:37:00Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + version: 2 + cloud: + account: + id: "123456789010" + destination: + address: 2001:db8:1234:a102:3304:8879:34cf:4071 + ip: 2001:db8:1234:a102:3304:8879:34cf:4071 + port: 22 + event: + end: 2016-10-31T11:37:00Z + outcome: success + start: 2016-10-31T11:35:08Z + message: 2 123456789010 eni-1235b8ca123456789 2001:db8:1234:a100:8d6e:3477:df66:f105 2001:db8:1234:a102:3304:8879:34cf:4071 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK + network: + bytes: 8855 + iana_number: "6" + packets: 54 + protocol: tcp + type: ipv6 + related: + ip: + - 2001:db8:1234:a100:8d6e:3477:df66:f105 + - 2001:db8:1234:a102:3304:8879:34cf:4071 + source: + address: 2001:db8:1234:a100:8d6e:3477:df66:f105 + bytes: 8855 + ip: 2001:db8:1234:a100:8d6e:3477:df66:f105 + packets: 54 + port: 34892 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.yml new file mode 100644 index 000000000000..6b87826786f0 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.yml @@ -0,0 +1,100 @@ +- '@timestamp': 2021-03-26T03:29:09Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 10.0.0.71 + pkt_src_aws_service: S3 + pkt_srcaddr: 52.95.128.179 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 19 + tcp_flags_array: + - fin + - syn + - ack + type: IPv4 + version: 5 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789012" + availability_zone: apse2-az3 + instance: + id: i-0c50d5961bcb2d47b + region: ap-southeast-2 + destination: + address: 10.0.0.71 + ip: 10.0.0.71 + port: 34210 + event: + end: 2021-03-26T03:29:09Z + outcome: success + start: 2021-03-26T03:28:12Z + message: 5 52.95.128.179 10.0.0.71 80 34210 6 1616729292 1616729349 IPv4 14 15044 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 19 52.95.128.179 10.0.0.71 S3 - - ingress OK + network: + bytes: 15044 + direction: ingress + iana_number: "6" + packets: 14 + protocol: tcp + type: ipv4 + related: + ip: + - 52.95.128.179 + - 10.0.0.71 + source: + address: 52.95.128.179 + bytes: 15044 + ip: 52.95.128.179 + packets: 14 + port: 80 +- '@timestamp': 2021-03-26T03:29:09Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dst_aws_service: S3 + pkt_dstaddr: 52.95.128.179 + pkt_srcaddr: 10.0.0.71 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 3 + tcp_flags_array: + - fin + - syn + traffic_path: 8 + type: IPv4 + version: 5 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789012" + availability_zone: apse2-az3 + instance: + id: i-0c50d5961bcb2d47b + region: ap-southeast-2 + destination: + address: 52.95.128.179 + ip: 52.95.128.179 + port: 80 + event: + end: 2021-03-26T03:29:09Z + outcome: success + start: 2021-03-26T03:28:12Z + message: 5 10.0.0.71 52.95.128.179 34210 80 6 1616729292 1616729349 IPv4 7 471 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 3 10.0.0.71 52.95.128.179 - S3 8 egress OK + network: + bytes: 471 + direction: egress + iana_number: "6" + packets: 7 + protocol: tcp + type: ipv4 + related: + ip: + - 10.0.0.71 + - 52.95.128.179 + source: + address: 10.0.0.71 + bytes: 471 + ip: 10.0.0.71 + packets: 7 + port: 34210 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.yml new file mode 100644 index 000000000000..8b1dd4965840 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.yml @@ -0,0 +1,357 @@ +- '@timestamp': 2019-08-26T19:48:53Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 10.0.0.62 + pkt_srcaddr: 52.213.180.42 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 2 + tcp_flags_array: + - syn + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 10.0.0.62 + ip: 10.0.0.62 + port: 5001 + event: + end: 2019-08-26T19:48:53Z + outcome: success + start: 2019-08-26T19:47:55Z + message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK + network: + bytes: 568 + iana_number: "6" + packets: 8 + protocol: tcp + type: ipv4 + related: + ip: + - 52.213.180.42 + - 10.0.0.62 + source: + address: 52.213.180.42 + bytes: 568 + ip: 52.213.180.42 + packets: 8 + port: 43416 +- '@timestamp': 2019-08-26T19:48:53Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 52.213.180.42 + pkt_srcaddr: 10.0.0.62 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 18 + tcp_flags_array: + - syn + - ack + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 52.213.180.42 + ip: 52.213.180.42 + port: 43416 + event: + end: 2019-08-26T19:48:53Z + outcome: success + start: 2019-08-26T19:47:55Z + message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43416 10.0.0.62 52.213.180.42 6 376 7 1566848875 1566848933 ACCEPT 18 OK + network: + bytes: 376 + iana_number: "6" + packets: 7 + protocol: tcp + type: ipv4 + related: + ip: + - 10.0.0.62 + - 52.213.180.42 + source: + address: 10.0.0.62 + bytes: 376 + ip: 10.0.0.62 + packets: 7 + port: 5001 +- '@timestamp': 2019-08-26T19:48:53Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 10.0.0.62 + pkt_srcaddr: 52.213.180.42 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 2 + tcp_flags_array: + - syn + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 10.0.0.62 + ip: 10.0.0.62 + port: 5001 + event: + end: 2019-08-26T19:48:53Z + outcome: success + start: 2019-08-26T19:47:55Z + message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 100701 70 1566848875 1566848933 ACCEPT 2 OK + network: + bytes: 100701 + iana_number: "6" + packets: 70 + protocol: tcp + type: ipv4 + related: + ip: + - 52.213.180.42 + - 10.0.0.62 + source: + address: 52.213.180.42 + bytes: 100701 + ip: 52.213.180.42 + packets: 70 + port: 43418 +- '@timestamp': 2019-08-26T19:48:53Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 52.213.180.42 + pkt_srcaddr: 10.0.0.62 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 18 + tcp_flags_array: + - syn + - ack + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 52.213.180.42 + ip: 52.213.180.42 + port: 43418 + event: + end: 2019-08-26T19:48:53Z + outcome: success + start: 2019-08-26T19:47:55Z + message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 632 12 1566848875 1566848933 ACCEPT 18 OK + network: + bytes: 632 + iana_number: "6" + packets: 12 + protocol: tcp + type: ipv4 + related: + ip: + - 10.0.0.62 + - 52.213.180.42 + source: + address: 10.0.0.62 + bytes: 632 + ip: 10.0.0.62 + packets: 12 + port: 5001 +- '@timestamp': 2019-08-26T19:51:53Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 52.213.180.42 + pkt_srcaddr: 10.0.0.62 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 1 + tcp_flags_array: + - fin + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 52.213.180.42 + ip: 52.213.180.42 + port: 43418 + event: + end: 2019-08-26T19:51:53Z + outcome: success + start: 2019-08-26T19:48:53Z + message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 63388 1219 1566848933 1566849113 ACCEPT 1 OK + network: + bytes: 63388 + iana_number: "6" + packets: 1219 + protocol: tcp + type: ipv4 + related: + ip: + - 10.0.0.62 + - 52.213.180.42 + source: + address: 10.0.0.62 + bytes: 63388 + ip: 10.0.0.62 + packets: 1219 + port: 5001 +- '@timestamp': 2019-08-26T19:51:53Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 10.0.0.62 + pkt_srcaddr: 52.213.180.42 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 1 + tcp_flags_array: + - fin + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 10.0.0.62 + ip: 10.0.0.62 + port: 5001 + event: + end: 2019-08-26T19:51:53Z + outcome: success + start: 2019-08-26T19:48:53Z + message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 23294588 15774 1566848933 1566849113 ACCEPT 1 OK + network: + bytes: 23294588 + iana_number: "6" + packets: 15774 + protocol: tcp + type: ipv4 + related: + ip: + - 52.213.180.42 + - 10.0.0.62 + source: + address: 52.213.180.42 + bytes: 23294588 + ip: 52.213.180.42 + packets: 15774 + port: 43418 +- '@timestamp': 2019-08-27T19:13:13Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 10.0.0.62 + pkt_srcaddr: 52.213.180.42 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 3 + tcp_flags_array: + - fin + - syn + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 10.0.0.62 + ip: 10.0.0.62 + port: 5001 + event: + end: 2019-08-27T19:13:13Z + outcome: success + start: 2019-08-27T19:12:13Z + message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43638 5001 52.213.180.42 10.0.0.62 6 1260 17 1566933133 1566933193 ACCEPT 3 OK + network: + bytes: 1260 + iana_number: "6" + packets: 17 + protocol: tcp + type: ipv4 + related: + ip: + - 52.213.180.42 + - 10.0.0.62 + source: + address: 52.213.180.42 + bytes: 1260 + ip: 52.213.180.42 + packets: 17 + port: 43638 +- '@timestamp': 2019-08-27T19:13:13Z + aws: + vpcflow: + interface_id: eni-1235b8ca123456789 + log_status: OK + pkt_dstaddr: 52.213.180.42 + pkt_srcaddr: 10.0.0.62 + subnet_id: subnet-aaaaaaaa012345678 + tcp_flags: 19 + tcp_flags_array: + - fin + - syn + - ack + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 52.213.180.42 + ip: 52.213.180.42 + port: 43638 + event: + end: 2019-08-27T19:13:13Z + outcome: success + start: 2019-08-27T19:12:13Z + message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43638 10.0.0.62 52.213.180.42 6 967 14 1566933133 1566933193 ACCEPT 19 OK + network: + bytes: 967 + iana_number: "6" + packets: 14 + protocol: tcp + type: ipv4 + related: + ip: + - 10.0.0.62 + - 52.213.180.42 + source: + address: 10.0.0.62 + bytes: 967 + ip: 10.0.0.62 + packets: 14 + port: 5001 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.yml new file mode 100644 index 000000000000..b604c2438657 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.yml @@ -0,0 +1,152 @@ +- aws: + vpcflow: + interface_id: eni-33333333333333333 + log_status: OK + pkt_dstaddr: 10.40.2.236 + pkt_srcaddr: 10.20.33.164 + subnet_id: subnet-22222222bbbbbbbbb + tcp_flags: 3 + tcp_flags_array: + - fin + - syn + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 10.40.2.236 + ip: 10.40.2.236 + port: 80 + event: + outcome: success + message: 3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.20.33.164 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK + network: + iana_number: "6" + protocol: tcp + type: ipv4 + related: + ip: + - 10.20.33.164 + - 10.40.2.236 + source: + address: 10.20.33.164 + ip: 10.20.33.164 + port: 39812 +- aws: + vpcflow: + interface_id: eni-33333333333333333 + log_status: OK + pkt_dstaddr: 10.20.33.164 + pkt_srcaddr: 10.40.2.236 + subnet_id: subnet-22222222bbbbbbbbb + tcp_flags: 19 + tcp_flags_array: + - fin + - syn + - ack + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + instance: + id: i-01234567890123456 + destination: + address: 10.20.33.164 + ip: 10.20.33.164 + port: 39812 + event: + outcome: success + message: 3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.40.2.236 10.20.33.164 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK + network: + iana_number: "6" + protocol: tcp + type: ipv4 + related: + ip: + - 10.40.2.236 + - 10.20.33.164 + source: + address: 10.40.2.236 + ip: 10.40.2.236 + port: 80 +- aws: + vpcflow: + interface_id: eni-11111111111111111 + log_status: OK + pkt_dstaddr: 10.40.2.236 + pkt_srcaddr: 10.20.33.164 + subnet_id: subnet-11111111aaaaaaaaa + tcp_flags: 3 + tcp_flags_array: + - fin + - syn + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + destination: + address: 10.40.2.236 + ip: 10.40.2.236 + port: 80 + event: + outcome: success + message: 3 eni-11111111111111111 123456789010 vpc-abcdefab012345678 subnet-11111111aaaaaaaaa - 10.40.1.175 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK + network: + iana_number: "6" + protocol: tcp + type: ipv4 + related: + ip: + - 10.40.1.175 + - 10.40.2.236 + - 10.20.33.164 + source: + address: 10.40.1.175 + ip: 10.40.1.175 + port: 39812 +- aws: + vpcflow: + interface_id: eni-22222222222222222 + log_status: OK + pkt_dstaddr: 10.20.33.164 + pkt_srcaddr: 10.40.2.236 + subnet_id: subnet-22222222bbbbbbbbb + tcp_flags: 19 + tcp_flags_array: + - fin + - syn + - ack + type: IPv4 + version: 3 + vpc_id: vpc-abcdefab012345678 + cloud: + account: + id: "123456789010" + destination: + address: 10.40.2.31 + ip: 10.40.2.31 + port: 39812 + event: + outcome: success + message: 3 eni-22222222222222222 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb - 10.40.2.236 10.40.2.31 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK + network: + iana_number: "6" + protocol: tcp + type: ipv4 + related: + ip: + - 10.40.2.236 + - 10.40.2.31 + - 10.20.33.164 + source: + address: 10.40.2.236 + ip: 10.40.2.236 + port: 80 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.yml new file mode 100644 index 000000000000..be8719d56e9e --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.yml @@ -0,0 +1,53 @@ +- '@timestamp': 2021-03-26T03:29:09Z + aws: + vpcflow: + interface_id: eni-069xxxxxb7a490 + log_status: OK + pkt_dst_aws_service: CLOUDFRONT + pkt_dstaddr: 10.200.0.80 + pkt_src_aws_service: AMAZON + pkt_srcaddr: 89.160.20.156 + sublocation_id: fake-id + sublocation_type: wavelength + subnet_id: subnet-02d645xxxxxxxdbc0 + tcp_flags: 1 + tcp_flags_array: + - fin + traffic_path: 1 + type: IPv4 + version: 5 + vpc_id: vpc-09676f97xxxxxb8a7 + cloud: + account: + id: "64111117617" + availability_zone: use1-az5 + instance: + id: i-0axxxxxx1ad77 + region: us-east-1 + destination: + address: 10.200.0.0 + ip: 10.200.0.0 + port: 33004 + event: + end: 2021-03-26T03:29:09Z + outcome: failure + start: 2021-03-26T03:28:12Z + message: 5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1 + network: + bytes: 1 + direction: ingress + iana_number: "17" + packets: 52 + protocol: udp + type: ipv4 + related: + ip: + - 89.160.20.156 + - 10.200.0.0 + - 10.200.0.80 + source: + address: 89.160.20.156 + bytes: 1 + ip: 89.160.20.156 + packets: 52 + port: 50041 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.yml new file mode 100644 index 000000000000..eca253eb8a77 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.yml @@ -0,0 +1,68 @@ +- '@timestamp': 2021-03-26T03:29:09Z + aws: + vpcflow: + account_id: "64111117617" + action: REJECT + az_id: use1-az5 + bytes: 1 + dstaddr: 10.200.0.0 + dstport: 33004 + end: 2021-03-26T03:29:09Z + flow_direction: ingress + instance_id: i-0axxxxxx1ad77 + interface_id: eni-069xxxxxb7a490 + log_status: OK + packets: 52 + pkt_dst_aws_service: CLOUDFRONT + pkt_dstaddr: 10.200.0.80 + pkt_src_aws_service: AMAZON + pkt_srcaddr: 89.160.20.156 + protocol: 17 + region: us-east-1 + srcaddr: 89.160.20.156 + srcport: 50041 + start: 2021-03-26T03:28:12Z + sublocation_id: fake-id + sublocation_type: wavelength + subnet_id: subnet-02d645xxxxxxxdbc0 + tcp_flags: 1 + tcp_flags_array: + - fin + traffic_path: 1 + type: IPv4 + version: 5 + vpc_id: vpc-09676f97xxxxxb8a7 + cloud: + account: + id: "64111117617" + availability_zone: use1-az5 + instance: + id: i-0axxxxxx1ad77 + region: us-east-1 + destination: + address: 10.200.0.0 + ip: 10.200.0.0 + port: 33004 + event: + end: 2021-03-26T03:29:09Z + outcome: failure + start: 2021-03-26T03:28:12Z + message: 5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1 + network: + bytes: 1 + direction: ingress + iana_number: "17" + packets: 52 + protocol: udp + type: ipv4 + related: + ip: + - 89.160.20.156 + - 10.200.0.0 + - 10.200.0.80 + source: + address: 89.160.20.156 + bytes: 1 + ip: 89.160.20.156 + packets: 52 + port: 50041 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.yml new file mode 100644 index 000000000000..240d46ddaa93 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.yml @@ -0,0 +1,34 @@ +- aws: + vpcflow: + account_id: "64111117617" + action: REJECT + az_id: use1-az5 + bytes: 1 + dstaddr: 10.200.0.0 + dstport: 33004 + end: 2021-03-26T03:29:09Z + flow_direction: ingress + instance_id: i-0axxxxxx1ad77 + interface_id: eni-069xxxxxb7a490 + log_status: OK + packets: 52 + pkt_dst_aws_service: CLOUDFRONT + pkt_dstaddr: 10.200.0.80 + pkt_src_aws_service: AMAZON + pkt_srcaddr: 89.160.20.156 + protocol: 17 + region: us-east-1 + srcaddr: 89.160.20.156 + srcport: 50041 + start: 2021-03-26T03:28:12Z + sublocation_id: fake-id + sublocation_type: wavelength + subnet_id: subnet-02d645xxxxxxxdbc0 + tcp_flags: 1 + tcp_flags_array: + - fin + traffic_path: 1 + type: IPv4 + version: 5 + vpc_id: vpc-09676f97xxxxxb8a7 + message: 5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1 diff --git a/x-pack/filebeat/processors/aws_vpcflow/types.go b/x-pack/filebeat/processors/aws_vpcflow/types.go new file mode 100644 index 000000000000..1326f723409c --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/types.go @@ -0,0 +1,80 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws_vpcflow + +import ( + "errors" + "fmt" + "net" + "strconv" + "time" +) + +// dataType specifies one of AWS VPC flow field data types. +type dataType uint8 + +// List of DataTypes. +const ( + integerType dataType = iota + 1 + longType + stringType + ipType + timestampType +) + +var dataTypeNames = map[dataType]string{ + integerType: "integer", + longType: "long", + stringType: "string", + ipType: "ip", + timestampType: "timestamp", +} + +func (dt dataType) String() string { + return dataTypeNames[dt] +} + +// toType converts the given value string value to the specified data type. +func toType(value string, typ dataType) (interface{}, error) { + switch typ { + case stringType: + return value, nil + case longType: + return toLong(value) + case integerType: + return toInteger(value) + case ipType: + return toIP(value) + case timestampType: + return toTimestamp(value) + default: + return nil, fmt.Errorf("invalid data type: %v", typ) + } +} + +func toLong(v string) (int64, error) { + return strconv.ParseInt(v, 0, 64) +} + +func toInteger(v string) (int32, error) { + i, err := strconv.ParseInt(v, 0, 32) + return int32(i), err +} + +func toIP(v string) (string, error) { + // This is validating that the value is an IP. + if net.ParseIP(v) != nil { + return v, nil + } + return "", errors.New("value is not a valid IP address") +} + +func toTimestamp(v string) (time.Time, error) { + sec, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return time.Time{}, err + } + return time.Unix(sec, 0).UTC(), nil +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/word_iterator.go b/x-pack/filebeat/processors/aws_vpcflow/word_iterator.go new file mode 100644 index 000000000000..0958d6fb5eb2 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/word_iterator.go @@ -0,0 +1,78 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws_vpcflow + +// NOTE: This is a minor optimization to avoid one allocation +// from strings.Fields during the run() method. In the benchmarks +// it saves 480 bytes and one allocation. That equates to an increase +// of +628 EPS on an M1 Max, but relatively minor ns/op improvement +// of +0.21%. In other words, it might not be worth it. + +// wordIterator iterates over space separated words in an ASCII string. +type wordIterator struct { + source string + currentWord string + currentPosition int + wordIndex int +} + +func newWordIterator(s string) *wordIterator { + return &wordIterator{source: s, wordIndex: -1} +} + +func (itr *wordIterator) Next() bool { + // ASCII fast path + s := itr.source[itr.currentPosition:] + fieldStart := 0 + i := 0 + // Skip spaces in the front of the input. + for i < len(s) && s[i] == ' ' { + i++ + } + fieldStart = i + + for i < len(s) { + if s[i] != ' ' { + i++ + continue + } + itr.currentWord = s[fieldStart:i] + itr.currentPosition += i + itr.wordIndex++ + return true + } + if fieldStart < len(s) { // Last field might end at EOF. + itr.currentWord = s[fieldStart:] + itr.currentPosition += len(s) + itr.wordIndex++ + return true + } + itr.currentWord = "" + itr.wordIndex = -1 + return false +} + +func (itr *wordIterator) Word() string { + return itr.currentWord +} + +func (itr *wordIterator) WordIndex() int { + return itr.wordIndex +} + +func (itr *wordIterator) Count() int { + n := 0 + wasSpace := true + + for i := 0; i < len(itr.source); i++ { + r := itr.source[i] + isSpace := r == ' ' + if wasSpace && !isSpace { + n++ + } + wasSpace = isSpace + } + return n +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/word_iterator_test.go b/x-pack/filebeat/processors/aws_vpcflow/word_iterator_test.go new file mode 100644 index 000000000000..d1d4c9fb0502 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/word_iterator_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws_vpcflow + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWordIterator(t *testing.T) { + testCases := []struct { + sentence string + words []string + }{ + {"", nil}, + {" ", nil}, + {" ", nil}, + {"word", []string{"word"}}, + {" word", []string{"word"}}, + {"word ", []string{"word"}}, + {" word ", []string{"word"}}, + {"foo bar baz", []string{"foo", "bar", "baz"}}, + } + + for _, tc := range testCases { + itr := wordIterator{source: tc.sentence} + + var collectedWords []string + for itr.Next() { + collectedWords = append(collectedWords, itr.Word()) + } + + assert.Equal(t, tc.words, collectedWords) + assert.Equal(t, len(tc.words), itr.Count()) + } +} diff --git a/x-pack/functionbeat/docs/index.asciidoc b/x-pack/functionbeat/docs/index.asciidoc index 2fc5cdc98538..0ecfb31faee5 100644 --- a/x-pack/functionbeat/docs/index.asciidoc +++ b/x-pack/functionbeat/docs/index.asciidoc @@ -29,6 +29,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :no_repos: :no_decode_cef_processor: :no_decode_csv_fields_processor: +:no_parse_aws_vpc_flow_log_processor: :no_script_processor: :no_timestamp_processor: :no_keystore: From 6bcd888581fcb08e7393d3ef16b0d1c21f48780a Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Sat, 12 Nov 2022 02:26:46 -0500 Subject: [PATCH 02/22] Use yaml.v2 b/c it's in go.mod --- x-pack/filebeat/processors/aws_vpcflow/config.go | 10 ++++++---- .../aws_vpcflow/parse_aws_vpc_flow_log_test.go | 4 +--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/config.go b/x-pack/filebeat/processors/aws_vpcflow/config.go index 967aba127da4..b4c027c29dc5 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/config.go +++ b/x-pack/filebeat/processors/aws_vpcflow/config.go @@ -8,8 +8,6 @@ import ( "errors" "fmt" "strings" - - "gopkg.in/yaml.v3" ) // mode represents the processing mode (original, ecs, ecs_and_original). @@ -37,8 +35,12 @@ func (m *mode) Unpack(s string) error { return fmt.Errorf("invalid mode type <%v> for "+procName, s) } -func (m *mode) UnmarshalYAML(value *yaml.Node) error { - return m.Unpack(value.Value) +func (m *mode) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + return m.Unpack(str) } func (m *mode) String() string { diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index 1356ae270bb2..5928ce3507f6 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "gopkg.in/yaml.v3" + "gopkg.in/yaml.v2" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" @@ -202,7 +202,6 @@ func readGoldenTestCase(t testing.TB) []GoldenTestCase { } dec := yaml.NewDecoder(f) - dec.KnownFields(true) var testCases map[string]GoldenTestCase if err = dec.Decode(&testCases); err != nil { @@ -226,7 +225,6 @@ func writeGolden(t testing.TB, path string, events []mapstr.M) { defer f.Close() enc := yaml.NewEncoder(f) - enc.SetIndent(2) if err = enc.Encode(events); err != nil { t.Fatal() } From 26c01b1a466ce975151788addef42383cf900d95 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Sat, 12 Nov 2022 02:33:37 -0500 Subject: [PATCH 03/22] Add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 42685e384ecd..0b622c91c64a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -156,6 +156,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Improve httpjson documentation for split processor. {pull}33473[33473] - Added separation of transform context object inside httpjson. Introduced new clause `.parent_last_response.*` {pull}33499[33499] - Cloud Foundry input uses server-side filtering when retrieving logs. {pull}33456[33456] +- Add `parse_aws_vpc_flow_log` processor. {pull}33656[33656] *Auditbeat* From 3f3effbd52031209d83a67dec2ce2edbf2e16849 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Nov 2022 16:28:49 -0500 Subject: [PATCH 04/22] Apply suggestions from code review Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- x-pack/filebeat/processors/aws_vpcflow/config.go | 5 ++++- .../processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go | 1 + x-pack/filebeat/processors/aws_vpcflow/types.go | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/config.go b/x-pack/filebeat/processors/aws_vpcflow/config.go index b4c027c29dc5..6031eea46bfc 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/config.go +++ b/x-pack/filebeat/processors/aws_vpcflow/config.go @@ -32,7 +32,7 @@ func (m *mode) Unpack(s string) error { return nil } } - return fmt.Errorf("invalid mode type <%v> for "+procName, s) + return fmt.Errorf("invalid mode type %q for "+procName, s) } func (m *mode) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -44,6 +44,9 @@ func (m *mode) UnmarshalYAML(unmarshal func(interface{}) error) error { } func (m *mode) String() string { + if m == nil { + return "" + } if s, found := modeStrings[*m]; found { return s } diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index 5928ce3507f6..ae222562a9b4 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -279,6 +279,7 @@ func BenchmarkProcessorRun(b *testing.B) { }, } + b.ResetTimer() for i := 0; i < b.N; i++ { if _, err = p.Run(&evt); err != nil { b.Fatal(err) diff --git a/x-pack/filebeat/processors/aws_vpcflow/types.go b/x-pack/filebeat/processors/aws_vpcflow/types.go index 1326f723409c..08926f944e47 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/types.go +++ b/x-pack/filebeat/processors/aws_vpcflow/types.go @@ -33,6 +33,9 @@ var dataTypeNames = map[dataType]string{ } func (dt dataType) String() string { + if dt < integerType || timeStampType < dt { + return fmt.Sprintf("invaild(%d)", dt) + } return dataTypeNames[dt] } From 610e0cd8d95f847a45ba4b6587b0dc0046408625 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Nov 2022 16:31:09 -0500 Subject: [PATCH 05/22] Add json encoding comment to String() --- x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go index fa065e1fd0ea..32f6cc7ab95f 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go @@ -72,6 +72,7 @@ func newParseAWSVPCFlowLog(c config) (*processor, error) { } func (p *processor) String() string { + // JSON encoding of the config struct should never cause an error. json, _ := json.Marshal(p.config) return procName + "=" + string(json) } From ab1dd4ae06403b8fd0a3508977c1104b5cfcc9bc Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Nov 2022 16:31:40 -0500 Subject: [PATCH 06/22] rename GoldenTestCase -> goldenTestCase --- .../processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index ae222562a9b4..679f147cf224 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -186,14 +186,14 @@ func TestGoldenFile(t *testing.T) { } } -type GoldenTestCase struct { +type goldenTestCase struct { Name string `yaml:"-"` // Name of test. Mode *mode // Processing mode (what fields to generate). Format string // Flow log format. Samples []string // List of sample logs to parse. } -func readGoldenTestCase(t testing.TB) []GoldenTestCase { +func readGoldenTestCase(t testing.TB) []goldenTestCase { t.Helper() f, err := os.Open("testdata/aws-vpc-flow-logs.yml") @@ -203,12 +203,12 @@ func readGoldenTestCase(t testing.TB) []GoldenTestCase { dec := yaml.NewDecoder(f) - var testCases map[string]GoldenTestCase + var testCases map[string]goldenTestCase if err = dec.Decode(&testCases); err != nil { t.Fatal(err) } - testCasesList := make([]GoldenTestCase, 0, len(testCases)) + testCasesList := make([]goldenTestCase, 0, len(testCases)) for k, v := range testCases { v.Name = k testCasesList = append(testCasesList, v) From be1977f50f24ef997842061a5b2196377d4c3784 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Nov 2022 16:32:27 -0500 Subject: [PATCH 07/22] Add ResetTimer --- .../processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index 679f147cf224..cc32454342bd 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -257,6 +257,7 @@ func BenchmarkProcessorRun(b *testing.B) { }, } + b.ResetTimer() for i := 0; i < b.N; i++ { if _, err = p.Run(&evt); err != nil { b.Fatal(err) From 5897d054bcc5e4acd76191011b4b55e6c2ac74c2 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Nov 2022 16:33:33 -0500 Subject: [PATCH 08/22] Typo in timestampType --- x-pack/filebeat/processors/aws_vpcflow/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/types.go b/x-pack/filebeat/processors/aws_vpcflow/types.go index 08926f944e47..6c733c331fb1 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/types.go +++ b/x-pack/filebeat/processors/aws_vpcflow/types.go @@ -33,7 +33,7 @@ var dataTypeNames = map[dataType]string{ } func (dt dataType) String() string { - if dt < integerType || timeStampType < dt { + if dt < integerType || timestampType < dt { return fmt.Sprintf("invaild(%d)", dt) } return dataTypeNames[dt] From a22d80c4e398da5d852bc3d0676ba8a1d3266651 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Nov 2022 21:19:00 -0500 Subject: [PATCH 09/22] testing.TB -> *testing.T --- .../processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index cc32454342bd..a345738f5e6e 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -193,7 +193,7 @@ type goldenTestCase struct { Samples []string // List of sample logs to parse. } -func readGoldenTestCase(t testing.TB) []goldenTestCase { +func readGoldenTestCase(t *testing.T) []goldenTestCase { t.Helper() f, err := os.Open("testdata/aws-vpc-flow-logs.yml") @@ -217,7 +217,7 @@ func readGoldenTestCase(t testing.TB) []goldenTestCase { return testCasesList } -func writeGolden(t testing.TB, path string, events []mapstr.M) { +func writeGolden(t *testing.T, path string, events []mapstr.M) { t.Helper() f, err := os.Create(path) @@ -230,7 +230,7 @@ func writeGolden(t testing.TB, path string, events []mapstr.M) { } } -func readGolden(t testing.TB, path string) string { +func readGolden(t *testing.T, path string) string { t.Helper() yamlData, err := os.ReadFile(path) From 7ab42c911415997ac6a571d2b4740bf0431172b1 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Nov 2022 21:25:42 -0500 Subject: [PATCH 10/22] Store golden.json instead of golden.yml --- .../parse_aws_vpc_flow_log_test.go | 19 +- .../testdata/custom-nat-gateway.golden.json | 176 +++++++ .../testdata/custom-nat-gateway.golden.yml | 116 ----- .../testdata/default-v2-format.golden.json | 272 ++++++++++ .../testdata/default-v2-format.golden.yml | 196 -------- .../service-name-path-direction.golden.json | 130 +++++ .../service-name-path-direction.golden.yml | 100 ---- .../testdata/tcp-flag-sequence.golden.json | 471 ++++++++++++++++++ .../testdata/tcp-flag-sequence.golden.yml | 357 ------------- .../testdata/transit-gateway.golden.json | 208 ++++++++ .../testdata/transit-gateway.golden.yml | 152 ------ .../testdata/v5-fields-ecs.golden.json | 69 +++ .../testdata/v5-fields-ecs.golden.yml | 53 -- .../v5-fields-ecs_and_original.golden.json | 84 ++++ .../v5-fields-ecs_and_original.golden.yml | 68 --- .../testdata/v5-fields-original.golden.json | 41 ++ .../testdata/v5-fields-original.golden.yml | 34 -- 17 files changed, 1462 insertions(+), 1084 deletions(-) create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.json delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.yml create mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.json delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.yml diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index a345738f5e6e..07717b67e83a 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -5,6 +5,7 @@ package aws_vpcflow import ( + "encoding/json" "flag" "os" "path/filepath" @@ -140,7 +141,7 @@ func TestGoldenFile(t *testing.T) { if *updateGolden { // Delete existing golden files. - goldens, _ := filepath.Glob("testdata/*.golden.yml") + goldens, _ := filepath.Glob("testdata/*.golden.*") for _, golden := range goldens { os.Remove(golden) } @@ -171,16 +172,16 @@ func TestGoldenFile(t *testing.T) { observed = append(observed, out.Fields) } - goldenFile := filepath.Join("testdata", tc.Name+".golden.yml") + goldenFile := filepath.Join("testdata", tc.Name+".golden.json") if *updateGolden { writeGolden(t, goldenFile, observed) } else { - expectedYAML := readGolden(t, goldenFile) + expectedJSON := readGolden(t, goldenFile) - observedYAML, err := yaml.Marshal(observed) + observedJSON, err := json.Marshal(observed) require.NoError(t, err) - assert.YAMLEq(t, expectedYAML, string(observedYAML)) + assert.JSONEq(t, expectedJSON, string(observedJSON)) } }) } @@ -224,7 +225,9 @@ func writeGolden(t *testing.T, path string, events []mapstr.M) { require.NoError(t, err) defer f.Close() - enc := yaml.NewEncoder(f) + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + enc.SetEscapeHTML(false) if err = enc.Encode(events); err != nil { t.Fatal() } @@ -233,12 +236,12 @@ func writeGolden(t *testing.T, path string, events []mapstr.M) { func readGolden(t *testing.T, path string) string { t.Helper() - yamlData, err := os.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { t.Fatal(err) } - return string(yamlData) + return string(data) } func BenchmarkProcessorRun(b *testing.B) { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.json new file mode 100644 index 000000000000..08f94b6b5f46 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.json @@ -0,0 +1,176 @@ +[ + { + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "pkt_dstaddr": "203.0.113.5", + "pkt_srcaddr": "10.0.1.5" + } + }, + "destination": { + "address": "10.0.0.220", + "ip": "10.0.0.220" + }, + "message": "- eni-1235b8ca123456789 10.0.1.5 10.0.0.220 10.0.1.5 203.0.113.5", + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.1.5", + "10.0.0.220", + "203.0.113.5" + ] + }, + "source": { + "address": "10.0.1.5", + "ip": "10.0.1.5" + } + }, + { + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "pkt_dstaddr": "203.0.113.5", + "pkt_srcaddr": "10.0.0.220" + } + }, + "destination": { + "address": "203.0.113.5", + "ip": "203.0.113.5" + }, + "message": "- eni-1235b8ca123456789 10.0.0.220 203.0.113.5 10.0.0.220 203.0.113.5", + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.0.220", + "203.0.113.5" + ] + }, + "source": { + "address": "10.0.0.220", + "ip": "10.0.0.220" + } + }, + { + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "pkt_dstaddr": "10.0.0.220", + "pkt_srcaddr": "203.0.113.5" + } + }, + "destination": { + "address": "10.0.0.220", + "ip": "10.0.0.220" + }, + "message": "- eni-1235b8ca123456789 203.0.113.5 10.0.0.220 203.0.113.5 10.0.0.220", + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "203.0.113.5", + "10.0.0.220" + ] + }, + "source": { + "address": "203.0.113.5", + "ip": "203.0.113.5" + } + }, + { + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "pkt_dstaddr": "10.0.1.5", + "pkt_srcaddr": "203.0.113.5" + } + }, + "destination": { + "address": "10.0.1.5", + "ip": "10.0.1.5" + }, + "message": "- eni-1235b8ca123456789 10.0.0.220 10.0.1.5 203.0.113.5 10.0.1.5", + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.0.220", + "10.0.1.5", + "203.0.113.5" + ] + }, + "source": { + "address": "10.0.0.220", + "ip": "10.0.0.220" + } + }, + { + "aws": { + "vpcflow": { + "interface_id": "eni-1111aaaa2222bbbb3", + "pkt_dstaddr": "203.0.113.5", + "pkt_srcaddr": "10.0.1.5" + } + }, + "cloud": { + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "203.0.113.5", + "ip": "203.0.113.5" + }, + "message": "i-01234567890123456 eni-1111aaaa2222bbbb3 10.0.1.5 203.0.113.5 10.0.1.5 203.0.113.5", + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.1.5", + "203.0.113.5" + ] + }, + "source": { + "address": "10.0.1.5", + "ip": "10.0.1.5" + } + }, + { + "aws": { + "vpcflow": { + "interface_id": "eni-1111aaaa2222bbbb3", + "pkt_dstaddr": "10.0.1.5", + "pkt_srcaddr": "203.0.113.5" + } + }, + "cloud": { + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "10.0.1.5", + "ip": "10.0.1.5" + }, + "message": "i-01234567890123456 eni-1111aaaa2222bbbb3 203.0.113.5 10.0.1.5 203.0.113.5 10.0.1.5", + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "203.0.113.5", + "10.0.1.5" + ] + }, + "source": { + "address": "203.0.113.5", + "ip": "203.0.113.5" + } + } +] diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.yml deleted file mode 100644 index d2a28dd8aa58..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.yml +++ /dev/null @@ -1,116 +0,0 @@ -- aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - pkt_dstaddr: 203.0.113.5 - pkt_srcaddr: 10.0.1.5 - destination: - address: 10.0.0.220 - ip: 10.0.0.220 - message: '- eni-1235b8ca123456789 10.0.1.5 10.0.0.220 10.0.1.5 203.0.113.5' - network: - type: ipv4 - related: - ip: - - 10.0.1.5 - - 10.0.0.220 - - 203.0.113.5 - source: - address: 10.0.1.5 - ip: 10.0.1.5 -- aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - pkt_dstaddr: 203.0.113.5 - pkt_srcaddr: 10.0.0.220 - destination: - address: 203.0.113.5 - ip: 203.0.113.5 - message: '- eni-1235b8ca123456789 10.0.0.220 203.0.113.5 10.0.0.220 203.0.113.5' - network: - type: ipv4 - related: - ip: - - 10.0.0.220 - - 203.0.113.5 - source: - address: 10.0.0.220 - ip: 10.0.0.220 -- aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - pkt_dstaddr: 10.0.0.220 - pkt_srcaddr: 203.0.113.5 - destination: - address: 10.0.0.220 - ip: 10.0.0.220 - message: '- eni-1235b8ca123456789 203.0.113.5 10.0.0.220 203.0.113.5 10.0.0.220' - network: - type: ipv4 - related: - ip: - - 203.0.113.5 - - 10.0.0.220 - source: - address: 203.0.113.5 - ip: 203.0.113.5 -- aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - pkt_dstaddr: 10.0.1.5 - pkt_srcaddr: 203.0.113.5 - destination: - address: 10.0.1.5 - ip: 10.0.1.5 - message: '- eni-1235b8ca123456789 10.0.0.220 10.0.1.5 203.0.113.5 10.0.1.5' - network: - type: ipv4 - related: - ip: - - 10.0.0.220 - - 10.0.1.5 - - 203.0.113.5 - source: - address: 10.0.0.220 - ip: 10.0.0.220 -- aws: - vpcflow: - interface_id: eni-1111aaaa2222bbbb3 - pkt_dstaddr: 203.0.113.5 - pkt_srcaddr: 10.0.1.5 - cloud: - instance: - id: i-01234567890123456 - destination: - address: 203.0.113.5 - ip: 203.0.113.5 - message: i-01234567890123456 eni-1111aaaa2222bbbb3 10.0.1.5 203.0.113.5 10.0.1.5 203.0.113.5 - network: - type: ipv4 - related: - ip: - - 10.0.1.5 - - 203.0.113.5 - source: - address: 10.0.1.5 - ip: 10.0.1.5 -- aws: - vpcflow: - interface_id: eni-1111aaaa2222bbbb3 - pkt_dstaddr: 10.0.1.5 - pkt_srcaddr: 203.0.113.5 - cloud: - instance: - id: i-01234567890123456 - destination: - address: 10.0.1.5 - ip: 10.0.1.5 - message: i-01234567890123456 eni-1111aaaa2222bbbb3 203.0.113.5 10.0.1.5 203.0.113.5 10.0.1.5 - network: - type: ipv4 - related: - ip: - - 203.0.113.5 - - 10.0.1.5 - source: - address: 203.0.113.5 - ip: 203.0.113.5 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json new file mode 100644 index 000000000000..13fdd4d238d4 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json @@ -0,0 +1,272 @@ +[ + { + "@timestamp": "2014-12-14T04:07:50Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "version": 2 + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "destination": { + "address": "172.31.16.21", + "ip": "172.31.16.21", + "port": 22 + }, + "event": { + "end": "2014-12-14T04:07:50Z", + "outcome": "success", + "start": "2014-12-14T04:06:50Z" + }, + "message": "2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK", + "network": { + "bytes": 4249, + "iana_number": "6", + "packets": 20, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "172.31.16.139", + "172.31.16.21" + ] + }, + "source": { + "address": "172.31.16.139", + "bytes": 4249, + "ip": "172.31.16.139", + "packets": 20, + "port": 20641 + } + }, + { + "@timestamp": "2014-12-14T04:07:50Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "version": 2 + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "destination": { + "address": "172.31.9.12", + "ip": "172.31.9.12", + "port": 3389 + }, + "event": { + "end": "2014-12-14T04:07:50Z", + "outcome": "failure", + "start": "2014-12-14T04:06:50Z" + }, + "message": "2 123456789010 eni-1235b8ca123456789 172.31.9.69 172.31.9.12 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK", + "network": { + "bytes": 4249, + "iana_number": "6", + "packets": 20, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "172.31.9.69", + "172.31.9.12" + ] + }, + "source": { + "address": "172.31.9.69", + "bytes": 4249, + "ip": "172.31.9.69", + "packets": 20, + "port": 49761 + } + }, + { + "@timestamp": "2015-05-10T18:02:14Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "NODATA", + "version": 2 + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "event": { + "end": "2015-05-10T18:02:14Z", + "start": "2015-05-10T18:01:16Z" + }, + "message": "2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA" + }, + { + "@timestamp": "2015-05-10T18:02:14Z", + "aws": { + "vpcflow": { + "interface_id": "eni-11111111aaaaaaaaa", + "log_status": "SKIPDATA", + "version": 2 + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "event": { + "end": "2015-05-10T18:02:14Z", + "start": "2015-05-10T18:01:16Z" + }, + "message": "2 123456789010 eni-11111111aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA" + }, + { + "@timestamp": "2015-05-29T16:32:22Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "version": 2 + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "destination": { + "address": "172.31.16.139", + "ip": "172.31.16.139", + "port": 0 + }, + "event": { + "end": "2015-05-29T16:32:22Z", + "outcome": "success", + "start": "2015-05-29T16:30:27Z" + }, + "message": "2 123456789010 eni-1235b8ca123456789 203.0.113.12 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK", + "network": { + "bytes": 336, + "iana_number": "1", + "packets": 4, + "protocol": "icmp", + "type": "ipv4" + }, + "related": { + "ip": [ + "203.0.113.12", + "172.31.16.139" + ] + }, + "source": { + "address": "203.0.113.12", + "bytes": 336, + "ip": "203.0.113.12", + "packets": 4, + "port": 0 + } + }, + { + "@timestamp": "2015-05-29T16:32:22Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "version": 2 + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "destination": { + "address": "203.0.113.12", + "ip": "203.0.113.12", + "port": 0 + }, + "event": { + "end": "2015-05-29T16:32:22Z", + "outcome": "failure", + "start": "2015-05-29T16:31:34Z" + }, + "message": "2 123456789010 eni-1235b8ca123456789 172.31.16.139 203.0.113.12 0 0 1 4 336 1432917094 1432917142 REJECT OK", + "network": { + "bytes": 336, + "iana_number": "1", + "packets": 4, + "protocol": "icmp", + "type": "ipv4" + }, + "related": { + "ip": [ + "172.31.16.139", + "203.0.113.12" + ] + }, + "source": { + "address": "172.31.16.139", + "bytes": 336, + "ip": "172.31.16.139", + "packets": 4, + "port": 0 + } + }, + { + "@timestamp": "2016-10-31T11:37:00Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "version": 2 + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "destination": { + "address": "2001:db8:1234:a102:3304:8879:34cf:4071", + "ip": "2001:db8:1234:a102:3304:8879:34cf:4071", + "port": 22 + }, + "event": { + "end": "2016-10-31T11:37:00Z", + "outcome": "success", + "start": "2016-10-31T11:35:08Z" + }, + "message": "2 123456789010 eni-1235b8ca123456789 2001:db8:1234:a100:8d6e:3477:df66:f105 2001:db8:1234:a102:3304:8879:34cf:4071 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK", + "network": { + "bytes": 8855, + "iana_number": "6", + "packets": 54, + "protocol": "tcp", + "type": "ipv6" + }, + "related": { + "ip": [ + "2001:db8:1234:a100:8d6e:3477:df66:f105", + "2001:db8:1234:a102:3304:8879:34cf:4071" + ] + }, + "source": { + "address": "2001:db8:1234:a100:8d6e:3477:df66:f105", + "bytes": 8855, + "ip": "2001:db8:1234:a100:8d6e:3477:df66:f105", + "packets": 54, + "port": 34892 + } + } +] diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.yml deleted file mode 100644 index 5c2f91044ce4..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.yml +++ /dev/null @@ -1,196 +0,0 @@ -- '@timestamp': 2014-12-14T04:07:50Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - version: 2 - cloud: - account: - id: "123456789010" - destination: - address: 172.31.16.21 - ip: 172.31.16.21 - port: 22 - event: - end: 2014-12-14T04:07:50Z - outcome: success - start: 2014-12-14T04:06:50Z - message: 2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK - network: - bytes: 4249 - iana_number: "6" - packets: 20 - protocol: tcp - type: ipv4 - related: - ip: - - 172.31.16.139 - - 172.31.16.21 - source: - address: 172.31.16.139 - bytes: 4249 - ip: 172.31.16.139 - packets: 20 - port: 20641 -- '@timestamp': 2014-12-14T04:07:50Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - version: 2 - cloud: - account: - id: "123456789010" - destination: - address: 172.31.9.12 - ip: 172.31.9.12 - port: 3389 - event: - end: 2014-12-14T04:07:50Z - outcome: failure - start: 2014-12-14T04:06:50Z - message: 2 123456789010 eni-1235b8ca123456789 172.31.9.69 172.31.9.12 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK - network: - bytes: 4249 - iana_number: "6" - packets: 20 - protocol: tcp - type: ipv4 - related: - ip: - - 172.31.9.69 - - 172.31.9.12 - source: - address: 172.31.9.69 - bytes: 4249 - ip: 172.31.9.69 - packets: 20 - port: 49761 -- '@timestamp': 2015-05-10T18:02:14Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: NODATA - version: 2 - cloud: - account: - id: "123456789010" - event: - end: 2015-05-10T18:02:14Z - start: 2015-05-10T18:01:16Z - message: 2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA -- '@timestamp': 2015-05-10T18:02:14Z - aws: - vpcflow: - interface_id: eni-11111111aaaaaaaaa - log_status: SKIPDATA - version: 2 - cloud: - account: - id: "123456789010" - event: - end: 2015-05-10T18:02:14Z - start: 2015-05-10T18:01:16Z - message: 2 123456789010 eni-11111111aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA -- '@timestamp': 2015-05-29T16:32:22Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - version: 2 - cloud: - account: - id: "123456789010" - destination: - address: 172.31.16.139 - ip: 172.31.16.139 - port: 0 - event: - end: 2015-05-29T16:32:22Z - outcome: success - start: 2015-05-29T16:30:27Z - message: 2 123456789010 eni-1235b8ca123456789 203.0.113.12 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK - network: - bytes: 336 - iana_number: "1" - packets: 4 - protocol: icmp - type: ipv4 - related: - ip: - - 203.0.113.12 - - 172.31.16.139 - source: - address: 203.0.113.12 - bytes: 336 - ip: 203.0.113.12 - packets: 4 - port: 0 -- '@timestamp': 2015-05-29T16:32:22Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - version: 2 - cloud: - account: - id: "123456789010" - destination: - address: 203.0.113.12 - ip: 203.0.113.12 - port: 0 - event: - end: 2015-05-29T16:32:22Z - outcome: failure - start: 2015-05-29T16:31:34Z - message: 2 123456789010 eni-1235b8ca123456789 172.31.16.139 203.0.113.12 0 0 1 4 336 1432917094 1432917142 REJECT OK - network: - bytes: 336 - iana_number: "1" - packets: 4 - protocol: icmp - type: ipv4 - related: - ip: - - 172.31.16.139 - - 203.0.113.12 - source: - address: 172.31.16.139 - bytes: 336 - ip: 172.31.16.139 - packets: 4 - port: 0 -- '@timestamp': 2016-10-31T11:37:00Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - version: 2 - cloud: - account: - id: "123456789010" - destination: - address: 2001:db8:1234:a102:3304:8879:34cf:4071 - ip: 2001:db8:1234:a102:3304:8879:34cf:4071 - port: 22 - event: - end: 2016-10-31T11:37:00Z - outcome: success - start: 2016-10-31T11:35:08Z - message: 2 123456789010 eni-1235b8ca123456789 2001:db8:1234:a100:8d6e:3477:df66:f105 2001:db8:1234:a102:3304:8879:34cf:4071 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK - network: - bytes: 8855 - iana_number: "6" - packets: 54 - protocol: tcp - type: ipv6 - related: - ip: - - 2001:db8:1234:a100:8d6e:3477:df66:f105 - - 2001:db8:1234:a102:3304:8879:34cf:4071 - source: - address: 2001:db8:1234:a100:8d6e:3477:df66:f105 - bytes: 8855 - ip: 2001:db8:1234:a100:8d6e:3477:df66:f105 - packets: 54 - port: 34892 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json new file mode 100644 index 000000000000..3ee4021df5f5 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json @@ -0,0 +1,130 @@ +[ + { + "@timestamp": "2021-03-26T03:29:09Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "10.0.0.71", + "pkt_src_aws_service": "S3", + "pkt_srcaddr": "52.95.128.179", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 19, + "tcp_flags_array": [ + "fin", + "syn", + "ack" + ], + "type": "IPv4", + "version": 5, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789012" + }, + "availability_zone": "apse2-az3", + "instance": { + "id": "i-0c50d5961bcb2d47b" + }, + "region": "ap-southeast-2" + }, + "destination": { + "address": "10.0.0.71", + "ip": "10.0.0.71", + "port": 34210 + }, + "event": { + "end": "2021-03-26T03:29:09Z", + "outcome": "success", + "start": "2021-03-26T03:28:12Z" + }, + "message": "5 52.95.128.179 10.0.0.71 80 34210 6 1616729292 1616729349 IPv4 14 15044 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 19 52.95.128.179 10.0.0.71 S3 - - ingress OK", + "network": { + "bytes": 15044, + "direction": "ingress", + "iana_number": "6", + "packets": 14, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "52.95.128.179", + "10.0.0.71" + ] + }, + "source": { + "address": "52.95.128.179", + "bytes": 15044, + "ip": "52.95.128.179", + "packets": 14, + "port": 80 + } + }, + { + "@timestamp": "2021-03-26T03:29:09Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dst_aws_service": "S3", + "pkt_dstaddr": "52.95.128.179", + "pkt_srcaddr": "10.0.0.71", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 3, + "tcp_flags_array": [ + "fin", + "syn" + ], + "traffic_path": 8, + "type": "IPv4", + "version": 5, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789012" + }, + "availability_zone": "apse2-az3", + "instance": { + "id": "i-0c50d5961bcb2d47b" + }, + "region": "ap-southeast-2" + }, + "destination": { + "address": "52.95.128.179", + "ip": "52.95.128.179", + "port": 80 + }, + "event": { + "end": "2021-03-26T03:29:09Z", + "outcome": "success", + "start": "2021-03-26T03:28:12Z" + }, + "message": "5 10.0.0.71 52.95.128.179 34210 80 6 1616729292 1616729349 IPv4 7 471 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 3 10.0.0.71 52.95.128.179 - S3 8 egress OK", + "network": { + "bytes": 471, + "direction": "egress", + "iana_number": "6", + "packets": 7, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.0.71", + "52.95.128.179" + ] + }, + "source": { + "address": "10.0.0.71", + "bytes": 471, + "ip": "10.0.0.71", + "packets": 7, + "port": 34210 + } + } +] diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.yml deleted file mode 100644 index 6b87826786f0..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.yml +++ /dev/null @@ -1,100 +0,0 @@ -- '@timestamp': 2021-03-26T03:29:09Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 10.0.0.71 - pkt_src_aws_service: S3 - pkt_srcaddr: 52.95.128.179 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 19 - tcp_flags_array: - - fin - - syn - - ack - type: IPv4 - version: 5 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789012" - availability_zone: apse2-az3 - instance: - id: i-0c50d5961bcb2d47b - region: ap-southeast-2 - destination: - address: 10.0.0.71 - ip: 10.0.0.71 - port: 34210 - event: - end: 2021-03-26T03:29:09Z - outcome: success - start: 2021-03-26T03:28:12Z - message: 5 52.95.128.179 10.0.0.71 80 34210 6 1616729292 1616729349 IPv4 14 15044 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 19 52.95.128.179 10.0.0.71 S3 - - ingress OK - network: - bytes: 15044 - direction: ingress - iana_number: "6" - packets: 14 - protocol: tcp - type: ipv4 - related: - ip: - - 52.95.128.179 - - 10.0.0.71 - source: - address: 52.95.128.179 - bytes: 15044 - ip: 52.95.128.179 - packets: 14 - port: 80 -- '@timestamp': 2021-03-26T03:29:09Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dst_aws_service: S3 - pkt_dstaddr: 52.95.128.179 - pkt_srcaddr: 10.0.0.71 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 3 - tcp_flags_array: - - fin - - syn - traffic_path: 8 - type: IPv4 - version: 5 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789012" - availability_zone: apse2-az3 - instance: - id: i-0c50d5961bcb2d47b - region: ap-southeast-2 - destination: - address: 52.95.128.179 - ip: 52.95.128.179 - port: 80 - event: - end: 2021-03-26T03:29:09Z - outcome: success - start: 2021-03-26T03:28:12Z - message: 5 10.0.0.71 52.95.128.179 34210 80 6 1616729292 1616729349 IPv4 7 471 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 3 10.0.0.71 52.95.128.179 - S3 8 egress OK - network: - bytes: 471 - direction: egress - iana_number: "6" - packets: 7 - protocol: tcp - type: ipv4 - related: - ip: - - 10.0.0.71 - - 52.95.128.179 - source: - address: 10.0.0.71 - bytes: 471 - ip: 10.0.0.71 - packets: 7 - port: 34210 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json new file mode 100644 index 000000000000..2304ffda13df --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json @@ -0,0 +1,471 @@ +[ + { + "@timestamp": "2019-08-26T19:48:53Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "10.0.0.62", + "pkt_srcaddr": "52.213.180.42", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 2, + "tcp_flags_array": [ + "syn" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "10.0.0.62", + "ip": "10.0.0.62", + "port": 5001 + }, + "event": { + "end": "2019-08-26T19:48:53Z", + "outcome": "success", + "start": "2019-08-26T19:47:55Z" + }, + "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK", + "network": { + "bytes": 568, + "iana_number": "6", + "packets": 8, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "52.213.180.42", + "10.0.0.62" + ] + }, + "source": { + "address": "52.213.180.42", + "bytes": 568, + "ip": "52.213.180.42", + "packets": 8, + "port": 43416 + } + }, + { + "@timestamp": "2019-08-26T19:48:53Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "52.213.180.42", + "pkt_srcaddr": "10.0.0.62", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 18, + "tcp_flags_array": [ + "syn", + "ack" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "52.213.180.42", + "ip": "52.213.180.42", + "port": 43416 + }, + "event": { + "end": "2019-08-26T19:48:53Z", + "outcome": "success", + "start": "2019-08-26T19:47:55Z" + }, + "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43416 10.0.0.62 52.213.180.42 6 376 7 1566848875 1566848933 ACCEPT 18 OK", + "network": { + "bytes": 376, + "iana_number": "6", + "packets": 7, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.0.62", + "52.213.180.42" + ] + }, + "source": { + "address": "10.0.0.62", + "bytes": 376, + "ip": "10.0.0.62", + "packets": 7, + "port": 5001 + } + }, + { + "@timestamp": "2019-08-26T19:48:53Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "10.0.0.62", + "pkt_srcaddr": "52.213.180.42", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 2, + "tcp_flags_array": [ + "syn" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "10.0.0.62", + "ip": "10.0.0.62", + "port": 5001 + }, + "event": { + "end": "2019-08-26T19:48:53Z", + "outcome": "success", + "start": "2019-08-26T19:47:55Z" + }, + "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 100701 70 1566848875 1566848933 ACCEPT 2 OK", + "network": { + "bytes": 100701, + "iana_number": "6", + "packets": 70, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "52.213.180.42", + "10.0.0.62" + ] + }, + "source": { + "address": "52.213.180.42", + "bytes": 100701, + "ip": "52.213.180.42", + "packets": 70, + "port": 43418 + } + }, + { + "@timestamp": "2019-08-26T19:48:53Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "52.213.180.42", + "pkt_srcaddr": "10.0.0.62", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 18, + "tcp_flags_array": [ + "syn", + "ack" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "52.213.180.42", + "ip": "52.213.180.42", + "port": 43418 + }, + "event": { + "end": "2019-08-26T19:48:53Z", + "outcome": "success", + "start": "2019-08-26T19:47:55Z" + }, + "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 632 12 1566848875 1566848933 ACCEPT 18 OK", + "network": { + "bytes": 632, + "iana_number": "6", + "packets": 12, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.0.62", + "52.213.180.42" + ] + }, + "source": { + "address": "10.0.0.62", + "bytes": 632, + "ip": "10.0.0.62", + "packets": 12, + "port": 5001 + } + }, + { + "@timestamp": "2019-08-26T19:51:53Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "52.213.180.42", + "pkt_srcaddr": "10.0.0.62", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 1, + "tcp_flags_array": [ + "fin" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "52.213.180.42", + "ip": "52.213.180.42", + "port": 43418 + }, + "event": { + "end": "2019-08-26T19:51:53Z", + "outcome": "success", + "start": "2019-08-26T19:48:53Z" + }, + "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 63388 1219 1566848933 1566849113 ACCEPT 1 OK", + "network": { + "bytes": 63388, + "iana_number": "6", + "packets": 1219, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.0.62", + "52.213.180.42" + ] + }, + "source": { + "address": "10.0.0.62", + "bytes": 63388, + "ip": "10.0.0.62", + "packets": 1219, + "port": 5001 + } + }, + { + "@timestamp": "2019-08-26T19:51:53Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "10.0.0.62", + "pkt_srcaddr": "52.213.180.42", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 1, + "tcp_flags_array": [ + "fin" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "10.0.0.62", + "ip": "10.0.0.62", + "port": 5001 + }, + "event": { + "end": "2019-08-26T19:51:53Z", + "outcome": "success", + "start": "2019-08-26T19:48:53Z" + }, + "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 23294588 15774 1566848933 1566849113 ACCEPT 1 OK", + "network": { + "bytes": 23294588, + "iana_number": "6", + "packets": 15774, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "52.213.180.42", + "10.0.0.62" + ] + }, + "source": { + "address": "52.213.180.42", + "bytes": 23294588, + "ip": "52.213.180.42", + "packets": 15774, + "port": 43418 + } + }, + { + "@timestamp": "2019-08-27T19:13:13Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "10.0.0.62", + "pkt_srcaddr": "52.213.180.42", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 3, + "tcp_flags_array": [ + "fin", + "syn" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "10.0.0.62", + "ip": "10.0.0.62", + "port": 5001 + }, + "event": { + "end": "2019-08-27T19:13:13Z", + "outcome": "success", + "start": "2019-08-27T19:12:13Z" + }, + "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43638 5001 52.213.180.42 10.0.0.62 6 1260 17 1566933133 1566933193 ACCEPT 3 OK", + "network": { + "bytes": 1260, + "iana_number": "6", + "packets": 17, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "52.213.180.42", + "10.0.0.62" + ] + }, + "source": { + "address": "52.213.180.42", + "bytes": 1260, + "ip": "52.213.180.42", + "packets": 17, + "port": 43638 + } + }, + { + "@timestamp": "2019-08-27T19:13:13Z", + "aws": { + "vpcflow": { + "interface_id": "eni-1235b8ca123456789", + "log_status": "OK", + "pkt_dstaddr": "52.213.180.42", + "pkt_srcaddr": "10.0.0.62", + "subnet_id": "subnet-aaaaaaaa012345678", + "tcp_flags": 19, + "tcp_flags_array": [ + "fin", + "syn", + "ack" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "52.213.180.42", + "ip": "52.213.180.42", + "port": 43638 + }, + "event": { + "end": "2019-08-27T19:13:13Z", + "outcome": "success", + "start": "2019-08-27T19:12:13Z" + }, + "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43638 10.0.0.62 52.213.180.42 6 967 14 1566933133 1566933193 ACCEPT 19 OK", + "network": { + "bytes": 967, + "iana_number": "6", + "packets": 14, + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.0.0.62", + "52.213.180.42" + ] + }, + "source": { + "address": "10.0.0.62", + "bytes": 967, + "ip": "10.0.0.62", + "packets": 14, + "port": 5001 + } + } +] diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.yml deleted file mode 100644 index 8b1dd4965840..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.yml +++ /dev/null @@ -1,357 +0,0 @@ -- '@timestamp': 2019-08-26T19:48:53Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 10.0.0.62 - pkt_srcaddr: 52.213.180.42 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 2 - tcp_flags_array: - - syn - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 10.0.0.62 - ip: 10.0.0.62 - port: 5001 - event: - end: 2019-08-26T19:48:53Z - outcome: success - start: 2019-08-26T19:47:55Z - message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK - network: - bytes: 568 - iana_number: "6" - packets: 8 - protocol: tcp - type: ipv4 - related: - ip: - - 52.213.180.42 - - 10.0.0.62 - source: - address: 52.213.180.42 - bytes: 568 - ip: 52.213.180.42 - packets: 8 - port: 43416 -- '@timestamp': 2019-08-26T19:48:53Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 52.213.180.42 - pkt_srcaddr: 10.0.0.62 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 18 - tcp_flags_array: - - syn - - ack - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 52.213.180.42 - ip: 52.213.180.42 - port: 43416 - event: - end: 2019-08-26T19:48:53Z - outcome: success - start: 2019-08-26T19:47:55Z - message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43416 10.0.0.62 52.213.180.42 6 376 7 1566848875 1566848933 ACCEPT 18 OK - network: - bytes: 376 - iana_number: "6" - packets: 7 - protocol: tcp - type: ipv4 - related: - ip: - - 10.0.0.62 - - 52.213.180.42 - source: - address: 10.0.0.62 - bytes: 376 - ip: 10.0.0.62 - packets: 7 - port: 5001 -- '@timestamp': 2019-08-26T19:48:53Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 10.0.0.62 - pkt_srcaddr: 52.213.180.42 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 2 - tcp_flags_array: - - syn - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 10.0.0.62 - ip: 10.0.0.62 - port: 5001 - event: - end: 2019-08-26T19:48:53Z - outcome: success - start: 2019-08-26T19:47:55Z - message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 100701 70 1566848875 1566848933 ACCEPT 2 OK - network: - bytes: 100701 - iana_number: "6" - packets: 70 - protocol: tcp - type: ipv4 - related: - ip: - - 52.213.180.42 - - 10.0.0.62 - source: - address: 52.213.180.42 - bytes: 100701 - ip: 52.213.180.42 - packets: 70 - port: 43418 -- '@timestamp': 2019-08-26T19:48:53Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 52.213.180.42 - pkt_srcaddr: 10.0.0.62 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 18 - tcp_flags_array: - - syn - - ack - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 52.213.180.42 - ip: 52.213.180.42 - port: 43418 - event: - end: 2019-08-26T19:48:53Z - outcome: success - start: 2019-08-26T19:47:55Z - message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 632 12 1566848875 1566848933 ACCEPT 18 OK - network: - bytes: 632 - iana_number: "6" - packets: 12 - protocol: tcp - type: ipv4 - related: - ip: - - 10.0.0.62 - - 52.213.180.42 - source: - address: 10.0.0.62 - bytes: 632 - ip: 10.0.0.62 - packets: 12 - port: 5001 -- '@timestamp': 2019-08-26T19:51:53Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 52.213.180.42 - pkt_srcaddr: 10.0.0.62 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 1 - tcp_flags_array: - - fin - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 52.213.180.42 - ip: 52.213.180.42 - port: 43418 - event: - end: 2019-08-26T19:51:53Z - outcome: success - start: 2019-08-26T19:48:53Z - message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 63388 1219 1566848933 1566849113 ACCEPT 1 OK - network: - bytes: 63388 - iana_number: "6" - packets: 1219 - protocol: tcp - type: ipv4 - related: - ip: - - 10.0.0.62 - - 52.213.180.42 - source: - address: 10.0.0.62 - bytes: 63388 - ip: 10.0.0.62 - packets: 1219 - port: 5001 -- '@timestamp': 2019-08-26T19:51:53Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 10.0.0.62 - pkt_srcaddr: 52.213.180.42 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 1 - tcp_flags_array: - - fin - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 10.0.0.62 - ip: 10.0.0.62 - port: 5001 - event: - end: 2019-08-26T19:51:53Z - outcome: success - start: 2019-08-26T19:48:53Z - message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 23294588 15774 1566848933 1566849113 ACCEPT 1 OK - network: - bytes: 23294588 - iana_number: "6" - packets: 15774 - protocol: tcp - type: ipv4 - related: - ip: - - 52.213.180.42 - - 10.0.0.62 - source: - address: 52.213.180.42 - bytes: 23294588 - ip: 52.213.180.42 - packets: 15774 - port: 43418 -- '@timestamp': 2019-08-27T19:13:13Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 10.0.0.62 - pkt_srcaddr: 52.213.180.42 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 3 - tcp_flags_array: - - fin - - syn - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 10.0.0.62 - ip: 10.0.0.62 - port: 5001 - event: - end: 2019-08-27T19:13:13Z - outcome: success - start: 2019-08-27T19:12:13Z - message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43638 5001 52.213.180.42 10.0.0.62 6 1260 17 1566933133 1566933193 ACCEPT 3 OK - network: - bytes: 1260 - iana_number: "6" - packets: 17 - protocol: tcp - type: ipv4 - related: - ip: - - 52.213.180.42 - - 10.0.0.62 - source: - address: 52.213.180.42 - bytes: 1260 - ip: 52.213.180.42 - packets: 17 - port: 43638 -- '@timestamp': 2019-08-27T19:13:13Z - aws: - vpcflow: - interface_id: eni-1235b8ca123456789 - log_status: OK - pkt_dstaddr: 52.213.180.42 - pkt_srcaddr: 10.0.0.62 - subnet_id: subnet-aaaaaaaa012345678 - tcp_flags: 19 - tcp_flags_array: - - fin - - syn - - ack - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 52.213.180.42 - ip: 52.213.180.42 - port: 43638 - event: - end: 2019-08-27T19:13:13Z - outcome: success - start: 2019-08-27T19:12:13Z - message: 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43638 10.0.0.62 52.213.180.42 6 967 14 1566933133 1566933193 ACCEPT 19 OK - network: - bytes: 967 - iana_number: "6" - packets: 14 - protocol: tcp - type: ipv4 - related: - ip: - - 10.0.0.62 - - 52.213.180.42 - source: - address: 10.0.0.62 - bytes: 967 - ip: 10.0.0.62 - packets: 14 - port: 5001 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json new file mode 100644 index 000000000000..1a76557633c3 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json @@ -0,0 +1,208 @@ +[ + { + "aws": { + "vpcflow": { + "interface_id": "eni-33333333333333333", + "log_status": "OK", + "pkt_dstaddr": "10.40.2.236", + "pkt_srcaddr": "10.20.33.164", + "subnet_id": "subnet-22222222bbbbbbbbb", + "tcp_flags": 3, + "tcp_flags_array": [ + "fin", + "syn" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "10.40.2.236", + "ip": "10.40.2.236", + "port": 80 + }, + "event": { + "outcome": "success" + }, + "message": "3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.20.33.164 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK", + "network": { + "iana_number": "6", + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.20.33.164", + "10.40.2.236" + ] + }, + "source": { + "address": "10.20.33.164", + "ip": "10.20.33.164", + "port": 39812 + } + }, + { + "aws": { + "vpcflow": { + "interface_id": "eni-33333333333333333", + "log_status": "OK", + "pkt_dstaddr": "10.20.33.164", + "pkt_srcaddr": "10.40.2.236", + "subnet_id": "subnet-22222222bbbbbbbbb", + "tcp_flags": 19, + "tcp_flags_array": [ + "fin", + "syn", + "ack" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "destination": { + "address": "10.20.33.164", + "ip": "10.20.33.164", + "port": 39812 + }, + "event": { + "outcome": "success" + }, + "message": "3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.40.2.236 10.20.33.164 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK", + "network": { + "iana_number": "6", + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.40.2.236", + "10.20.33.164" + ] + }, + "source": { + "address": "10.40.2.236", + "ip": "10.40.2.236", + "port": 80 + } + }, + { + "aws": { + "vpcflow": { + "interface_id": "eni-11111111111111111", + "log_status": "OK", + "pkt_dstaddr": "10.40.2.236", + "pkt_srcaddr": "10.20.33.164", + "subnet_id": "subnet-11111111aaaaaaaaa", + "tcp_flags": 3, + "tcp_flags_array": [ + "fin", + "syn" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "destination": { + "address": "10.40.2.236", + "ip": "10.40.2.236", + "port": 80 + }, + "event": { + "outcome": "success" + }, + "message": "3 eni-11111111111111111 123456789010 vpc-abcdefab012345678 subnet-11111111aaaaaaaaa - 10.40.1.175 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK", + "network": { + "iana_number": "6", + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.40.1.175", + "10.40.2.236", + "10.20.33.164" + ] + }, + "source": { + "address": "10.40.1.175", + "ip": "10.40.1.175", + "port": 39812 + } + }, + { + "aws": { + "vpcflow": { + "interface_id": "eni-22222222222222222", + "log_status": "OK", + "pkt_dstaddr": "10.20.33.164", + "pkt_srcaddr": "10.40.2.236", + "subnet_id": "subnet-22222222bbbbbbbbb", + "tcp_flags": 19, + "tcp_flags_array": [ + "fin", + "syn", + "ack" + ], + "type": "IPv4", + "version": 3, + "vpc_id": "vpc-abcdefab012345678" + } + }, + "cloud": { + "account": { + "id": "123456789010" + } + }, + "destination": { + "address": "10.40.2.31", + "ip": "10.40.2.31", + "port": 39812 + }, + "event": { + "outcome": "success" + }, + "message": "3 eni-22222222222222222 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb - 10.40.2.236 10.40.2.31 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK", + "network": { + "iana_number": "6", + "protocol": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "10.40.2.236", + "10.40.2.31", + "10.20.33.164" + ] + }, + "source": { + "address": "10.40.2.236", + "ip": "10.40.2.236", + "port": 80 + } + } +] diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.yml deleted file mode 100644 index b604c2438657..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.yml +++ /dev/null @@ -1,152 +0,0 @@ -- aws: - vpcflow: - interface_id: eni-33333333333333333 - log_status: OK - pkt_dstaddr: 10.40.2.236 - pkt_srcaddr: 10.20.33.164 - subnet_id: subnet-22222222bbbbbbbbb - tcp_flags: 3 - tcp_flags_array: - - fin - - syn - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 10.40.2.236 - ip: 10.40.2.236 - port: 80 - event: - outcome: success - message: 3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.20.33.164 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK - network: - iana_number: "6" - protocol: tcp - type: ipv4 - related: - ip: - - 10.20.33.164 - - 10.40.2.236 - source: - address: 10.20.33.164 - ip: 10.20.33.164 - port: 39812 -- aws: - vpcflow: - interface_id: eni-33333333333333333 - log_status: OK - pkt_dstaddr: 10.20.33.164 - pkt_srcaddr: 10.40.2.236 - subnet_id: subnet-22222222bbbbbbbbb - tcp_flags: 19 - tcp_flags_array: - - fin - - syn - - ack - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - instance: - id: i-01234567890123456 - destination: - address: 10.20.33.164 - ip: 10.20.33.164 - port: 39812 - event: - outcome: success - message: 3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.40.2.236 10.20.33.164 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK - network: - iana_number: "6" - protocol: tcp - type: ipv4 - related: - ip: - - 10.40.2.236 - - 10.20.33.164 - source: - address: 10.40.2.236 - ip: 10.40.2.236 - port: 80 -- aws: - vpcflow: - interface_id: eni-11111111111111111 - log_status: OK - pkt_dstaddr: 10.40.2.236 - pkt_srcaddr: 10.20.33.164 - subnet_id: subnet-11111111aaaaaaaaa - tcp_flags: 3 - tcp_flags_array: - - fin - - syn - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - destination: - address: 10.40.2.236 - ip: 10.40.2.236 - port: 80 - event: - outcome: success - message: 3 eni-11111111111111111 123456789010 vpc-abcdefab012345678 subnet-11111111aaaaaaaaa - 10.40.1.175 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK - network: - iana_number: "6" - protocol: tcp - type: ipv4 - related: - ip: - - 10.40.1.175 - - 10.40.2.236 - - 10.20.33.164 - source: - address: 10.40.1.175 - ip: 10.40.1.175 - port: 39812 -- aws: - vpcflow: - interface_id: eni-22222222222222222 - log_status: OK - pkt_dstaddr: 10.20.33.164 - pkt_srcaddr: 10.40.2.236 - subnet_id: subnet-22222222bbbbbbbbb - tcp_flags: 19 - tcp_flags_array: - - fin - - syn - - ack - type: IPv4 - version: 3 - vpc_id: vpc-abcdefab012345678 - cloud: - account: - id: "123456789010" - destination: - address: 10.40.2.31 - ip: 10.40.2.31 - port: 39812 - event: - outcome: success - message: 3 eni-22222222222222222 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb - 10.40.2.236 10.40.2.31 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK - network: - iana_number: "6" - protocol: tcp - type: ipv4 - related: - ip: - - 10.40.2.236 - - 10.40.2.31 - - 10.20.33.164 - source: - address: 10.40.2.236 - ip: 10.40.2.236 - port: 80 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json new file mode 100644 index 000000000000..32e537dba444 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json @@ -0,0 +1,69 @@ +[ + { + "@timestamp": "2021-03-26T03:29:09Z", + "aws": { + "vpcflow": { + "interface_id": "eni-069xxxxxb7a490", + "log_status": "OK", + "pkt_dst_aws_service": "CLOUDFRONT", + "pkt_dstaddr": "10.200.0.80", + "pkt_src_aws_service": "AMAZON", + "pkt_srcaddr": "89.160.20.156", + "sublocation_id": "fake-id", + "sublocation_type": "wavelength", + "subnet_id": "subnet-02d645xxxxxxxdbc0", + "tcp_flags": 1, + "tcp_flags_array": [ + "fin" + ], + "traffic_path": 1, + "type": "IPv4", + "version": 5, + "vpc_id": "vpc-09676f97xxxxxb8a7" + } + }, + "cloud": { + "account": { + "id": "64111117617" + }, + "availability_zone": "use1-az5", + "instance": { + "id": "i-0axxxxxx1ad77" + }, + "region": "us-east-1" + }, + "destination": { + "address": "10.200.0.0", + "ip": "10.200.0.0", + "port": 33004 + }, + "event": { + "end": "2021-03-26T03:29:09Z", + "outcome": "failure", + "start": "2021-03-26T03:28:12Z" + }, + "message": "5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1", + "network": { + "bytes": 1, + "direction": "ingress", + "iana_number": "17", + "packets": 52, + "protocol": "udp", + "type": "ipv4" + }, + "related": { + "ip": [ + "89.160.20.156", + "10.200.0.0", + "10.200.0.80" + ] + }, + "source": { + "address": "89.160.20.156", + "bytes": 1, + "ip": "89.160.20.156", + "packets": 52, + "port": 50041 + } + } +] diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.yml deleted file mode 100644 index be8719d56e9e..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.yml +++ /dev/null @@ -1,53 +0,0 @@ -- '@timestamp': 2021-03-26T03:29:09Z - aws: - vpcflow: - interface_id: eni-069xxxxxb7a490 - log_status: OK - pkt_dst_aws_service: CLOUDFRONT - pkt_dstaddr: 10.200.0.80 - pkt_src_aws_service: AMAZON - pkt_srcaddr: 89.160.20.156 - sublocation_id: fake-id - sublocation_type: wavelength - subnet_id: subnet-02d645xxxxxxxdbc0 - tcp_flags: 1 - tcp_flags_array: - - fin - traffic_path: 1 - type: IPv4 - version: 5 - vpc_id: vpc-09676f97xxxxxb8a7 - cloud: - account: - id: "64111117617" - availability_zone: use1-az5 - instance: - id: i-0axxxxxx1ad77 - region: us-east-1 - destination: - address: 10.200.0.0 - ip: 10.200.0.0 - port: 33004 - event: - end: 2021-03-26T03:29:09Z - outcome: failure - start: 2021-03-26T03:28:12Z - message: 5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1 - network: - bytes: 1 - direction: ingress - iana_number: "17" - packets: 52 - protocol: udp - type: ipv4 - related: - ip: - - 89.160.20.156 - - 10.200.0.0 - - 10.200.0.80 - source: - address: 89.160.20.156 - bytes: 1 - ip: 89.160.20.156 - packets: 52 - port: 50041 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json new file mode 100644 index 000000000000..da73ac7471d8 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json @@ -0,0 +1,84 @@ +[ + { + "@timestamp": "2021-03-26T03:29:09Z", + "aws": { + "vpcflow": { + "account_id": "64111117617", + "action": "REJECT", + "az_id": "use1-az5", + "bytes": 1, + "dstaddr": "10.200.0.0", + "dstport": 33004, + "end": "2021-03-26T03:29:09Z", + "flow_direction": "ingress", + "instance_id": "i-0axxxxxx1ad77", + "interface_id": "eni-069xxxxxb7a490", + "log_status": "OK", + "packets": 52, + "pkt_dst_aws_service": "CLOUDFRONT", + "pkt_dstaddr": "10.200.0.80", + "pkt_src_aws_service": "AMAZON", + "pkt_srcaddr": "89.160.20.156", + "protocol": 17, + "region": "us-east-1", + "srcaddr": "89.160.20.156", + "srcport": 50041, + "start": "2021-03-26T03:28:12Z", + "sublocation_id": "fake-id", + "sublocation_type": "wavelength", + "subnet_id": "subnet-02d645xxxxxxxdbc0", + "tcp_flags": 1, + "tcp_flags_array": [ + "fin" + ], + "traffic_path": 1, + "type": "IPv4", + "version": 5, + "vpc_id": "vpc-09676f97xxxxxb8a7" + } + }, + "cloud": { + "account": { + "id": "64111117617" + }, + "availability_zone": "use1-az5", + "instance": { + "id": "i-0axxxxxx1ad77" + }, + "region": "us-east-1" + }, + "destination": { + "address": "10.200.0.0", + "ip": "10.200.0.0", + "port": 33004 + }, + "event": { + "end": "2021-03-26T03:29:09Z", + "outcome": "failure", + "start": "2021-03-26T03:28:12Z" + }, + "message": "5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1", + "network": { + "bytes": 1, + "direction": "ingress", + "iana_number": "17", + "packets": 52, + "protocol": "udp", + "type": "ipv4" + }, + "related": { + "ip": [ + "89.160.20.156", + "10.200.0.0", + "10.200.0.80" + ] + }, + "source": { + "address": "89.160.20.156", + "bytes": 1, + "ip": "89.160.20.156", + "packets": 52, + "port": 50041 + } + } +] diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.yml deleted file mode 100644 index eca253eb8a77..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.yml +++ /dev/null @@ -1,68 +0,0 @@ -- '@timestamp': 2021-03-26T03:29:09Z - aws: - vpcflow: - account_id: "64111117617" - action: REJECT - az_id: use1-az5 - bytes: 1 - dstaddr: 10.200.0.0 - dstport: 33004 - end: 2021-03-26T03:29:09Z - flow_direction: ingress - instance_id: i-0axxxxxx1ad77 - interface_id: eni-069xxxxxb7a490 - log_status: OK - packets: 52 - pkt_dst_aws_service: CLOUDFRONT - pkt_dstaddr: 10.200.0.80 - pkt_src_aws_service: AMAZON - pkt_srcaddr: 89.160.20.156 - protocol: 17 - region: us-east-1 - srcaddr: 89.160.20.156 - srcport: 50041 - start: 2021-03-26T03:28:12Z - sublocation_id: fake-id - sublocation_type: wavelength - subnet_id: subnet-02d645xxxxxxxdbc0 - tcp_flags: 1 - tcp_flags_array: - - fin - traffic_path: 1 - type: IPv4 - version: 5 - vpc_id: vpc-09676f97xxxxxb8a7 - cloud: - account: - id: "64111117617" - availability_zone: use1-az5 - instance: - id: i-0axxxxxx1ad77 - region: us-east-1 - destination: - address: 10.200.0.0 - ip: 10.200.0.0 - port: 33004 - event: - end: 2021-03-26T03:29:09Z - outcome: failure - start: 2021-03-26T03:28:12Z - message: 5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1 - network: - bytes: 1 - direction: ingress - iana_number: "17" - packets: 52 - protocol: udp - type: ipv4 - related: - ip: - - 89.160.20.156 - - 10.200.0.0 - - 10.200.0.80 - source: - address: 89.160.20.156 - bytes: 1 - ip: 89.160.20.156 - packets: 52 - port: 50041 diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.json new file mode 100644 index 000000000000..3493c7e256f6 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.json @@ -0,0 +1,41 @@ +[ + { + "aws": { + "vpcflow": { + "account_id": "64111117617", + "action": "REJECT", + "az_id": "use1-az5", + "bytes": 1, + "dstaddr": "10.200.0.0", + "dstport": 33004, + "end": "2021-03-26T03:29:09Z", + "flow_direction": "ingress", + "instance_id": "i-0axxxxxx1ad77", + "interface_id": "eni-069xxxxxb7a490", + "log_status": "OK", + "packets": 52, + "pkt_dst_aws_service": "CLOUDFRONT", + "pkt_dstaddr": "10.200.0.80", + "pkt_src_aws_service": "AMAZON", + "pkt_srcaddr": "89.160.20.156", + "protocol": 17, + "region": "us-east-1", + "srcaddr": "89.160.20.156", + "srcport": 50041, + "start": "2021-03-26T03:28:12Z", + "sublocation_id": "fake-id", + "sublocation_type": "wavelength", + "subnet_id": "subnet-02d645xxxxxxxdbc0", + "tcp_flags": 1, + "tcp_flags_array": [ + "fin" + ], + "traffic_path": 1, + "type": "IPv4", + "version": 5, + "vpc_id": "vpc-09676f97xxxxxb8a7" + } + }, + "message": "5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1" + } +] diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.yml b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.yml deleted file mode 100644 index 240d46ddaa93..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-original.golden.yml +++ /dev/null @@ -1,34 +0,0 @@ -- aws: - vpcflow: - account_id: "64111117617" - action: REJECT - az_id: use1-az5 - bytes: 1 - dstaddr: 10.200.0.0 - dstport: 33004 - end: 2021-03-26T03:29:09Z - flow_direction: ingress - instance_id: i-0axxxxxx1ad77 - interface_id: eni-069xxxxxb7a490 - log_status: OK - packets: 52 - pkt_dst_aws_service: CLOUDFRONT - pkt_dstaddr: 10.200.0.80 - pkt_src_aws_service: AMAZON - pkt_srcaddr: 89.160.20.156 - protocol: 17 - region: us-east-1 - srcaddr: 89.160.20.156 - srcport: 50041 - start: 2021-03-26T03:28:12Z - sublocation_id: fake-id - sublocation_type: wavelength - subnet_id: subnet-02d645xxxxxxxdbc0 - tcp_flags: 1 - tcp_flags_array: - - fin - traffic_path: 1 - type: IPv4 - version: 5 - vpc_id: vpc-09676f97xxxxxb8a7 - message: 5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1 From 66851a5603dffcf7af24d21d467f8fa71f482524 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Nov 2022 17:13:32 -0500 Subject: [PATCH 11/22] Refactor benchmarks --- .../parse_aws_vpc_flow_log_test.go | 69 ++++++++----------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index 07717b67e83a..e15017734678 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -245,49 +245,40 @@ func readGolden(t *testing.T, path string) string { } func BenchmarkProcessorRun(b *testing.B) { - b.Run("v5-mode-original", func(b *testing.B) { - c := defaultConfig() - c.Format = formatV5 - c.Mode = originalMode + benchmarks := []struct { + name string + mode mode + format string + message string + }{ + {"original-mode-v5-message", originalMode, formatV5, formatV5Sample}, + {"ecs-mode-v5-message", ecsMode, formatV5, formatV5Sample}, + {"ecs_and_original-mode-v5-message", ecsAndOriginalMode, formatV5, formatV5Sample}, + } - p, err := newParseAWSVPCFlowLog(c) - require.NoError(b, err) + for _, benchmark := range benchmarks { + benchmark := benchmark + b.Run(benchmark.name, func(b *testing.B) { + c := defaultConfig() + c.Format = benchmark.format + c.Mode = benchmark.mode - evt := beat.Event{ - Timestamp: time.Now().UTC(), - Fields: map[string]interface{}{ - "message": formatV5Sample, - }, - } + p, err := newParseAWSVPCFlowLog(c) + require.NoError(b, err) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = p.Run(&evt); err != nil { - b.Fatal(err) + evt := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: map[string]interface{}{ + "message": benchmark.message, + }, } - } - }) - b.Run("v5-mode-ecs_and_original", func(b *testing.B) { - c := defaultConfig() - c.Format = formatV5 - c.Mode = ecsAndOriginalMode - - p, err := newParseAWSVPCFlowLog(c) - require.NoError(b, err) - - evt := beat.Event{ - Timestamp: time.Now().UTC(), - Fields: map[string]interface{}{ - "message": formatV5Sample, - }, - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = p.Run(&evt); err != nil { - b.Fatal(err) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = p.Run(&evt); err != nil { + b.Fatal(err) + } } - } - }) + }) + } } From fde32b49e0def2c5953091cb84d43298301f76b8 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Nov 2022 22:20:08 -0500 Subject: [PATCH 12/22] Clone strings.Fields and FieldsFunc --- .../aws_vpcflow/internal/strings/strings.go | 116 ++++++++++++++++++ .../internal/strings/strings_test.go | 77 ++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go create mode 100644 x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go diff --git a/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go new file mode 100644 index 000000000000..7c1b77d473b2 --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go @@ -0,0 +1,116 @@ +package strings + +import ( + "unicode" + "unicode/utf8" +) + +var errTooManySubstrings = errors.New("len of dst slice is less than the number of substrings") + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} + +// Fields splits the string s around each instance of one or more consecutive white space +// characters, as defined by unicode.IsSpace, returning a slice of substrings of s or an +// empty slice if s contains only white space. +func Fields(s string) []string { + // First count the fields. + // This is an exact count if s is ASCII, otherwise it is an approximation. + n := 0 + wasSpace := 1 + // setBits is used to track which bits are set in the bytes of s. + setBits := uint8(0) + for i := 0; i < len(s); i++ { + r := s[i] + setBits |= r + isSpace := int(asciiSpace[r]) + n += wasSpace & ^isSpace + wasSpace = isSpace + } + + if setBits >= utf8.RuneSelf { + // Some runes in the input string are not ASCII. + return FieldsFunc(s, unicode.IsSpace) + } + // ASCII fast path + a := make([]string, n) + na := 0 + fieldStart := 0 + i := 0 + // Skip spaces in the front of the input. + for i < len(s) && asciiSpace[s[i]] != 0 { + i++ + } + fieldStart = i + for i < len(s) { + if asciiSpace[s[i]] == 0 { + i++ + continue + } + a[na] = s[fieldStart:i] + na++ + i++ + // Skip spaces in between fields. + for i < len(s) && asciiSpace[s[i]] != 0 { + i++ + } + fieldStart = i + } + if fieldStart < len(s) { // Last field might end at EOF. + a[na] = s[fieldStart:] + } + return a +} + +// FieldsFunc splits the string s at each run of Unicode code points c satisfying f(c) +// and returns an array of slices of s. If all code points in s satisfy f(c) or the +// string is empty, an empty slice is returned. +// +// FieldsFunc makes no guarantees about the order in which it calls f(c) +// and assumes that f always returns the same value for a given c. +func FieldsFunc(s string, f func(rune) bool) []string { + // A span is used to record a slice of s of the form s[start:end]. + // The start index is inclusive and the end index is exclusive. + type span struct { + start int + end int + } + spans := make([]span, 0, 32) + + // Find the field start and end indices. + // Doing this in a separate pass (rather than slicing the string s + // and collecting the result substrings right away) is significantly + // more efficient, possibly due to cache effects. + start := -1 // valid span start if >= 0 + for end, rune := range s { + if f(rune) { + if start >= 0 { + spans = append(spans, span{start, end}) + // Set start to a negative value. + // Note: using -1 here consistently and reproducibly + // slows down this code by a several percent on amd64. + start = ^start + } + } else { + if start < 0 { + start = end + } + } + } + + // Last field might end at EOF. + if start >= 0 { + spans = append(spans, span{start, len(s)}) + } + + // Create strings from recorded field indices. + a := make([]string, len(spans)) + for i, span := range spans { + a[i] = s[span.start:span.end] + } + + return a +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go new file mode 100644 index 000000000000..18dea5d73dab --- /dev/null +++ b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go @@ -0,0 +1,77 @@ +package strings + +import ( + "testing" + "unicode" +) + +func eq(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +var faces = "☺☻☹" + +type FieldsTest struct { + s string + a []string +} + +var fieldstests = []FieldsTest{ + {"", []string{}}, + {" ", []string{}}, + {" \t ", []string{}}, + {"\u2000", []string{}}, + {" abc ", []string{"abc"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1\t\t2\t\t3\t4", []string{"1", "2", "3", "4"}}, + {"1\u20002\u20013\u20024", []string{"1", "2", "3", "4"}}, + {"\u2000\u2001\u2002", []string{}}, + {"\n™\t™\n", []string{"™", "™"}}, + {"\n\u20001™2\u2000 \u2001 ™", []string{"1™2", "™"}}, + {"\n1\uFFFD \uFFFD2\u20003\uFFFD4", []string{"1\uFFFD", "\uFFFD2", "3\uFFFD4"}}, + {"1\xFF\u2000\xFF2\xFF \xFF", []string{"1\xFF", "\xFF2\xFF", "\xFF"}}, + {faces, []string{faces}}, +} + +func TestFields(t *testing.T) { + for _, tt := range fieldstests { + a := Fields(tt.s) + if !eq(a, tt.a) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a) + continue + } + } +} + +var FieldsFuncTests = []FieldsTest{ + {"", []string{}}, + {"XX", []string{}}, + {"XXhiXXX", []string{"hi"}}, + {"aXXbXXXcX", []string{"a", "b", "c"}}, +} + +func TestFieldsFunc(t *testing.T) { + for _, tt := range fieldstests { + a := FieldsFunc(tt.s, unicode.IsSpace) + if !eq(a, tt.a) { + t.Errorf("FieldsFunc(%q, unicode.IsSpace) = %v; want %v", tt.s, a, tt.a) + continue + } + } + pred := func(c rune) bool { return c == 'X' } + for _, tt := range FieldsFuncTests { + a := FieldsFunc(tt.s, pred) + if !eq(a, tt.a) { + t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) + } + } +} From 6a771b806846fe39a892935a78d4533dba1e2a6d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Nov 2022 06:35:36 -0500 Subject: [PATCH 13/22] Hack strings.Fields to accept dst []string slice --- .../aws_vpcflow/internal/strings/strings.go | 45 ++++++--- .../internal/strings/strings_test.go | 96 +++++++++++++++++-- 2 files changed, 118 insertions(+), 23 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go index 7c1b77d473b2..1aede98cd492 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go +++ b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go @@ -1,6 +1,11 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package strings import ( + "errors" "unicode" "unicode/utf8" ) @@ -16,7 +21,10 @@ var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} // Fields splits the string s around each instance of one or more consecutive white space // characters, as defined by unicode.IsSpace, returning a slice of substrings of s or an // empty slice if s contains only white space. -func Fields(s string) []string { +// +// It writes at most len(dst) substrings from s into dst. It returns the number +// of substrings copied and an error if there were more than len(dst) substrings. +func Fields(dst []string, s string) (int, error) { // First count the fields. // This is an exact count if s is ASCII, otherwise it is an approximation. n := 0 @@ -33,10 +41,9 @@ func Fields(s string) []string { if setBits >= utf8.RuneSelf { // Some runes in the input string are not ASCII. - return FieldsFunc(s, unicode.IsSpace) + return fieldsFunc(dst, s, unicode.IsSpace) } // ASCII fast path - a := make([]string, n) na := 0 fieldStart := 0 i := 0 @@ -50,7 +57,10 @@ func Fields(s string) []string { i++ continue } - a[na] = s[fieldStart:i] + if na >= len(dst) { + return na, errTooManySubstrings + } + dst[na] = s[fieldStart:i] na++ i++ // Skip spaces in between fields. @@ -60,25 +70,29 @@ func Fields(s string) []string { fieldStart = i } if fieldStart < len(s) { // Last field might end at EOF. - a[na] = s[fieldStart:] + if na >= len(dst) { + return na, errTooManySubstrings + } + dst[na] = s[fieldStart:] + na++ } - return a + return na, nil } -// FieldsFunc splits the string s at each run of Unicode code points c satisfying f(c) +// fieldsFunc splits the string s at each run of Unicode code points c satisfying f(c) // and returns an array of slices of s. If all code points in s satisfy f(c) or the // string is empty, an empty slice is returned. // // FieldsFunc makes no guarantees about the order in which it calls f(c) // and assumes that f always returns the same value for a given c. -func FieldsFunc(s string, f func(rune) bool) []string { +func fieldsFunc(dst []string, s string, f func(rune) bool) (int, error) { // A span is used to record a slice of s of the form s[start:end]. // The start index is inclusive and the end index is exclusive. type span struct { start int end int } - spans := make([]span, 0, 32) + spans := make([]span, 0, len(dst)) // Find the field start and end indices. // Doing this in a separate pass (rather than slicing the string s @@ -88,7 +102,11 @@ func FieldsFunc(s string, f func(rune) bool) []string { for end, rune := range s { if f(rune) { if start >= 0 { - spans = append(spans, span{start, end}) + if len(spans) < len(dst) { + spans = append(spans, span{start, end}) + } else { + break + } // Set start to a negative value. // Note: using -1 here consistently and reproducibly // slows down this code by a several percent on amd64. @@ -102,15 +120,14 @@ func FieldsFunc(s string, f func(rune) bool) []string { } // Last field might end at EOF. - if start >= 0 { + if start >= 0 && len(spans) < len(dst) { spans = append(spans, span{start, len(s)}) } // Create strings from recorded field indices. - a := make([]string, len(spans)) for i, span := range spans { - a[i] = s[span.start:span.end] + dst[i] = s[span.start:span.end] } - return a + return len(spans), nil } diff --git a/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go index 18dea5d73dab..aa9e7b25f3cf 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package strings import ( @@ -43,10 +47,36 @@ var fieldstests = []FieldsTest{ } func TestFields(t *testing.T) { + var dst [4]string for _, tt := range fieldstests { - a := Fields(tt.s) - if !eq(a, tt.a) { - t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a) + n, err := Fields(dst[:], tt.s) + if err != nil { + t.Fatal(err) + } + if !eq(dst[:n], tt.a) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, dst[:n], tt.a) + continue + } + if len(tt.a) != n { + t.Errorf("Return count n = %d; want %d", n, len(tt.a)) + } + } + + // Smaller + var smallDst [2]string + for _, tt := range fieldstests { + n, err := Fields(smallDst[:], tt.s) + if err == errTooManySubstrings { + if len(tt.a) > len(smallDst) { + continue + } + } + if err != nil { + t.Fatal(err) + } + + if !eq(smallDst[:n], tt.a[:n]) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, smallDst[:n], tt.a) continue } } @@ -60,18 +90,66 @@ var FieldsFuncTests = []FieldsTest{ } func TestFieldsFunc(t *testing.T) { + var dst [4]string for _, tt := range fieldstests { - a := FieldsFunc(tt.s, unicode.IsSpace) - if !eq(a, tt.a) { - t.Errorf("FieldsFunc(%q, unicode.IsSpace) = %v; want %v", tt.s, a, tt.a) + n, err := fieldsFunc(dst[:], tt.s, unicode.IsSpace) + if err != nil { + t.Fatal(err) + } + if !eq(dst[:n], tt.a) { + t.Errorf("FieldsFunc(%q, unicode.IsSpace) = %v; want %v", tt.s, dst, tt.a) continue } + if len(tt.a) != n { + t.Errorf("Return count n = %d; want %d", n, len(tt.a)) + } } pred := func(c rune) bool { return c == 'X' } for _, tt := range FieldsFuncTests { - a := FieldsFunc(tt.s, pred) - if !eq(a, tt.a) { - t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) + n, err := fieldsFunc(dst[:], tt.s, pred) + if err != nil { + t.Fatal(err) + } + if !eq(dst[:n], tt.a) { + t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, dst[:n], tt.a) + } + if len(tt.a) != n { + t.Errorf("Return count n = %d; want %d", n, len(tt.a)) + } + } + + // Smaller + var smallDst [2]string + for _, tt := range fieldstests { + n, err := Fields(smallDst[:], tt.s) + if err == errTooManySubstrings { + if len(tt.a) > len(smallDst) { + continue + } + } + if err != nil { + t.Fatal(err) + } + + if !eq(smallDst[:n], tt.a[:n]) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, smallDst[:n], tt.a) + continue + } + } + for _, tt := range FieldsFuncTests { + n, err := fieldsFunc(smallDst[:], tt.s, pred) + if err == errTooManySubstrings { + if len(tt.a) > len(smallDst) { + continue + } + } + if err != nil { + t.Fatal(err) + } + + if !eq(smallDst[:n], tt.a[:n]) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, smallDst[:n], tt.a) + continue } } } From bd221082b80fdf88dc1516682b3d845d252b99c0 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Nov 2022 17:17:29 -0500 Subject: [PATCH 14/22] Replace word iterator with strings.Fields --- .../processors/aws_vpcflow/mapping.go | 6 +- .../aws_vpcflow/parse_aws_vpc_flow_log.go | 38 +++++---- .../processors/aws_vpcflow/word_iterator.go | 78 ------------------- .../aws_vpcflow/word_iterator_test.go | 39 ---------- 4 files changed, 27 insertions(+), 134 deletions(-) delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/word_iterator.go delete mode 100644 x-pack/filebeat/processors/aws_vpcflow/word_iterator_test.go diff --git a/x-pack/filebeat/processors/aws_vpcflow/mapping.go b/x-pack/filebeat/processors/aws_vpcflow/mapping.go index a3d69f4f467a..08670529ae52 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/mapping.go +++ b/x-pack/filebeat/processors/aws_vpcflow/mapping.go @@ -29,13 +29,13 @@ type ecsFieldMapping struct { var nameToFieldMap map[string]vpcFlowField func init() { - nameToFieldMap = make(map[string]vpcFlowField, len(fields)) - for _, field := range fields { + nameToFieldMap = make(map[string]vpcFlowField, len(vpcFlowFields)) + for _, field := range vpcFlowFields { nameToFieldMap[field.Name] = field } } -var fields = []vpcFlowField{ +var vpcFlowFields = [...]vpcFlowField{ { Name: "version", Type: integerType, diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go index 32f6cc7ab95f..3e4adaa787e9 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" + "github.com/elastic/beats/v7/x-pack/filebeat/processors/aws_vpcflow/internal/strings" ) const ( @@ -35,9 +36,9 @@ var ( type processor struct { config - fields []vpcFlowField - log *logp.Logger - expectedIPCount int + fields []vpcFlowField + log *logp.Logger + expectedIPCount int } // New constructs a new processor built from ucfg config. @@ -68,7 +69,12 @@ func newParseAWSVPCFlowLog(c config) (*processor, error) { } } - return &processor{config: c, fields: fields, expectedIPCount: ipCount, log: log}, nil + return &processor{ + config: c, + fields: fields, + expectedIPCount: ipCount, + log: log, + }, nil } func (p *processor) String() string { @@ -97,30 +103,34 @@ func (p *processor) run(event *beat.Event) error { return errValueNotString } - itr := newWordIterator(strValue) - if itr.Count() != len(p.fields) { + // Split the string at whitespace without allocating. + var dst [len(vpcFlowFields)]string + n, err := strings.Fields(dst[:], strValue) + if err != nil { return errInvalidFormat } + substrings := dst[:n] var relatedIPs []string if p.Mode > originalMode { + // Allocate space for the expected number of IPs assuming all are unique. relatedIPs = make([]string, 0, p.expectedIPCount) } originalFields := make(mapstr.M, len(p.fields)) - for itr.Next() { - // Read one word. - value := itr.Word() - if value == "-" { + // Iterate over the substrings in the source string and apply type + // conversion and then ECS mappings. + for i, word := range substrings { + if word == "-" { continue } - field := p.fields[itr.WordIndex()] + field := p.fields[i] - // Convert the string the expected type. - v, err := toType(value, field.Type) + // Convert the string to the expected type. + v, err := toType(word, field.Type) if err != nil { - return fmt.Errorf("failed to parse <%v>: %w", field.Name, err) + return fmt.Errorf("failed to parse %q: %w", field.Name, err) } // Add to the 'original' fields when we are in original mode diff --git a/x-pack/filebeat/processors/aws_vpcflow/word_iterator.go b/x-pack/filebeat/processors/aws_vpcflow/word_iterator.go deleted file mode 100644 index 0958d6fb5eb2..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/word_iterator.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package aws_vpcflow - -// NOTE: This is a minor optimization to avoid one allocation -// from strings.Fields during the run() method. In the benchmarks -// it saves 480 bytes and one allocation. That equates to an increase -// of +628 EPS on an M1 Max, but relatively minor ns/op improvement -// of +0.21%. In other words, it might not be worth it. - -// wordIterator iterates over space separated words in an ASCII string. -type wordIterator struct { - source string - currentWord string - currentPosition int - wordIndex int -} - -func newWordIterator(s string) *wordIterator { - return &wordIterator{source: s, wordIndex: -1} -} - -func (itr *wordIterator) Next() bool { - // ASCII fast path - s := itr.source[itr.currentPosition:] - fieldStart := 0 - i := 0 - // Skip spaces in the front of the input. - for i < len(s) && s[i] == ' ' { - i++ - } - fieldStart = i - - for i < len(s) { - if s[i] != ' ' { - i++ - continue - } - itr.currentWord = s[fieldStart:i] - itr.currentPosition += i - itr.wordIndex++ - return true - } - if fieldStart < len(s) { // Last field might end at EOF. - itr.currentWord = s[fieldStart:] - itr.currentPosition += len(s) - itr.wordIndex++ - return true - } - itr.currentWord = "" - itr.wordIndex = -1 - return false -} - -func (itr *wordIterator) Word() string { - return itr.currentWord -} - -func (itr *wordIterator) WordIndex() int { - return itr.wordIndex -} - -func (itr *wordIterator) Count() int { - n := 0 - wasSpace := true - - for i := 0; i < len(itr.source); i++ { - r := itr.source[i] - isSpace := r == ' ' - if wasSpace && !isSpace { - n++ - } - wasSpace = isSpace - } - return n -} diff --git a/x-pack/filebeat/processors/aws_vpcflow/word_iterator_test.go b/x-pack/filebeat/processors/aws_vpcflow/word_iterator_test.go deleted file mode 100644 index d1d4c9fb0502..000000000000 --- a/x-pack/filebeat/processors/aws_vpcflow/word_iterator_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package aws_vpcflow - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestWordIterator(t *testing.T) { - testCases := []struct { - sentence string - words []string - }{ - {"", nil}, - {" ", nil}, - {" ", nil}, - {"word", []string{"word"}}, - {" word", []string{"word"}}, - {"word ", []string{"word"}}, - {" word ", []string{"word"}}, - {"foo bar baz", []string{"foo", "bar", "baz"}}, - } - - for _, tc := range testCases { - itr := wordIterator{source: tc.sentence} - - var collectedWords []string - for itr.Next() { - collectedWords = append(collectedWords, itr.Word()) - } - - assert.Equal(t, tc.words, collectedWords) - assert.Equal(t, len(tc.words), itr.Count()) - } -} From 1d35407cb37ba1c307048bdc5db63aee5e65a7c1 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Nov 2022 17:20:21 -0500 Subject: [PATCH 15/22] Allocate map based on expected field count --- .../aws_vpcflow/parse_aws_vpc_flow_log.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go index 3e4adaa787e9..547c5be042a3 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go @@ -38,6 +38,7 @@ type processor struct { config fields []vpcFlowField log *logp.Logger + originalFieldCount int expectedIPCount int } @@ -69,9 +70,21 @@ func newParseAWSVPCFlowLog(c config) (*processor, error) { } } + originalFieldCount := len(fields) + if c.Mode == ecsMode { + for _, f := range fields { + // If an ECS mapping exists then ECS mode will not include the + // original field. + if len(f.ECSMappings) > 0 { + originalFieldCount-- + } + } + } + return &processor{ config: c, fields: fields, + originalFieldCount: originalFieldCount, expectedIPCount: ipCount, log: log, }, nil @@ -117,7 +130,7 @@ func (p *processor) run(event *beat.Event) error { relatedIPs = make([]string, 0, p.expectedIPCount) } - originalFields := make(mapstr.M, len(p.fields)) + originalFields := make(mapstr.M, p.originalFieldCount) // Iterate over the substrings in the source string and apply type // conversion and then ECS mappings. From f91c155b727411ccb2e23f436a15bffdccfe83c0 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Nov 2022 21:45:36 -0500 Subject: [PATCH 16/22] Accept more than one format string --- .../filebeat/processors/aws_vpcflow/config.go | 58 ++++++++--- .../processors/aws_vpcflow/config_test.go | 28 +++++- .../docs/parse_aws_vpc_flow_log.asciidoc | 2 +- .../aws_vpcflow/parse_aws_vpc_flow_log.go | 99 +++++++++++++------ .../parse_aws_vpc_flow_log_test.go | 17 ++-- 5 files changed, 154 insertions(+), 50 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/config.go b/x-pack/filebeat/processors/aws_vpcflow/config.go index 6031eea46bfc..39616d49a6d1 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/config.go +++ b/x-pack/filebeat/processors/aws_vpcflow/config.go @@ -55,20 +55,32 @@ func (m *mode) String() string { // config contains the configuration options for the processor. type config struct { - Format string `config:"format" validate:"required"` // VPC flow log format. - Mode mode `config:"mode"` // Mode controls what fields are generated. - Field string `config:"field"` // Source field containing the VPC flow log message. - TargetField string `config:"target_field"` // Target field for the VPC flow log object. This applies only to the original VPC flow log fields. ECS fields are written to the standard location. - IgnoreMissing bool `config:"ignore_missing"` // Ignore missing source field. - IgnoreFailure bool `config:"ignore_failure"` // Ignore failures while parsing and transforming the flow log message. - ID string `config:"id"` // Instance ID for debugging purposes. + Format formats `config:"format" validate:"required"` // VPC flow log format. In config, it can accept a string or list of strings. Each format must have a unique number of fields to enable matching it to a flow log message. + Mode mode `config:"mode"` // Mode controls what fields are generated. + Field string `config:"field"` // Source field containing the VPC flow log message. + TargetField string `config:"target_field"` // Target field for the VPC flow log object. This applies only to the original VPC flow log fields. ECS fields are written to the standard location. + IgnoreMissing bool `config:"ignore_missing"` // Ignore missing source field. + IgnoreFailure bool `config:"ignore_failure"` // Ignore failures while parsing and transforming the flow log message. + ID string `config:"id"` // Instance ID for debugging purposes. } -// Validate validates the config settings. It returns an error if the format -// string is invalid. +// Validate validates the format strings. Each format must have a unique number +// of fields. func (c *config) Validate() error { - _, err := parseFormat(c.Format) - return err + counts := map[int]struct{}{} + for _, format := range c.Format { + fields, err := parseFormat(format) + if err != nil { + return err + } + + _, found := counts[len(fields)] + if found { + return fmt.Errorf("each format must have a unique number of fields") + } + counts[len(fields)] = struct{}{} + } + return nil } func defaultConfig() config { @@ -102,3 +114,27 @@ func parseFormat(format string) ([]vpcFlowField, error) { return fields, nil } + +type formats []string + +func (f *formats) Unpack(value interface{}) error { + switch v := value.(type) { + case string: + *f = []string{v} + case []string: + *f = v + case []interface{}: + list := make([]string, 0, len(v)) + for _, ifc := range v { + s, ok := ifc.(string) + if !ok { + return fmt.Errorf("format values must be strings, got %T", ifc) + } + list = append(list, s) + } + *f = list + default: + return fmt.Errorf("format must be a string or list of strings, got %T", v) + } + return nil +} diff --git a/x-pack/filebeat/processors/aws_vpcflow/config_test.go b/x-pack/filebeat/processors/aws_vpcflow/config_test.go index 5a8a99c7fe62..83c2db01e2d4 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/config_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/config_test.go @@ -53,6 +53,27 @@ format: version srcaddr dstaddr srcport dstport protocol start end type packets --- mode: invalid format: version +`, + }, + { + error: false, + yamlConfig: ` +--- +mode: ecs +format: + - version srcaddr dstaddr + - version srcaddr dstaddr srcport dstport protocol +`, + }, + { + // Each format must have a unique token count. + error: true, + yamlConfig: ` +--- +mode: ecs +format: + - version srcaddr dstaddr + - srcport dstport protocol `, }, } @@ -65,11 +86,16 @@ format: version c := defaultConfig() err := rawConfig.Unpack(&c) if tc.error { + require.Error(t, err, "config: %v", tc.yamlConfig) t.Log("Error:", err) - require.Error(t, err) return } require.NoError(t, err) + + // Make sure valid configs produce processors. + p, err := New(rawConfig) + require.NoError(t, err) + require.NotNil(t, p) }) } } diff --git a/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc b/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc index 6df0f25044fd..cacedfa21b81 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc +++ b/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc @@ -27,7 +27,7 @@ The `parse_aws_vpc_flow_log` processor has the following configuration settings. | Name | Required | Default | Description | | `field` | no | `message` | Source field containing the VPC flow log message. | | `target_field` | no | `aws.vpcflow` | Target field for the VPC flow log object. This applies only to the original VPC flow log fields. ECS fields are written to the standard location. | -| `format` | yes | | VPC flow log format. This supports VPC flow log fields from versions 2 through 5. | +| `format` | yes | | VPC flow log format. This supports VPC flow log fields from versions 2 through 5. It will accept a string or a list of strings. Each format must have a unique number of fields to enable matching it to a flow log message.| | `mode` | no | `ecs` | Mode controls what fields are generated. The available options are `original`, `ecs`, and `ecs_and_original`. `original` generates the fields specified in the format string. `ecs` maps the original fields to ECS and removes the original fields that are mapped to ECS. `ecs_and_original` maps the original fields to ECS and retains all the original fields. | | `ignore_missing` | no | false | Ignore missing source field. | | `ignore_failure` | no | false | Ignore failures while parsing and transforming the flow log message. | diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go index 547c5be042a3..ca495ee780e3 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go @@ -36,10 +36,7 @@ var ( type processor struct { config - fields []vpcFlowField - log *logp.Logger - originalFieldCount int - expectedIPCount int + formats []*formatProcessor } // New constructs a new processor built from ucfg config. @@ -58,35 +55,23 @@ func newParseAWSVPCFlowLog(c config) (*processor, error) { log = log.With("instance_id", c.ID) } - fields, err := parseFormat(c.Format) - if err != nil { - return nil, fmt.Errorf("failed to parse vpc flow log format: %w", err) - } - - var ipCount int - for _, f := range fields { - if f.Type == ipType { - ipCount++ - } + // Validate configs that did not pass through go-ucfg. + if err := c.Validate(); err != nil { + return nil, err } - originalFieldCount := len(fields) - if c.Mode == ecsMode { - for _, f := range fields { - // If an ECS mapping exists then ECS mode will not include the - // original field. - if len(f.ECSMappings) > 0 { - originalFieldCount-- - } + var formatProcessors []*formatProcessor + for _, f := range c.Format { + p, err := newFormatProcessor(c, log, f) + if err != nil { + return nil, err } + formatProcessors = append(formatProcessors, p) } return &processor{ - config: c, - fields: fields, - originalFieldCount: originalFieldCount, - expectedIPCount: ipCount, - log: log, + config: c, + formats: formatProcessors, }, nil } @@ -124,14 +109,66 @@ func (p *processor) run(event *beat.Event) error { } substrings := dst[:n] + // Find the matching format based on substring count. + for _, format := range p.formats { + if len(format.fields) == n { + return format.process(substrings, event) + } + } + return errInvalidFormat +} + +// formatProcessor processes an event using a single VPC flow log format. +type formatProcessor struct { + config + log *logp.Logger + fields []vpcFlowField + originalFieldCount int + expectedIPCount int +} + +func newFormatProcessor(c config, log *logp.Logger, format string) (*formatProcessor, error) { + fields, err := parseFormat(format) + if err != nil { + return nil, fmt.Errorf("failed to parse vpc flow log format: %w", err) + } + + originalFieldCount := len(fields) + if c.Mode == ecsMode { + for _, f := range fields { + // If an ECS mapping exists then ECS mode will not include the + // original field. + if len(f.ECSMappings) > 0 { + originalFieldCount-- + } + } + } + + var ipCount int + for _, f := range fields { + if f.Type == ipType { + ipCount++ + } + } + + return &formatProcessor{ + config: c, + log: log, + fields: fields, + originalFieldCount: originalFieldCount, + expectedIPCount: ipCount, + }, nil +} + +func (p *formatProcessor) process(substrings []string, event *beat.Event) error { + originalFields := make(mapstr.M, p.originalFieldCount) + var relatedIPs []string if p.Mode > originalMode { // Allocate space for the expected number of IPs assuming all are unique. relatedIPs = make([]string, 0, p.expectedIPCount) } - originalFields := make(mapstr.M, p.originalFieldCount) - // Iterate over the substrings in the source string and apply type // conversion and then ECS mappings. for i, word := range substrings { @@ -175,12 +212,12 @@ func (p *processor) run(event *beat.Event) error { } } - if _, err = event.PutValue(p.TargetField, originalFields); err != nil { + if _, err := event.PutValue(p.TargetField, originalFields); err != nil { return err } if len(relatedIPs) > 0 { - if _, err = event.PutValue("related.ip", relatedIPs); err != nil { + if _, err := event.PutValue("related.ip", relatedIPs); err != nil { return err } } diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index e15017734678..dfc2a8a513c6 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -12,11 +12,10 @@ import ( "testing" "time" - "gopkg.in/yaml.v2" - "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" "github.com/elastic/elastic-agent-libs/mapstr" @@ -31,14 +30,20 @@ const ( ) func TestProcessorRun(t *testing.T) { - t.Run("v5-mode-ecs_and_original", func(t *testing.T) { + t.Run("ecs_and_original-mode-v5-message", func(t *testing.T) { c := defaultConfig() - c.Format = formatV5 + c.Format = []string{ + "version account-id", // Not a match. + formatV5, + } c.Mode = ecsAndOriginalMode p, err := newParseAWSVPCFlowLog(c) require.NoError(t, err) + assert.Contains(t, p.String(), procName+"=") + assert.Contains(t, p.String(), formatV5) + evt := beat.Event{ Timestamp: time.Now().UTC(), Fields: map[string]interface{}{ @@ -152,7 +157,7 @@ func TestGoldenFile(t *testing.T) { t.Run(tc.Name, func(t *testing.T) { c := defaultConfig() - c.Format = tc.Format + c.Format = []string{tc.Format} if tc.Mode != nil { c.Mode = *tc.Mode } @@ -260,7 +265,7 @@ func BenchmarkProcessorRun(b *testing.B) { benchmark := benchmark b.Run(benchmark.name, func(b *testing.B) { c := defaultConfig() - c.Format = benchmark.format + c.Format = []string{benchmark.format} c.Mode = benchmark.mode p, err := newParseAWSVPCFlowLog(c) From ee5924b1d5c434fd2780f37cf142fb15568449dd Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Nov 2022 21:47:55 -0500 Subject: [PATCH 17/22] nolint errorlint on unit test --- .../processors/aws_vpcflow/internal/strings/strings_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go index aa9e7b25f3cf..dc111f168c0b 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go @@ -66,7 +66,7 @@ func TestFields(t *testing.T) { var smallDst [2]string for _, tt := range fieldstests { n, err := Fields(smallDst[:], tt.s) - if err == errTooManySubstrings { + if err == errTooManySubstrings { //nolint:errorlint // errTooManySubstrings is never wrapped. if len(tt.a) > len(smallDst) { continue } @@ -89,6 +89,7 @@ var FieldsFuncTests = []FieldsTest{ {"aXXbXXXcX", []string{"a", "b", "c"}}, } +//nolint:errorlint // errTooManySubstrings is never wrapped. func TestFieldsFunc(t *testing.T) { var dst [4]string for _, tt := range fieldstests { From 70bde352fa5351dcd456e6c6784b26664db5a483 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 16 Nov 2022 07:31:47 -0500 Subject: [PATCH 18/22] Store []formatProcessor instead of []*formatProcessor This gives a speedup and reduces the cost of adding multiple format support. benchmark old ns/op new ns/op delta BenchmarkProcessorRun/original-mode-v5-message-10 2225 2136 -4.00% BenchmarkProcessorRun/ecs-mode-v5-message-10 2875 2817 -2.02% BenchmarkProcessorRun/ecs_and_original-mode-v5-message-10 3352 3233 -3.55% --- .../processors/aws_vpcflow/parse_aws_vpc_flow_log.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go index ca495ee780e3..99a35bb28c15 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go @@ -36,7 +36,7 @@ var ( type processor struct { config - formats []*formatProcessor + formats []formatProcessor } // New constructs a new processor built from ucfg config. @@ -60,13 +60,13 @@ func newParseAWSVPCFlowLog(c config) (*processor, error) { return nil, err } - var formatProcessors []*formatProcessor + var formatProcessors []formatProcessor for _, f := range c.Format { p, err := newFormatProcessor(c, log, f) if err != nil { return nil, err } - formatProcessors = append(formatProcessors, p) + formatProcessors = append(formatProcessors, *p) } return &processor{ From 53f7cc99531c4d6ab4445d28ff05d38fbadf8a25 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 16 Nov 2022 09:14:44 -0500 Subject: [PATCH 19/22] Add event.action, Map protocol to network.transport --- .../processors/aws_vpcflow/mapping.go | 8 ++++++- .../parse_aws_vpc_flow_log_test.go | 3 ++- .../testdata/default-v2-format.golden.json | 15 ++++++++---- .../service-name-path-direction.golden.json | 6 +++-- .../testdata/tcp-flag-sequence.golden.json | 24 ++++++++++++------- .../testdata/transit-gateway.golden.json | 12 ++++++---- .../testdata/v5-fields-ecs.golden.json | 3 ++- .../v5-fields-ecs_and_original.golden.json | 3 ++- 8 files changed, 51 insertions(+), 23 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/mapping.go b/x-pack/filebeat/processors/aws_vpcflow/mapping.go index 08670529ae52..2f44821bcb85 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/mapping.go +++ b/x-pack/filebeat/processors/aws_vpcflow/mapping.go @@ -103,7 +103,7 @@ var vpcFlowFields = [...]vpcFlowField{ }, }, { - Target: "network.protocol", + Target: "network.transport", Transform: func(targetField string, value interface{}, event *beat.Event) { var name string switch protocol := value.(int32); protocol { @@ -190,6 +190,12 @@ var vpcFlowFields = [...]vpcFlowField{ } }, }, + { + Target: "event.action", + Transform: func(targetField string, value interface{}, event *beat.Event) { + event.PutValue(targetField, strings.ToLower(value.(string))) //nolint:errcheck // This can only fail if 'event' is not an object. + }, + }, }, }, {Name: "log_status", Type: stringType}, diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index dfc2a8a513c6..e99128fd1886 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -109,6 +109,7 @@ func TestProcessorRun(t *testing.T) { "port": int32(33004), }, "event": mapstr.M{ + "action": "reject", "end": end, "outcome": "failure", "start": start, @@ -119,7 +120,7 @@ func TestProcessorRun(t *testing.T) { "direction": "ingress", "iana_number": "17", "packets": int64(52), - "protocol": "udp", + "transport": "udp", "type": "ipv4", }, "related": mapstr.M{ diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json index 13fdd4d238d4..3dc5e832a3a7 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json @@ -19,6 +19,7 @@ "port": 22 }, "event": { + "action": "accept", "end": "2014-12-14T04:07:50Z", "outcome": "success", "start": "2014-12-14T04:06:50Z" @@ -28,7 +29,7 @@ "bytes": 4249, "iana_number": "6", "packets": 20, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -65,6 +66,7 @@ "port": 3389 }, "event": { + "action": "reject", "end": "2014-12-14T04:07:50Z", "outcome": "failure", "start": "2014-12-14T04:06:50Z" @@ -74,7 +76,7 @@ "bytes": 4249, "iana_number": "6", "packets": 20, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -151,6 +153,7 @@ "port": 0 }, "event": { + "action": "accept", "end": "2015-05-29T16:32:22Z", "outcome": "success", "start": "2015-05-29T16:30:27Z" @@ -160,7 +163,7 @@ "bytes": 336, "iana_number": "1", "packets": 4, - "protocol": "icmp", + "transport": "icmp", "type": "ipv4" }, "related": { @@ -197,6 +200,7 @@ "port": 0 }, "event": { + "action": "reject", "end": "2015-05-29T16:32:22Z", "outcome": "failure", "start": "2015-05-29T16:31:34Z" @@ -206,7 +210,7 @@ "bytes": 336, "iana_number": "1", "packets": 4, - "protocol": "icmp", + "transport": "icmp", "type": "ipv4" }, "related": { @@ -243,6 +247,7 @@ "port": 22 }, "event": { + "action": "accept", "end": "2016-10-31T11:37:00Z", "outcome": "success", "start": "2016-10-31T11:35:08Z" @@ -252,7 +257,7 @@ "bytes": 8855, "iana_number": "6", "packets": 54, - "protocol": "tcp", + "transport": "tcp", "type": "ipv6" }, "related": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json index 3ee4021df5f5..8b9905c4e858 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json @@ -36,6 +36,7 @@ "port": 34210 }, "event": { + "action": "accept", "end": "2021-03-26T03:29:09Z", "outcome": "success", "start": "2021-03-26T03:28:12Z" @@ -46,7 +47,7 @@ "direction": "ingress", "iana_number": "6", "packets": 14, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -100,6 +101,7 @@ "port": 80 }, "event": { + "action": "accept", "end": "2021-03-26T03:29:09Z", "outcome": "success", "start": "2021-03-26T03:28:12Z" @@ -110,7 +112,7 @@ "direction": "egress", "iana_number": "6", "packets": 7, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json index 2304ffda13df..c47dff8ff163 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json @@ -31,6 +31,7 @@ "port": 5001 }, "event": { + "action": "accept", "end": "2019-08-26T19:48:53Z", "outcome": "success", "start": "2019-08-26T19:47:55Z" @@ -40,7 +41,7 @@ "bytes": 568, "iana_number": "6", "packets": 8, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -90,6 +91,7 @@ "port": 43416 }, "event": { + "action": "accept", "end": "2019-08-26T19:48:53Z", "outcome": "success", "start": "2019-08-26T19:47:55Z" @@ -99,7 +101,7 @@ "bytes": 376, "iana_number": "6", "packets": 7, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -148,6 +150,7 @@ "port": 5001 }, "event": { + "action": "accept", "end": "2019-08-26T19:48:53Z", "outcome": "success", "start": "2019-08-26T19:47:55Z" @@ -157,7 +160,7 @@ "bytes": 100701, "iana_number": "6", "packets": 70, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -207,6 +210,7 @@ "port": 43418 }, "event": { + "action": "accept", "end": "2019-08-26T19:48:53Z", "outcome": "success", "start": "2019-08-26T19:47:55Z" @@ -216,7 +220,7 @@ "bytes": 632, "iana_number": "6", "packets": 12, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -265,6 +269,7 @@ "port": 43418 }, "event": { + "action": "accept", "end": "2019-08-26T19:51:53Z", "outcome": "success", "start": "2019-08-26T19:48:53Z" @@ -274,7 +279,7 @@ "bytes": 63388, "iana_number": "6", "packets": 1219, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -323,6 +328,7 @@ "port": 5001 }, "event": { + "action": "accept", "end": "2019-08-26T19:51:53Z", "outcome": "success", "start": "2019-08-26T19:48:53Z" @@ -332,7 +338,7 @@ "bytes": 23294588, "iana_number": "6", "packets": 15774, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -382,6 +388,7 @@ "port": 5001 }, "event": { + "action": "accept", "end": "2019-08-27T19:13:13Z", "outcome": "success", "start": "2019-08-27T19:12:13Z" @@ -391,7 +398,7 @@ "bytes": 1260, "iana_number": "6", "packets": 17, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -442,6 +449,7 @@ "port": 43638 }, "event": { + "action": "accept", "end": "2019-08-27T19:13:13Z", "outcome": "success", "start": "2019-08-27T19:12:13Z" @@ -451,7 +459,7 @@ "bytes": 967, "iana_number": "6", "packets": 14, - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json index 1a76557633c3..55f22e4b00e0 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json @@ -31,12 +31,13 @@ "port": 80 }, "event": { + "action": "accept", "outcome": "success" }, "message": "3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.20.33.164 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK", "network": { "iana_number": "6", - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -84,12 +85,13 @@ "port": 39812 }, "event": { + "action": "accept", "outcome": "success" }, "message": "3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.40.2.236 10.20.33.164 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK", "network": { "iana_number": "6", - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -133,12 +135,13 @@ "port": 80 }, "event": { + "action": "accept", "outcome": "success" }, "message": "3 eni-11111111111111111 123456789010 vpc-abcdefab012345678 subnet-11111111aaaaaaaaa - 10.40.1.175 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK", "network": { "iana_number": "6", - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { @@ -184,12 +187,13 @@ "port": 39812 }, "event": { + "action": "accept", "outcome": "success" }, "message": "3 eni-22222222222222222 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb - 10.40.2.236 10.40.2.31 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK", "network": { "iana_number": "6", - "protocol": "tcp", + "transport": "tcp", "type": "ipv4" }, "related": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json index 32e537dba444..94179bc11cbd 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json @@ -38,6 +38,7 @@ "port": 33004 }, "event": { + "action": "reject", "end": "2021-03-26T03:29:09Z", "outcome": "failure", "start": "2021-03-26T03:28:12Z" @@ -48,7 +49,7 @@ "direction": "ingress", "iana_number": "17", "packets": 52, - "protocol": "udp", + "transport": "udp", "type": "ipv4" }, "related": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json index da73ac7471d8..cfbd807ff0cb 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json @@ -53,6 +53,7 @@ "port": 33004 }, "event": { + "action": "reject", "end": "2021-03-26T03:29:09Z", "outcome": "failure", "start": "2021-03-26T03:28:12Z" @@ -63,7 +64,7 @@ "direction": "ingress", "iana_number": "17", "packets": 52, - "protocol": "udp", + "transport": "udp", "type": "ipv4" }, "related": { From 024b82be58af7e3510ff68d8972037b89ac2c261 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 16 Nov 2022 09:16:05 -0500 Subject: [PATCH 20/22] Update ECS mapping in docs --- .../aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc b/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc index cacedfa21b81..8977a84a2970 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc +++ b/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc @@ -95,6 +95,7 @@ transformations applied to derive the ECS field. | VPC Flow Log Field | ECS Field | | account_id | cloud.account.id | | action | event.outcome | +| action | event.action | | az_id | cloud.availability_zone | | bytes | network.bytes | | bytes | source.bytes | @@ -108,7 +109,7 @@ transformations applied to derive the ECS field. | packets | network.packets | | packets | source.packets | | protocol | network.iana_number | -| protocol | network.protocol | +| protocol | network.transport | | region | cloud.region | | srcaddr | network.type | | srcaddr | source.address | From c644868bfc87d5f848d840f2413ece3c33916b65 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 16 Nov 2022 11:01:23 -0500 Subject: [PATCH 21/22] Add event.type Event type is a list. It will always contain "connection". Add "allowed" or "denied" will be added based on the vpc flow action of "ACCEPT" or "REJECT". --- .../processors/aws_vpcflow/mapping.go | 24 ++++++++++ .../aws_vpcflow/parse_aws_vpc_flow_log.go | 7 +++ .../parse_aws_vpc_flow_log_test.go | 1 + .../testdata/custom-nat-gateway.golden.json | 30 ++++++++++++ .../testdata/default-v2-format.golden.json | 40 +++++++++++++--- .../service-name-path-direction.golden.json | 12 ++++- .../testdata/tcp-flag-sequence.golden.json | 48 +++++++++++++++---- .../testdata/transit-gateway.golden.json | 24 ++++++++-- .../testdata/v5-fields-ecs.golden.json | 6 ++- .../v5-fields-ecs_and_original.golden.json | 6 ++- 10 files changed, 175 insertions(+), 23 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/mapping.go b/x-pack/filebeat/processors/aws_vpcflow/mapping.go index 2f44821bcb85..ce4679f89e6b 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/mapping.go +++ b/x-pack/filebeat/processors/aws_vpcflow/mapping.go @@ -196,6 +196,30 @@ var vpcFlowFields = [...]vpcFlowField{ event.PutValue(targetField, strings.ToLower(value.(string))) //nolint:errcheck // This can only fail if 'event' is not an object. }, }, + { + Target: "event.type", + Transform: func(targetField string, value interface{}, event *beat.Event) { + var eventType string + + switch s := value.(string); s { + case "ACCEPT": + eventType = "allowed" + case "REJECT": + eventType = "denied" + } + + if len(eventType) > 0 { + // The processor always adds event.type: [connection] in ECS mode. + v, _ := event.GetValue(targetField) + if eventTypes, ok := v.([]string); ok { + event.PutValue(targetField, append(eventTypes, eventType)) //nolint:errcheck // This can only fail if 'event' is not an object. + return + } + + event.PutValue(targetField, []string{eventType}) //nolint:errcheck // This can only fail if 'event' is not an object. + } + }, + }, }, }, {Name: "log_status", Type: stringType}, diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go index 99a35bb28c15..dafbe748dcde 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go @@ -167,6 +167,13 @@ func (p *formatProcessor) process(substrings []string, event *beat.Event) error if p.Mode > originalMode { // Allocate space for the expected number of IPs assuming all are unique. relatedIPs = make([]string, 0, p.expectedIPCount) + + // Preallocate event.type with extra capacity for "allowed" or "denied". + eventTypes := make([]string, 1, 2) + eventTypes[0] = "connection" + if _, err := event.PutValue("event.type", eventTypes); err != nil { + return err + } } // Iterate over the substrings in the source string and apply type diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go index e99128fd1886..2744256a1bab 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log_test.go @@ -113,6 +113,7 @@ func TestProcessorRun(t *testing.T) { "end": end, "outcome": "failure", "start": start, + "type": []string{"connection", "denied"}, }, "message": formatV5Sample, "network": mapstr.M{ diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.json index 08f94b6b5f46..ca511ed5c8e1 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/custom-nat-gateway.golden.json @@ -11,6 +11,11 @@ "address": "10.0.0.220", "ip": "10.0.0.220" }, + "event": { + "type": [ + "connection" + ] + }, "message": "- eni-1235b8ca123456789 10.0.1.5 10.0.0.220 10.0.1.5 203.0.113.5", "network": { "type": "ipv4" @@ -39,6 +44,11 @@ "address": "203.0.113.5", "ip": "203.0.113.5" }, + "event": { + "type": [ + "connection" + ] + }, "message": "- eni-1235b8ca123456789 10.0.0.220 203.0.113.5 10.0.0.220 203.0.113.5", "network": { "type": "ipv4" @@ -66,6 +76,11 @@ "address": "10.0.0.220", "ip": "10.0.0.220" }, + "event": { + "type": [ + "connection" + ] + }, "message": "- eni-1235b8ca123456789 203.0.113.5 10.0.0.220 203.0.113.5 10.0.0.220", "network": { "type": "ipv4" @@ -93,6 +108,11 @@ "address": "10.0.1.5", "ip": "10.0.1.5" }, + "event": { + "type": [ + "connection" + ] + }, "message": "- eni-1235b8ca123456789 10.0.0.220 10.0.1.5 203.0.113.5 10.0.1.5", "network": { "type": "ipv4" @@ -126,6 +146,11 @@ "address": "203.0.113.5", "ip": "203.0.113.5" }, + "event": { + "type": [ + "connection" + ] + }, "message": "i-01234567890123456 eni-1111aaaa2222bbbb3 10.0.1.5 203.0.113.5 10.0.1.5 203.0.113.5", "network": { "type": "ipv4" @@ -158,6 +183,11 @@ "address": "10.0.1.5", "ip": "10.0.1.5" }, + "event": { + "type": [ + "connection" + ] + }, "message": "i-01234567890123456 eni-1111aaaa2222bbbb3 203.0.113.5 10.0.1.5 203.0.113.5 10.0.1.5", "network": { "type": "ipv4" diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json index 3dc5e832a3a7..72a0bac1fd00 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/default-v2-format.golden.json @@ -22,7 +22,11 @@ "action": "accept", "end": "2014-12-14T04:07:50Z", "outcome": "success", - "start": "2014-12-14T04:06:50Z" + "start": "2014-12-14T04:06:50Z", + "type": [ + "connection", + "allowed" + ] }, "message": "2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK", "network": { @@ -69,7 +73,11 @@ "action": "reject", "end": "2014-12-14T04:07:50Z", "outcome": "failure", - "start": "2014-12-14T04:06:50Z" + "start": "2014-12-14T04:06:50Z", + "type": [ + "connection", + "denied" + ] }, "message": "2 123456789010 eni-1235b8ca123456789 172.31.9.69 172.31.9.12 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK", "network": { @@ -109,7 +117,10 @@ }, "event": { "end": "2015-05-10T18:02:14Z", - "start": "2015-05-10T18:01:16Z" + "start": "2015-05-10T18:01:16Z", + "type": [ + "connection" + ] }, "message": "2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA" }, @@ -129,7 +140,10 @@ }, "event": { "end": "2015-05-10T18:02:14Z", - "start": "2015-05-10T18:01:16Z" + "start": "2015-05-10T18:01:16Z", + "type": [ + "connection" + ] }, "message": "2 123456789010 eni-11111111aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA" }, @@ -156,7 +170,11 @@ "action": "accept", "end": "2015-05-29T16:32:22Z", "outcome": "success", - "start": "2015-05-29T16:30:27Z" + "start": "2015-05-29T16:30:27Z", + "type": [ + "connection", + "allowed" + ] }, "message": "2 123456789010 eni-1235b8ca123456789 203.0.113.12 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK", "network": { @@ -203,7 +221,11 @@ "action": "reject", "end": "2015-05-29T16:32:22Z", "outcome": "failure", - "start": "2015-05-29T16:31:34Z" + "start": "2015-05-29T16:31:34Z", + "type": [ + "connection", + "denied" + ] }, "message": "2 123456789010 eni-1235b8ca123456789 172.31.16.139 203.0.113.12 0 0 1 4 336 1432917094 1432917142 REJECT OK", "network": { @@ -250,7 +272,11 @@ "action": "accept", "end": "2016-10-31T11:37:00Z", "outcome": "success", - "start": "2016-10-31T11:35:08Z" + "start": "2016-10-31T11:35:08Z", + "type": [ + "connection", + "allowed" + ] }, "message": "2 123456789010 eni-1235b8ca123456789 2001:db8:1234:a100:8d6e:3477:df66:f105 2001:db8:1234:a102:3304:8879:34cf:4071 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK", "network": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json index 8b9905c4e858..ddf0a807ff50 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/service-name-path-direction.golden.json @@ -39,7 +39,11 @@ "action": "accept", "end": "2021-03-26T03:29:09Z", "outcome": "success", - "start": "2021-03-26T03:28:12Z" + "start": "2021-03-26T03:28:12Z", + "type": [ + "connection", + "allowed" + ] }, "message": "5 52.95.128.179 10.0.0.71 80 34210 6 1616729292 1616729349 IPv4 14 15044 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 19 52.95.128.179 10.0.0.71 S3 - - ingress OK", "network": { @@ -104,7 +108,11 @@ "action": "accept", "end": "2021-03-26T03:29:09Z", "outcome": "success", - "start": "2021-03-26T03:28:12Z" + "start": "2021-03-26T03:28:12Z", + "type": [ + "connection", + "allowed" + ] }, "message": "5 10.0.0.71 52.95.128.179 34210 80 6 1616729292 1616729349 IPv4 7 471 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 3 10.0.0.71 52.95.128.179 - S3 8 egress OK", "network": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json index c47dff8ff163..b879422eeca4 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/tcp-flag-sequence.golden.json @@ -34,7 +34,11 @@ "action": "accept", "end": "2019-08-26T19:48:53Z", "outcome": "success", - "start": "2019-08-26T19:47:55Z" + "start": "2019-08-26T19:47:55Z", + "type": [ + "connection", + "allowed" + ] }, "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK", "network": { @@ -94,7 +98,11 @@ "action": "accept", "end": "2019-08-26T19:48:53Z", "outcome": "success", - "start": "2019-08-26T19:47:55Z" + "start": "2019-08-26T19:47:55Z", + "type": [ + "connection", + "allowed" + ] }, "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43416 10.0.0.62 52.213.180.42 6 376 7 1566848875 1566848933 ACCEPT 18 OK", "network": { @@ -153,7 +161,11 @@ "action": "accept", "end": "2019-08-26T19:48:53Z", "outcome": "success", - "start": "2019-08-26T19:47:55Z" + "start": "2019-08-26T19:47:55Z", + "type": [ + "connection", + "allowed" + ] }, "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 100701 70 1566848875 1566848933 ACCEPT 2 OK", "network": { @@ -213,7 +225,11 @@ "action": "accept", "end": "2019-08-26T19:48:53Z", "outcome": "success", - "start": "2019-08-26T19:47:55Z" + "start": "2019-08-26T19:47:55Z", + "type": [ + "connection", + "allowed" + ] }, "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 632 12 1566848875 1566848933 ACCEPT 18 OK", "network": { @@ -272,7 +288,11 @@ "action": "accept", "end": "2019-08-26T19:51:53Z", "outcome": "success", - "start": "2019-08-26T19:48:53Z" + "start": "2019-08-26T19:48:53Z", + "type": [ + "connection", + "allowed" + ] }, "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43418 10.0.0.62 52.213.180.42 6 63388 1219 1566848933 1566849113 ACCEPT 1 OK", "network": { @@ -331,7 +351,11 @@ "action": "accept", "end": "2019-08-26T19:51:53Z", "outcome": "success", - "start": "2019-08-26T19:48:53Z" + "start": "2019-08-26T19:48:53Z", + "type": [ + "connection", + "allowed" + ] }, "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43418 5001 52.213.180.42 10.0.0.62 6 23294588 15774 1566848933 1566849113 ACCEPT 1 OK", "network": { @@ -391,7 +415,11 @@ "action": "accept", "end": "2019-08-27T19:13:13Z", "outcome": "success", - "start": "2019-08-27T19:12:13Z" + "start": "2019-08-27T19:12:13Z", + "type": [ + "connection", + "allowed" + ] }, "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43638 5001 52.213.180.42 10.0.0.62 6 1260 17 1566933133 1566933193 ACCEPT 3 OK", "network": { @@ -452,7 +480,11 @@ "action": "accept", "end": "2019-08-27T19:13:13Z", "outcome": "success", - "start": "2019-08-27T19:12:13Z" + "start": "2019-08-27T19:12:13Z", + "type": [ + "connection", + "allowed" + ] }, "message": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43638 10.0.0.62 52.213.180.42 6 967 14 1566933133 1566933193 ACCEPT 19 OK", "network": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json index 55f22e4b00e0..57ca61c35520 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/transit-gateway.golden.json @@ -32,7 +32,11 @@ }, "event": { "action": "accept", - "outcome": "success" + "outcome": "success", + "type": [ + "connection", + "allowed" + ] }, "message": "3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.20.33.164 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK", "network": { @@ -86,7 +90,11 @@ }, "event": { "action": "accept", - "outcome": "success" + "outcome": "success", + "type": [ + "connection", + "allowed" + ] }, "message": "3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.40.2.236 10.20.33.164 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK", "network": { @@ -136,7 +144,11 @@ }, "event": { "action": "accept", - "outcome": "success" + "outcome": "success", + "type": [ + "connection", + "allowed" + ] }, "message": "3 eni-11111111111111111 123456789010 vpc-abcdefab012345678 subnet-11111111aaaaaaaaa - 10.40.1.175 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK", "network": { @@ -188,7 +200,11 @@ }, "event": { "action": "accept", - "outcome": "success" + "outcome": "success", + "type": [ + "connection", + "allowed" + ] }, "message": "3 eni-22222222222222222 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb - 10.40.2.236 10.40.2.31 80 39812 6 19 IPv4 10.40.2.236 10.20.33.164 ACCEPT OK", "network": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json index 94179bc11cbd..09cdb9c44756 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs.golden.json @@ -41,7 +41,11 @@ "action": "reject", "end": "2021-03-26T03:29:09Z", "outcome": "failure", - "start": "2021-03-26T03:28:12Z" + "start": "2021-03-26T03:28:12Z", + "type": [ + "connection", + "denied" + ] }, "message": "5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1", "network": { diff --git a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json index cfbd807ff0cb..8ad491c45ccf 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json +++ b/x-pack/filebeat/processors/aws_vpcflow/testdata/v5-fields-ecs_and_original.golden.json @@ -56,7 +56,11 @@ "action": "reject", "end": "2021-03-26T03:29:09Z", "outcome": "failure", - "start": "2021-03-26T03:28:12Z" + "start": "2021-03-26T03:28:12Z", + "type": [ + "connection", + "denied" + ] }, "message": "5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1", "network": { From e16f762607f34f24feafe15bb09fe8a5582ffbf9 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 16 Nov 2022 11:08:52 -0500 Subject: [PATCH 22/22] Update docs --- .../aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc b/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc index 8977a84a2970..9c9b1f4539cb 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc +++ b/x-pack/filebeat/processors/aws_vpcflow/docs/parse_aws_vpc_flow_log.asciidoc @@ -96,6 +96,7 @@ transformations applied to derive the ECS field. | account_id | cloud.account.id | | action | event.outcome | | action | event.action | +| action | event.type | | az_id | cloud.availability_zone | | bytes | network.bytes | | bytes | source.bytes | @@ -180,9 +181,14 @@ is an example document produced using `ecs_and_orignal` mode. "port": 33004 }, "event": { + "action": "reject", "end": "2021-03-26T03:29:09Z", "outcome": "failure", - "start": "2021-03-26T03:28:12Z" + "start": "2021-03-26T03:28:12Z", + "type": [ + "connection", + "denied" + ] }, "message": "5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 1616729292 1616729349 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 wavelength fake-id AMAZON CLOUDFRONT ingress 1", "network": { @@ -190,7 +196,7 @@ is an example document produced using `ecs_and_orignal` mode. "direction": "ingress", "iana_number": "17", "packets": 52, - "protocol": "udp", + "transport": "udp", "type": "ipv4" }, "related": {