[Filebeat] Add parse_aws_vpc_flow_log processor

x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings.go

@@ -1,6 +1,11 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package strings
 
 import (
+	"errors"
 	"unicode"
 	"unicode/utf8"
 )
@@ -16,7 +21,10 @@ var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1}
 // Fields splits the string s around each instance of one or more consecutive white space
 // characters, as defined by unicode.IsSpace, returning a slice of substrings of s or an
 // empty slice if s contains only white space.
-func Fields(s string) []string {
+//
+// It writes at most len(dst) substrings from s into dst. It returns the number
+// of substrings copied and an error if there were more than len(dst) substrings.
+func Fields(dst []string, s string) (int, error) {
 	// First count the fields.
 	// This is an exact count if s is ASCII, otherwise it is an approximation.
 	n := 0
@@ -33,10 +41,9 @@ func Fields(s string) []string {
 
 	if setBits >= utf8.RuneSelf {
 		// Some runes in the input string are not ASCII.
-		return FieldsFunc(s, unicode.IsSpace)
+		return fieldsFunc(dst, s, unicode.IsSpace)
 	}
 	// ASCII fast path
-	a := make([]string, n)
 	na := 0
 	fieldStart := 0
 	i := 0
@@ -50,7 +57,10 @@ func Fields(s string) []string {
 			i++
 			continue
 		}
-		a[na] = s[fieldStart:i]
+		if na >= len(dst) {
+			return na, errTooManySubstrings
+		}
+		dst[na] = s[fieldStart:i]
 		na++
 		i++
 		// Skip spaces in between fields.
@@ -60,25 +70,29 @@ func Fields(s string) []string {
 		fieldStart = i
 	}
 	if fieldStart < len(s) { // Last field might end at EOF.
-		a[na] = s[fieldStart:]
+		if na >= len(dst) {
+			return na, errTooManySubstrings
+		}
+		dst[na] = s[fieldStart:]
+		na++
 	}
-	return a
+	return na, nil
 }
 
-// FieldsFunc splits the string s at each run of Unicode code points c satisfying f(c)
+// fieldsFunc splits the string s at each run of Unicode code points c satisfying f(c)
 // and returns an array of slices of s. If all code points in s satisfy f(c) or the
 // string is empty, an empty slice is returned.
 //
 // FieldsFunc makes no guarantees about the order in which it calls f(c)
 // and assumes that f always returns the same value for a given c.
-func FieldsFunc(s string, f func(rune) bool) []string {
+func fieldsFunc(dst []string, s string, f func(rune) bool) (int, error) {
 	// A span is used to record a slice of s of the form s[start:end].
 	// The start index is inclusive and the end index is exclusive.
 	type span struct {
 		start int
 		end   int
 	}
-	spans := make([]span, 0, 32)
+	spans := make([]span, 0, len(dst))
 
 	// Find the field start and end indices.
 	// Doing this in a separate pass (rather than slicing the string s
@@ -88,7 +102,11 @@ func FieldsFunc(s string, f func(rune) bool) []string {
 	for end, rune := range s {
 		if f(rune) {
 			if start >= 0 {
-				spans = append(spans, span{start, end})
+				if len(spans) < len(dst) {
+					spans = append(spans, span{start, end})
+				} else {
+					break
+				}
 				// Set start to a negative value.
 				// Note: using -1 here consistently and reproducibly
 				// slows down this code by a several percent on amd64.
@@ -102,15 +120,14 @@ func FieldsFunc(s string, f func(rune) bool) []string {
 	}
 
 	// Last field might end at EOF.
-	if start >= 0 {
+	if start >= 0 && len(spans) < len(dst) {
 		spans = append(spans, span{start, len(s)})
 	}
 
 	// Create strings from recorded field indices.
-	a := make([]string, len(spans))
 	for i, span := range spans {
-		a[i] = s[span.start:span.end]
+		dst[i] = s[span.start:span.end]
 	}
 
-	return a
+	return len(spans), nil
 }

x-pack/filebeat/processors/aws_vpcflow/internal/strings/strings_test.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package strings
 
 import (
@@ -43,10 +47,36 @@ var fieldstests = []FieldsTest{
 }
 
 func TestFields(t *testing.T) {
+	var dst [4]string
 	for _, tt := range fieldstests {
-		a := Fields(tt.s)
-		if !eq(a, tt.a) {
-			t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a)
+		n, err := Fields(dst[:], tt.s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !eq(dst[:n], tt.a) {
+			t.Errorf("Fields(%q) = %v; want %v", tt.s, dst[:n], tt.a)
+			continue
+		}
+		if len(tt.a) != n {
+			t.Errorf("Return count n = %d; want %d", n, len(tt.a))
+		}
+	}
+
+	// Smaller
+	var smallDst [2]string
+	for _, tt := range fieldstests {
+		n, err := Fields(smallDst[:], tt.s)
+		if err == errTooManySubstrings {
+			if len(tt.a) > len(smallDst) {
+				continue
+			}
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !eq(smallDst[:n], tt.a[:n]) {
+			t.Errorf("Fields(%q) = %v; want %v", tt.s, smallDst[:n], tt.a)
 			continue
 		}
 	}
@@ -60,18 +90,66 @@ var FieldsFuncTests = []FieldsTest{
 }
 
 func TestFieldsFunc(t *testing.T) {
+	var dst [4]string
 	for _, tt := range fieldstests {
-		a := FieldsFunc(tt.s, unicode.IsSpace)
-		if !eq(a, tt.a) {
-			t.Errorf("FieldsFunc(%q, unicode.IsSpace) = %v; want %v", tt.s, a, tt.a)
+		n, err := fieldsFunc(dst[:], tt.s, unicode.IsSpace)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !eq(dst[:n], tt.a) {
+			t.Errorf("FieldsFunc(%q, unicode.IsSpace) = %v; want %v", tt.s, dst, tt.a)
 			continue
 		}
+		if len(tt.a) != n {
+			t.Errorf("Return count n = %d; want %d", n, len(tt.a))
+		}
 	}
 	pred := func(c rune) bool { return c == 'X' }
 	for _, tt := range FieldsFuncTests {
-		a := FieldsFunc(tt.s, pred)
-		if !eq(a, tt.a) {
-			t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a)
+		n, err := fieldsFunc(dst[:], tt.s, pred)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !eq(dst[:n], tt.a) {
+			t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, dst[:n], tt.a)
+		}
+		if len(tt.a) != n {
+			t.Errorf("Return count n = %d; want %d", n, len(tt.a))
+		}
+	}
+
+	// Smaller
+	var smallDst [2]string
+	for _, tt := range fieldstests {
+		n, err := Fields(smallDst[:], tt.s)
+		if err == errTooManySubstrings {
+			if len(tt.a) > len(smallDst) {
+				continue
+			}
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !eq(smallDst[:n], tt.a[:n]) {
+			t.Errorf("Fields(%q) = %v; want %v", tt.s, smallDst[:n], tt.a)
+			continue
+		}
+	}
+	for _, tt := range FieldsFuncTests {
+		n, err := fieldsFunc(smallDst[:], tt.s, pred)
+		if err == errTooManySubstrings {
+			if len(tt.a) > len(smallDst) {
+				continue
+			}
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !eq(smallDst[:n], tt.a[:n]) {
+			t.Errorf("Fields(%q) = %v; want %v", tt.s, smallDst[:n], tt.a)
+			continue
 		}
 	}
 }