Fix validation of empty numeric keywords arrays with normalization (#…

…1773)

internal/fields/testdata/fields/fields.yml

@@ -3,6 +3,10 @@
   fields:
     - name: code
       type: keyword
+    - name: pid
+      type: keyword
+    - name: ppid
+      type: keyword
     - name: flattened
       type: group
       fields:
@@ -78,3 +82,7 @@
   multi_fields:
     - name: text
       type: text
+- name: tags
+  type: keyword
+  normalize:
+    - array

internal/fields/testdata/numeric.json

@@ -1,11 +1,14 @@
 {
   "foo": {
     "code": 42,
+    "pid": [345, 678, 901],
+    "ppid": [],
     "flattened": {
       "request_parameters": {
         "userName": "Bob",
         "groupName": "admin"
       }
     }
-  }
-}
+  },
+  "tags": []
+}

internal/fields/validate.go

@@ -593,7 +593,7 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo
 	// Convert numeric keyword fields to string for validation.
 	_, found := v.numericKeywordFields[key]
 	if (found || v.defaultNumericConversion) && isNumericKeyword(*definition, val) {
-		val = fmt.Sprintf("%q", val)
+		val = convertNumericKeyword(val)
 	}
 
 	if !v.disabledNormalization {
@@ -692,12 +692,15 @@ func createDocExpandingObjects(doc common.MapStr) (common.MapStr, error) {
 	}
 	return newDoc, nil
 }
-func isNumericKeyword(definition FieldDefinition, val interface{}) bool {
+
+// isNumericKeyword is used to identify values that can be numbers in the documents, but are ingested
+// as keywords.
+func isNumericKeyword(definition FieldDefinition, val any) bool {
 	var isNumber bool
 	switch val := val.(type) {
 	case bool, []bool, float64, []float64:
 		isNumber = true
-	case []interface{}:
+	case []any:
 		isNumber = true
 	loop:
 		for _, v := range val {
@@ -712,6 +715,19 @@ func isNumericKeyword(definition FieldDefinition, val interface{}) bool {
 	return isNumber && (definition.Type == "keyword" || definition.Type == "constant_keyword")
 }
 
+func convertNumericKeyword(val any) any {
+	switch val := val.(type) {
+	case []any:
+		converted := make([]any, len(val))
+		for i, e := range val {
+			converted[i] = convertNumericKeyword(e)
+		}
+		return converted
+	default:
+		return fmt.Sprintf("%q", val)
+	}
+}
+
 // skipValidationForField skips field validation (field presence) of special fields. The special fields are present
 // in every (most?) documents collected by Elastic Agent, but aren't defined in any integration in `fields.yml` files.
 // FIXME https://github.com/elastic/elastic-package/issues/147

internal/fields/validate_test.go

@@ -66,7 +66,13 @@ func TestValidate_WithFlattenedFields(t *testing.T) {
 
 func TestValidate_WithNumericKeywordFields(t *testing.T) {
 	validator, err := CreateValidatorForDirectory("testdata",
-		WithNumericKeywordFields([]string{"foo.code"}),
+		WithNumericKeywordFields([]string{
+			"foo.code", // Contains a number.
+			"foo.pid",  // Contains an array of numbers.
+			"foo.ppid", // Contains an empty array.
+			"tags",     // Contains an empty array, and expects normalization as array.
+		}),
+		WithSpecVersion("2.3.0"), // Needed to validate normalization.
 		WithDisabledDependencyManagement())
 	require.NoError(t, err)
 	require.NotNil(t, validator)