Add false positive package to detect regressions in JSON marshalling

internal/testrunner/runners/pipeline/runner.go

@@ -179,8 +179,6 @@ func (r *runner) run() ([]testrunner.TestResult, error) {
 		}
 		startTime := time.Now()
 
-		// TODO: Add tests to cover regressive use of json.Unmarshal in loadTestCaseFile.
-		// See https://github.com/elastic/elastic-package/pull/717.
 		tc, err := r.loadTestCaseFile(testCaseFile)
 		if err != nil {
 			err := fmt.Errorf("loading test case failed: %w", err)
@@ -225,8 +223,6 @@ func (r *runner) run() ([]testrunner.TestResult, error) {
 			return nil, fmt.Errorf("creating fields validator for data stream failed (path: %s, test case file: %s): %w", dataStreamPath, testCaseFile, err)
 		}
 
-		// TODO: Add tests to cover regressive use of json.Unmarshal in verifyResults.
-		// See https://github.com/elastic/elastic-package/pull/717.
 		err = r.verifyResults(testCaseFile, tc.config, result, fieldsValidator)
 		if e, ok := err.(testrunner.ErrTestCaseFailed); ok {
 			tr.FailureMsg = e.Error()
@@ -328,8 +324,6 @@ func (r *runner) verifyResults(testCaseFile string, config *testConfig, result *
 	}
 
 	if r.options.GenerateTestResult {
-		// TODO: Add tests to cover regressive use of json.Unmarshal in writeTestResult.
-		// See https://github.com/elastic/elastic-package/pull/717.
 		err := writeTestResult(testCasePath, result, *specVersion)
 		if err != nil {
 			return fmt.Errorf("writing test result failed: %w", err)

test/packages/false_positives/long_integers_as_json_numbers.expected_errors

@@ -0,0 +1 @@
+<failure>test case failed: Expected results are different from actual ones:.*</failure>

test/packages/false_positives/long_integers_as_json_numbers/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@1.10

test/packages/false_positives/long_integers_as_json_numbers/_dev/build/docs/README.md

@@ -0,0 +1,5 @@
+# Long Integer Tests
+
+{{event "test"}}
+
+{{fields "test"}}

test/packages/false_positives/long_integers_as_json_numbers/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

test/packages/false_positives/long_integers_as_json_numbers/data_stream/test/_dev/test/pipeline/test-sequences.log

@@ -0,0 +1 @@
+4503599627370497,9007199254740991,9007199254740993,18014398509481985,9223372036854773807

test/packages/false_positives/long_integers_as_json_numbers/data_stream/test/_dev/test/pipeline/test-sequences.log-config.yml

@@ -0,0 +1,2 @@
+fields:
+  "@warning": "The values in sequence_number must match the values in message."

test/packages/false_positives/long_integers_as_json_numbers/data_stream/test/_dev/test/pipeline/test-sequences.log-expected.json

@@ -0,0 +1,15 @@
+{
+    "expected": [
+        {
+            "@warning": "The values in sequence_number must match the values in message.",
+            "message": "4503599627370497,9007199254740991,9007199254740993,18014398509481985,9223372036854773807",
+            "sequence_number": [
+                4503599627370497,
+                9007199254740991,
+                9007199254740992,
+                18014398509481984,
+                9223372036854774000
+            ]
+        }
+    ]
+}

test/packages/false_positives/long_integers_as_json_numbers/data_stream/test/agent/stream/stream.yml.hbs

@@ -0,0 +1,7 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~

test/packages/false_positives/long_integers_as_json_numbers/data_stream/test/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,16 @@
+---
+description: Pipeline for processing sample logs
+processors:
+- split:
+    field: message
+    separator: ","
+    target_field: sequence_number
+    ignore_missing: true
+- convert:
+    field: sequence_number
+    type: long
+
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

test/packages/false_positives/long_integers_as_json_numbers/data_stream/test/fields/base-fields.yml

@@ -0,0 +1,22 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
+- name: '@warning'
+  type: keyword
+  description: Warning for devs.
+- name: message
+  type: keyword
+  description: Original input.
+- name: sequence_number
+  type: long
+  description: |
+    Log entry identifier that is incremented sequentially. Unique for each log type.

test/packages/false_positives/long_integers_as_json_numbers/data_stream/test/manifest.yml

@@ -0,0 +1,13 @@
+title: "Test"
+type: logs
+streams:
+  - input: logfile
+    title: Sample logs
+    description: Collect sample logs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        default:
+          - /var/log/*.log

test/packages/false_positives/long_integers_as_json_numbers/data_stream/test/sample_event.json

@@ -0,0 +1,11 @@
+{
+    "@warning": "The values in sequence_number must match the values in message.",
+    "message": "4503599627370497,9007199254740991,9007199254740993,18014398509481985,9223372036854773807",
+    "sequence_number": [
+        4503599627370497,
+        9007199254740991,
+        9007199254740993,
+        18014398509481985,
+        9223372036854773807
+    ]
+}

test/packages/false_positives/long_integers_as_json_numbers/docs/README.md

@@ -0,0 +1,29 @@
+# Long Integer Tests
+
+An example event for `test` looks as following:
+
+```json
+{
+    "@warning": "The values in sequence_number must match the values in message.",
+    "message": "4503599627370497,9007199254740991,9007199254740993,18014398509481985,9223372036854773807",
+    "sequence_number": [
+        4503599627370497,
+        9007199254740991,
+        9007199254740993,
+        18014398509481985,
+        9223372036854773807
+    ]
+}
+```
+
+**Exported fields**
+
+| Field | Description | Type |
+|---|---|---|
+| @timestamp | Event timestamp. | date |
+| @warning | Warning for devs. | keyword |
+| data_stream.dataset | Data stream dataset. | constant_keyword |
+| data_stream.namespace | Data stream namespace. | constant_keyword |
+| data_stream.type | Data stream type. | constant_keyword |
+| message | Original input. | keyword |
+| sequence_number | Log entry identifier that is incremented sequentially. Unique for each log type. | long |

test/packages/false_positives/long_integers_as_json_numbers/manifest.yml

@@ -0,0 +1,22 @@
+format_version: 1.0.0
+name: long_integers_as_json_numbers
+title: "Long Integer Tests"
+version: 0.0.1
+license: basic
+description: "These are tests of long integer round tripping, they should fail if JSON is not decoded using JSON numbers."
+type: integration
+categories:
+  - custom
+release: experimental
+conditions:
+  kibana.version: "^8.0.0"
+policy_templates:
+  - name: test
+    title: Test
+    description: Description
+    inputs:
+      - type: file
+        title: Collect logs via file.
+        description: Collecting logs via file.
+owner:
+  github: elastic/integrations