Disable normalization validating fields in transforms if synthetics i…

…s enabled (#2011)

This PR allows to disable normalization when validating those fields
(from transforms) when synthetic is enabled in the data stream created for
testing purposes. Added a new test package to test this scenario with
logsdb enabled.

internal/testrunner/runners/system/tester.go

@@ -842,14 +842,35 @@ type scenarioTest struct {
 	startTestTime      time.Time
 }
 
+type pipelineTrace []string
+
+func (p *pipelineTrace) UnmarshalJSON(d []byte) error {
+	var alias interface{}
+	if err := json.Unmarshal(d, &alias); err != nil {
+		return err
+	}
+	switch v := alias.(type) {
+	case string:
+		*p = append(*p, v)
+	case []any:
+		// asume it is going to be an array of strings
+		for _, value := range v {
+			*p = append(*p, fmt.Sprint(value))
+		}
+	default:
+		return fmt.Errorf("unexpected type found for pipeline_trace: %T", v)
+	}
+	return nil
+}
+
 type failureStoreDocument struct {
 	Error struct {
-		Type          string   `json:"type"`
-		Message       string   `json:"message"`
-		StackTrace    string   `json:"stack_trace"`
-		PipelineTrace []string `json:"pipeline_trace"`
-		Pipeline      string   `json:"pipeline"`
-		ProcessorType string   `json:"processor_type"`
+		Type          string        `json:"type"`
+		Message       string        `json:"message"`
+		StackTrace    string        `json:"stack_trace"`
+		PipelineTrace pipelineTrace `json:"pipeline_trace"`
+		Pipeline      string        `json:"pipeline"`
+		ProcessorType string        `json:"processor_type"`
 	} `json:"error"`
 }
 
@@ -1462,7 +1483,7 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re
 	}
 
 	// Check transforms if present
-	if err := r.checkTransforms(ctx, config, r.pkgManifest, scenario.kibanaDataStream, scenario.dataStream); err != nil {
+	if err := r.checkTransforms(ctx, config, r.pkgManifest, scenario.kibanaDataStream, scenario.dataStream, scenario.syntheticEnabled); err != nil {
 		results, _ := result.WithError(err)
 		return results, nil
 	}
@@ -1793,7 +1814,7 @@ func selectPolicyTemplateByName(policies []packages.PolicyTemplate, name string)
 	return packages.PolicyTemplate{}, fmt.Errorf("policy template %q not found", name)
 }
 
-func (r *tester) checkTransforms(ctx context.Context, config *testConfig, pkgManifest *packages.PackageManifest, ds kibana.PackageDataStream, dataStream string) error {
+func (r *tester) checkTransforms(ctx context.Context, config *testConfig, pkgManifest *packages.PackageManifest, ds kibana.PackageDataStream, dataStream string, syntheticEnabled bool) error {
 	transforms, err := packages.ReadTransformsFromPackageRoot(r.packageRootPath)
 	if err != nil {
 		return fmt.Errorf("loading transforms for package failed (root: %s): %w", r.packageRootPath, err)
@@ -1839,6 +1860,7 @@ func (r *tester) checkTransforms(ctx context.Context, config *testConfig, pkgMan
 			fields.WithSpecVersion(pkgManifest.SpecVersion),
 			fields.WithNumericKeywordFields(config.NumericKeywordFields),
 			fields.WithEnabledImportAllECSSChema(true),
+			fields.WithDisableNormalization(syntheticEnabled),
 		)
 		if err != nil {
 			return fmt.Errorf("creating fields validator for data stream failed (path: %s): %w", transformRootPath, err)

scripts/stack_parameters.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+stack_version_args() {
+  if [[ -z "${PACKAGE_UNDER_TEST:-""}" ]]; then
+    # Don't force stack version if we are testing multiple packages.
+    return
+  fi
+
+  local package_root=test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST}/
+  local stack_version_file="${package_root%/}.stack_version"
+  if [[ ! -f "$stack_version_file" ]]; then
+    return
+  fi
+
+  echo -n "--version $(cat "$stack_version_file")"
+}
+
+stack_provider_args() {
+  if [[ -z "${PACKAGE_UNDER_TEST:-""}" ]]; then
+    # Don't force stack version if we are testing multiple packages.
+    return
+  fi
+
+  local package_root=test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST}/
+  local parameters="${package_root%/}.stack_provider_settings"
+  if [[ ! -f "$parameters" ]]; then
+    return
+  fi
+
+  echo -n "-U $(cat "${parameters}" | tr '\n' ',' | tr -d ' ' | sed 's/,$//')"
+}

scripts/test-check-false-positives.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+source "${SCRIPT_DIR}/stack_parameters.sh"
+
 set -euxo pipefail
 
 function cleanup() {
@@ -76,31 +80,20 @@ function check_build_output() {
   )
 }
 
-function stack_version_args() {
-  if [[ -z "$PACKAGE_UNDER_TEST" ]]; then
-    # Don't force stack version if we are testing multiple packages.
-    return
-  fi
-
-  local package_root=test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST}/
-  local stack_version_file="${package_root%/}.stack_version"
-  if [[ ! -f $stack_version_file ]]; then
-    return
-  fi
-
-  echo -n "--version $(cat $stack_version_file)"
-}
-
 trap cleanup EXIT
 
 ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml"
 export ELASTIC_PACKAGE_LINKS_FILE_PATH
 
-stack_args=$(stack_version_args)
+stack_args=$(stack_version_args) # --version <version>
 
 # Update the stack
 elastic-package stack update -v ${stack_args}
 
+# NOTE: if any provider argument is defined, the stack must be shutdown first to ensure
+# that all parameters are taken into account by the services
+stack_args="${stack_args} $(stack_provider_args)" # -U <setting=1,settings=2>
+
 # Boot up the stack
 elastic-package stack up -d -v ${stack_args}
 

scripts/test-check-packages.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+source "${SCRIPT_DIR}/stack_parameters.sh"
+
 set -euxo pipefail
 
 cleanup() {
@@ -69,11 +73,17 @@ fi
 # started to test all packages. In our CI, this Elastic serverless stack is started 
 # at the beginning of the pipeline and must be running for all packages.
 if [[ "${SERVERLESS}" != "true" ]]; then
+  stack_args=$(stack_version_args) # --version <version>
+
   # Update the stack
-  elastic-package stack update -v
+  elastic-package stack update -v ${stack_args}
+
+  # NOTE: if any provider argument is defined, the stack must be shutdown first to ensure
+  # that all parameters are taken into account by the services
+  stack_args="${stack_args} $(stack_provider_args)" # -U <setting=1,settings=2>
 
   # Boot up the stack
-  elastic-package stack up -d -v
+  elastic-package stack up -d -v ${stack_args}
 
   elastic-package stack status
 fi

test/packages/parallel/ti_anomali/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@8.7
+    reference: "git@v8.11.0"

test/packages/parallel/ti_anomali/_dev/build/docs/README.md

@@ -17,13 +17,23 @@ explanation on how to configure the Anomali ThreatStream to send indicator
 to this integration.
 
 ### Expiration of Indicators of Compromise (IOCs)
-The ingested IOCs expire after certain duration. An [Elastic Transform](https://www.elastic.co/guide/en/elasticsearch/reference/current/transforms.html) is created to faciliate only active IOCs be available to the end users. This transform creates a destination index named `logs-ti_anomali_latest.threatstream` which only contains active and unexpired IOCs. When setting up indicator match rules, use this latest destination index to avoid false positives from expired IOCs. Please read [ILM Policy](#ilm-policy) below which is added to avoid unbounded growth on source `.ds-logs-ti_anomali.threatstream-*` indices.
+The ingested IOCs expire after certain duration. An [Elastic Transform](https://www.elastic.co/guide/en/elasticsearch/reference/current/transforms.html) is created to faciliate only active IOCs be available to the end users. This transform creates a destination index named `logs-ti_anomali_latest.threatstream-2` which only contains active and unexpired IOCs. The destination index also has an alias `logs-ti_anomali_latest.threatstream`. When setting up indicator match rules, use this latest destination index to avoid false positives from expired IOCs. Please read [ILM Policy](#ilm-policy) below which is added to avoid unbounded growth on source `.ds-logs-ti_anomali.threatstream-*` indices.
 
 #### Handling Orphaned IOCs
 When an IOC expires, Anomali feed contains information about all IOCs that got `deleted`. However, some Anomali IOCs may never expire and will continue to stay in the latest destination index `logs-ti_anomali_latest.threatstream`. To avoid any false positives from such orphaned IOCs, users are allowed to configure `IOC Expiration Duration` parameter while setting up the integration. This parameter deletes all data inside the destination index `logs-ti_anomali_latest.threatstream` after this specified duration is reached. Users must pull entire feed instead of incremental feed when this expiration happens so that the IOCs get reset. 
 
 **NOTE:** `IOC Expiration Duration` parameter does not override the expiration provided by the Anomali for their IOCs. So, if Anomali IOC is expired and subsequently such `deleted` IOCs are sent into the feed, they are deleted immediately. `IOC Expiration Duration` parameter only exists to add a fail-safe default expiration in case Anomali IOCs never expire.
 
+### Destination index versioning and deleting older versions
+The destination indices created by the transform are versioned with an integer suffix such as `-1`, `-2`. Example index name - `logs-ti_anomali_latest.threatstream-1`. 
+
+Due to schema changes on destination index, the versioning on it could be bumped. For example, in integration version `1.15.1`, the destination index  is changed to `logs-ti_anomali_latest.threatstream-2` from `logs-ti_anomali_latest.threatstream-1`. 
+
+Since the transform does not have the functionality to auto-delete the old index, users must to delete this old index manually. This is to ensure duplicates are not present when using wildcard queries such as `logs-ti_anomali_latest.threatstream-*`. Please follow below steps:
+1. After upgrading the integration to latest, check the current transform's destination index version by navigating via: `Stack Management -> Transforms -> logs-ti_anomali.latest_ioc-default -> Details`. Check `destination_index` value.
+2. Run `GET _cat/indices?v` and check if any older versions exist. Such as `logs-ti_anomali_latest.threatstream-1`
+3. Run `DELETE logs-ti_anomali_latest.threatstream-<OLDVERSION>` to delete the old index.
+
 ### ILM Policy
 To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_anomali.threat-*` are allowed to contain duplicates from each polling interval. ILM policy is added to these source indices so it doesn't lead to unbounded growth. This means data in these source indices will be deleted after `5 days` from ingested date. 
 

test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2.3'
 services:
   limo-http:
-    image: docker.elastic.co/observability/stream:v0.6.1
+    image: docker.elastic.co/observability/stream:v0.15.0
     ports:
       - 8080
     volumes:
@@ -13,15 +13,15 @@ services:
       - --addr=:8080
       - --config=/files/config.yml
   threatstream-webhook-http:
-    image: docker.elastic.co/observability/stream:v0.6.1
+    image: docker.elastic.co/observability/stream:v0.15.0
     volumes:
       - ./sample_logs:/sample_logs:ro
     environment:
       - STREAM_PROTOCOL=webhook
       - STREAM_ADDR=http://elastic-agent:9080/
     command: log --webhook-content-type application/x-ndjson --start-signal=SIGHUP --delay=5s /sample_logs/test-threatstream-ndjson.log
   threatstream-webhook-https:
-    image: docker.elastic.co/observability/stream:v0.6.1
+    image: docker.elastic.co/observability/stream:v0.15.0
     volumes:
       - ./sample_logs:/sample_logs:ro
     environment:

test/packages/parallel/ti_anomali/changelog.yml

@@ -1,4 +1,92 @@
 # newer versions go on top
+- version: "1.22.1"
+  changes:
+    - description: Fix ECS date mapping on threat fields.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10674
+- version: "1.22.0"
+  changes:
+    - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10135
+- version: "1.21.0"
+  changes:
+    - description: Add destination index alias and fix docs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9618
+- version: "1.20.0"
+  changes:
+    - description: Set sensitive values as secret.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9129
+- version: "1.19.2"
+  changes:
+    - description: Changed owners
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8943
+- version: 1.19.1
+  changes:
+    - description: Fix IOC expiration duration character casting.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8786
+- version: 1.19.0
+  changes:
+    - description: ECS version updated to 8.11.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8436
+- version: 1.18.0
+  changes:
+    - description: ECS version updated to 8.10.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7920
+- version: "1.17.0"
+  changes:
+    - description: Add DLM policy. Add owner.type to package manifest. Update format_version to 3.0.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7849
+    - description: Add tags.yml file so that integration's dashboards and saved searches are tagged with "Security Solution" and displayed in the Security Solution UI.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7789
+- version: "1.16.1"
+  changes:
+    - description: Remove dotted YAML keys.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/7803
+- version: "1.16.0"
+  changes:
+    - description: Update package-spec to 2.10.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7613
+- version: "1.15.1"
+  changes:
+    - description: Fix destination fields mapping. Note - Since transform's destination index version is changed, users must manually delete the old index. See https://docs.elastic.co/integrations/ti_anomali#logs for details on destination index versioning.
+      type: bugfix
+      link: https://github.com/elastic/integrations/issues/6514
+- version: "1.15.0"
+  changes:
+    - description: Update package to ECS 8.9.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7111
+- version: "1.14.1"
+  changes:
+    - description: Change sort order field in latest transform
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/7000
+- version: "1.14.0"
+  changes:
+    - description: Ensure event.kind is correctly set for pipeline errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6630
+- version: "1.13.0"
+  changes:
+    - description: Update package to ECS 8.8.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6328
+- version: "1.12.0"
+  changes:
+    - description: Add IOC field to transform source to easily filter destination indices with unexpired indicators
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/6344
 - version: "1.11.0"
   changes:
     - description: Support for IoC Expiration

test/packages/parallel/ti_anomali/data_stream/threatstream/_dev/test/pipeline/test-anomali-threatstream.json

@@ -1,5 +1,57 @@
 {
     "events": [
+        {
+            "json": {
+                "added_at": "2023-06-05T12:35:14.860741",
+                "classification": "private",
+                "confidence": 92,
+                "country": "US",
+                "date_first": "2020-10-09T18:45:23.000Z",
+                "date_last": "2020-10-09T18:45:23.000Z",
+                "detail": "phish-kit-sig-id-43111996,Microsoft",
+                "detail2": "imported by user 1",
+                "id": 185029228,
+                "itype": "phish_url",
+                "lat": 37.751,
+                "maltype": "phish-kit-sig-id-43111996",
+                "resource_uri": "/api/v1/intelligence/22222/",
+                "severity": "very-high",
+                "source": "Default Organization",
+                "source_feed_id": 0,
+                "srcip": "81.2.69.142",
+                "state": "active",
+                "trusted_circle_ids": ",10015,",
+                "update_id": 376590230,
+                "url": "https://example.appspot.com/2/https%40securelogin-example.bp.poste.it",
+                "value_type": "url"
+            }
+        },
+        {
+            "json": {
+                "added_at": "2023-06-05T11:03:34.648208",
+                "classification": "private",
+                "confidence": 94,
+                "country": "IN",
+                "date_first": "2020-10-09T18:45:28.000Z",
+                "date_last": "2020-10-09T18:45:28.000Z",
+                "detail": "32-bit,date_added=2020-10-09T15:44:05,elf,mips",
+                "detail2": "imported by user 1",
+                "id": 111111,
+                "itype": "mal_url",
+                "lat": 28.6327,
+                "maltype": "32-bit",
+                "resource_uri": "/api/v1/intelligence/111111/",
+                "severity": "very-high",
+                "source": "Default Organization",
+                "source_feed_id": 0,
+                "srcip": "81.2.69.192",
+                "state": "active",
+                "trusted_circle_ids": ",10015,",
+                "update_id": 376590649,
+                "url": "http://81.2.69.192:34011/bin.sh",
+                "value_type": "url"
+            }
+        },
         {
             "json": {
                 "added_at": "2020-10-08T12:22:11",