elastic · fearful-symmetry · May 10, 2022 · May 10, 2022
diff --git a/packages/system/1.11.0/changelog.yml b/packages/system/1.11.0/changelog.yml
@@ -0,0 +1,246 @@
+# newer versions go on top
+- version: "1.11.0"
+  changes:
+    - description: Add option to configure ignored filesystem types
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2679
+- version: "1.10.0"
+  changes:
+    - description: Expose winlog input ignore_older option.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2542
+    - description: Fix preserve original event option
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2542
+    - description: Make order of Security, Application, System options consistent with other winlog based integrations.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2542
+- version: "1.9.0"
+  changes:
+    - description: Update to ECS 8.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2512
+- version: "1.8.0"
+  changes:
+    - description: Add routing pipeline to security data_stream, limit to specific providers.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2523
+- version: "1.7.0"
+  changes:
+    - description: Expose winlog input language option.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2344
+- version: "1.6.6"
+  changes:
+    - description: Regenerate test files using the new GeoIP database
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2339
+- version: "1.6.5"
+  changes:
+    - description: Change test public IPs to the supported subset
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2327
+- version: "1.6.4"
+  changes:
+    - description: More consistent use of Proc Filesystem Directory settings
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2201
+    - description: Support Kibana 8
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2201
+- version: "1.6.3"
+  changes:
+    - description: Fix AccessList and AccessMask processing in security data_stream
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2156
+- version: "1.6.2"
+  changes:
+    - description: Fix missing null check in security pipeline
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2148
+- version: "1.6.1"
+  changes:
+    - description: Uniform with guidelines
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2082
+- version: "1.6.0"
+  changes:
+    - description: Consistently map message field in Windows integrations.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2008
+- version: "1.5.0"
+  changes:
+    - description: Better user mappings for security events
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1944
+- version: "1.4.2"
+  changes:
+    - description: Prevent pipeline script error
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1869
+- version: "1.4.1"
+  changes:
+    - description: Fix logic that checks for the 'forwarded' tag
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1855
+- version: "1.4.0"
+  changes:
+    - description: Update to ECS 1.12.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1709
+- version: "1.3.0"
+  changes:
+    - description: Add custom processors and event_id to Application, Security & System data_streams
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1548
+- version: "1.2.1"
+  changes:
+    - description: Convert to generated ECS fields
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1508
+- version: "1.2.0"
+  changes:
+    - description: Update fields to include new cgroups fields
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1539
+- version: "1.1.5"
+  changes:
+    - description: Fix Windows links
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1525
+- version: "1.1.4"
+  changes:
+    - description: Fix issue with normalized CPU gauge
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1458
+- version: "1.1.3"
+  changes:
+    - description: update to ECS 1.11.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1429
+- version: "1.1.2"
+  changes:
+    - description: Mark integration as GA
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1435
+- version: "1.1.1"
+  changes:
+    - description: Escape special characters in docs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1405
+- version: "1.1.0"
+  changes:
+    - description: Update integration description
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1364
+- version: "1.0.1"
+  changes:
+    - description: Move visualizations to cpu.norm.pct
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1358
+- version: "1.0.0"
+  changes:
+    - description: GA the system module
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1282
+- version: "0.13.6"
+  changes:
+    - description: Use event.dataset and event.module
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1211
+- version: "0.13.5"
+  changes:
+    - description: Add support for Splunk authorization tokens
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1147
+- version: "0.13.4"
+  changes:
+    - description: Use `wildcard` type for relevant ECS fields in `security` stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1185
+- version: "0.13.3"
+  changes:
+    - description: Fix unneeded unit and metric type for field groups
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1114
+- version: "0.13.2"
+  changes:
+    - description: Fix security pipeline to support string event.code.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1089
+- version: "0.13.1"
+  changes:
+    - description: Add system tests for security data_stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1069
+- version: "0.13.0"
+  changes:
+    - description: Render units and metric types in exported fields table
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1028
+- version: "0.12.7"
+  changes:
+    - description: Fix security pipeline to support string event.code for 7.13.
+      type: bugfix
+      link: https://github.com/elastic/package-storage/pull/1372
+- version: "0.12.6"
+  changes:
+    - description: Report system_summary properly.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/778
+- version: "0.12.5"
+  changes:
+    - description: Make event.original optional for application, security, and system data streams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/990
+- version: "0.12.4"
+  changes:
+    - description: Fix inconsistent dashboard IDs
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/987
+- version: "0.12.3"
+  changes:
+    - description: Remove edge processing for httpjson input.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/969
+- version: "0.12.2"
+  changes:
+    - description: Add event.code mappings
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/932
+- version: "0.12.1"
+  changes:
+    - description: Convert Security processing to Ingest Node
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/917
+    - description: Change Splunk input to use the decode_xml_wineventlog processor.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/924
+- version: "0.12.0"
+  changes:
+    - description: Add Splunk input for application, system, and security data streams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/890
+- version: "0.11.3"
+  changes:
+    - description: Updating package owner
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/766
+    - description: update to ECS 1.9.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/874
+- version: "0.11.2"
+  changes:
+    - description: Update security data stream
+      type: bugfix # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/728
+- version: "0.11.1" # unreleased
+  changes:
+    - description: remove duplicate ingest pipeline for syslog data stream
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/725
+- version: "0.0.3"
+  changes:
+    - description: initial release
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/8
diff --git a/packages/system/1.11.0/data_stream/application/agent/stream/httpjson.yml.hbs b/packages/system/1.11.0/data_stream/application/agent/stream/httpjson.yml.hbs
@@ -0,0 +1,104 @@
+config_version: "2"
+interval: {{interval}}
+{{#unless token}}
+{{#if username}}
+{{#if password}}
+auth.basic.user: {{username}}
+auth.basic.password: {{password}}
+{{/if}}
+{{/if}}
+{{/unless}}
+cursor:
+  index_earliest:
+    value: '[[.last_event.result.max_indextime]]'
+request.url: {{url}}/services/search/jobs/export
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.transforms:
+  - set:
+      target: url.params.search
+      value: |-
+        {{search}} | streamstats max(_indextime) AS max_indextime
+  - set:
+      target: url.params.output_mode
+      value: "json"
+  - set:
+      target: url.params.index_earliest
+      value: '[[ .cursor.index_earliest ]]'
+      default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
+  - set:
+      target: url.params.index_latest
+      value: '[[(now).Unix]]'
+  - set:
+      target: header.Content-Type
+      value: application/x-www-form-urlencoded
+{{#unless username}}
+{{#unless password}}
+{{#if token}}
+  - set:
+      target: header.Authorization
+      value: {{token}}
+{{/if}}
+{{/unless}}
+{{/unless}}
+response.decode_as: application/x-ndjson
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - decode_json_fields:
+      fields: message
+      target: json
+      add_error_key: true
+  - drop_event:
+      when:
+        not:
+          has_fields: ['json.result']
+  - fingerprint:
+      fields:
+        - json.result._cd
+        - json.result._indextime
+        - json.result._raw
+        - json.result._time
+        - json.result.host
+        - json.result.source
+      target_field: "@metadata._id"
+  - drop_fields:
+      fields: message
+  - rename:
+      fields:
+        - from: json.result._raw
+          to: event.original
+        - from: json.result.host
+          to: host.name
+        - from: json.result.source
+          to: event.provider
+      ignore_missing: true
+      fail_on_error: false
+  - drop_fields:
+      fields: json
+  - decode_xml_wineventlog:
+      field: event.original
+      target_field: winlog
+      ignore_missing: true
+      ignore_failure: true
+      map_ecs_fields: true
+  - timestamp:
+      field: winlog.time_created
+      layouts:
+        - '2006-01-02T15:04:05Z'
+        - '2006-01-02T15:04:05.999Z'
+        - '2006-01-02T15:04:05.999-07:00'
+      test:
+        - '2019-06-22T16:33:51Z'
+        - '2019-11-18T04:59:51.123Z'
+        - '2020-08-03T07:10:20.123456+02:00'
diff --git a/packages/system/1.11.0/data_stream/application/agent/stream/winlog.yml.hbs b/packages/system/1.11.0/data_stream/application/agent/stream/winlog.yml.hbs
@@ -0,0 +1,24 @@
+name: Application
+condition: ${host.platform} == 'windows'
+{{#if event_id}}
+event_id: {{event_id}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+{{#if language}}
+language: {{language}}
+{{/if}}
+{{#if preserve_original_event}}
+include_xml: true
+{{/if}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{/if}}
diff --git a/packages/system/1.11.0/data_stream/application/elasticsearch/ingest_pipeline/default.yml b/packages/system/1.11.0/data_stream/application/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,13 @@
+---
+description: Pipeline for Windows Application Event Logs
+processors:
+  - set:
+      field: event.ingested
+      value: '{{_ingest.timestamp}}'
+  - set:
+      field: ecs.version
+      value: 8.0.0
+on_failure:
+  - set:
+      field: "error.message"
+      value: "{{ _ingest.on_failure_message }}"