Update readme, update system tests, changes version to beta instead o…

…f ga
elastic · Oct 5, 2021 · 5c2912c · 5c2912c
1 parent 0aceb80
commit 5c2912c
Show file tree

Hide file tree

Showing 10 changed files with 127 additions and 55 deletions.
diff --git a/packages/microsoft_dhcp/_dev/build/docs/README.md b/packages/microsoft_dhcp/_dev/build/docs/README.md
@@ -4,7 +4,7 @@ This integration collects logs and metrics from Microsoft DHCP logs.
 
 ## Compatibility
 
-This integration has been made to support the DHCP log format Windows Server 2008 and later.
+This integration has been made to support the DHCP log format from Windows Server 2008 and later.
 
 ### Logs
 

diff --git a/packages/microsoft_dhcp/_dev/deploy/docker/docker-compose.yml b/packages/microsoft_dhcp/_dev/deploy/docker/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.3"
+version: "3.0"
 services:
  dhcp-logfile:
  image: alpine

diff --git a/packages/microsoft_dhcp/changelog.yml b/packages/microsoft_dhcp/changelog.yml
@@ -1,5 +1,5 @@
 # newer versions go on top
-- version: "1.0.0"
+- version: "0.1.0"
  changes:
  - description: Initial release
  type: enhancement

diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json
@@ -6,7 +6,7 @@
  "version": "1.12.0"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559563100Z",
+ "ingested": "2021-10-05T12:22:29.761168700Z",
  "original": "01,04/19/20,13:11:13,Stopped,,,",
  "code": "01",
  "kind": "event",
@@ -31,7 +31,7 @@
  "version": "1.12.0"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559588400Z",
+ "ingested": "2021-10-05T12:22:29.761216300Z",
  "original": "00,04/19/20,12:43:06,Started,,,",
  "code": "00",
  "kind": "event",
@@ -60,7 +60,7 @@
  "domain": "057182593757.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559595700Z",
+ "ingested": "2021-10-05T12:22:29.761225300Z",
  "original": "30,09/20/21,09:16:15,DNS Update Request,172.28.43.169,057182593757.test.com,,,0,6,,,,,,,,,0",
  "code": "30",
  "kind": "event",
@@ -95,7 +95,7 @@
  "domain": "1-07.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559601400Z",
+ "ingested": "2021-10-05T12:22:29.761231800Z",
  "original": "30,09/20/21,09:16:09,DNS Update Request,172.28.53.173,1-07.test.com,,,0,6,,,,,,,,,0",
  "code": "30",
  "kind": "event",
@@ -130,7 +130,7 @@
  "domain": "3-07.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559606600Z",
+ "ingested": "2021-10-05T12:22:29.761237600Z",
  "original": "32,09/20/21,09:16:03,DNS Update Successful,172.28.53.36,3-07.test.com,,,0,6,,,,,,,,,0",
  "code": "32",
  "kind": "event",
@@ -165,7 +165,7 @@
  "ip": "172.28.52.0"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559611700Z",
+ "ingested": "2021-10-05T12:22:29.761243Z",
  "original": "36,09/20/21,09:18:01,Packet dropped because of Client ID hash mismatch or standby server.,172.28.52.0,,76691ED45C90,,0,6,,,,,,,,,0",
  "code": "36",
  "kind": "event",
@@ -200,7 +200,7 @@
  "domain": "035856103966.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559617700Z",
+ "ingested": "2021-10-05T12:22:29.761249200Z",
  "original": "31,09/20/21,09:18:00,DNS Update Failed,172.28.43.159,035856103966.test.com,,,0,6,,,,,,,,,10054",
  "code": "31",
  "kind": "event",
@@ -235,7 +235,7 @@
  "domain": "001100581357.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559622900Z",
+ "ingested": "2021-10-05T12:22:29.761254700Z",
  "original": "31,09/20/21,09:18:01,DNS Update Failed,172.28.40.35,001100581357.test.com,,,0,6,,,,,,,,,10054",
  "code": "31",
  "kind": "event",
@@ -271,7 +271,7 @@
  "domain": "host.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559627800Z",
+ "ingested": "2021-10-05T12:22:29.761260100Z",
  "original": "35,01/01/01,01:01:01,DNS update request failed,192.0.2.1,host.test.com,000000000000,",
  "code": "35",
  "kind": "event",
@@ -300,7 +300,7 @@
  "domain": "host.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559632500Z",
+ "ingested": "2021-10-05T12:22:29.761265100Z",
  "original": "10,01/01/01,01:01:01,Assign,192.0.2.10,host.test.com,000000000000,,17739,0,,,",
  "code": "10",
  "kind": "event",
@@ -336,7 +336,7 @@
  "domain": "host.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559639Z",
+ "ingested": "2021-10-05T12:22:29.761271200Z",
  "original": "10,01/01/01,01:01:01,Assign,192.0.2.20,host.test.com,000000000000,,3096562285,0,,,,0x4D53465420352E30,MSFT 5.0,,,,0",
  "code": "10",
  "kind": "event",
@@ -372,7 +372,7 @@
  "version": "1.12.0"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559644400Z",
+ "ingested": "2021-10-05T12:22:29.761277400Z",
  "original": "24,11/20/20,00:00:05,Database Cleanup Begin,,,,,0,6,,,,,,,,,0",
  "code": "24",
  "kind": "event",
@@ -407,7 +407,7 @@
  "domain": "hostname.test.com"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559649600Z",
+ "ingested": "2021-10-05T12:22:29.761283200Z",
  "original": "30,11/20/20,00:00:05,DNS Update Request,10.10.10.10,hostname.test.com,,,0,6,,,,,,,,,0",
  "code": "30",
  "kind": "event",
@@ -441,7 +441,7 @@
  "ip": "8.8.8.8"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559654600Z",
+ "ingested": "2021-10-05T12:22:29.761288300Z",
  "original": "17,11/20/20,00:00:05,DNS record not deleted,8.8.8.8,,,,0,6,,,,,,,,,0",
  "code": "17",
  "kind": "event",
@@ -475,7 +475,7 @@
  "domain": "domain.local"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559659500Z",
+ "ingested": "2021-10-05T12:22:29.761293300Z",
  "original": "55,04/19/20,12:43:54,Authorized(servicing),,domain.local,",
  "code": "55",
  "kind": "event",
@@ -502,7 +502,7 @@
  "domain": "domain.local"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559664200Z",
+ "ingested": "2021-10-05T12:22:29.761298600Z",
  "original": "60,04/19/20,12:43:21,No DC is DS Enabled,,domain.local,",
  "code": "60",
  "kind": "event",
@@ -528,7 +528,7 @@
  "version": "1.12.0"
  },
  "event": {
- "ingested": "2021-09-29T15:45:12.559669300Z",
+ "ingested": "2021-10-05T12:22:29.761303800Z",
  "original": "63,04/19/20,12:43:28,Restarting rogue detection,,,",
  "code": "63",
  "kind": "event",

diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/system/test-default-config.yml b/packages/microsoft_dhcp/data_stream/log/_dev/test/system/test-default-config.yml
@@ -2,6 +2,7 @@ service: dhcp-logfile
 input: logfile
 data_stream:
  vars:
+ tz_offset: America/New_York
  preserve_original_event: true
  paths:
- - "{{SERVICE_LOGS_DIR}}/log*.json"
+ - "{{SERVICE_LOGS_DIR}}/*.log"
diff --git a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -50,6 +50,10 @@ processors:
  formats:
  - "MM/dd/yy HH:mm:ss"
  timezone: "{{event.timezone}}"
+ on_failure:
+ - append:
+ field: error.message
+ value: "date processor failed to convert the timestamp"
  - convert:
  field: event.code
  target_field: _tmp_.code
@@ -130,6 +134,6 @@ processors:
  - _conf
  ignore_missing: true
 on_failure:
- - set:
+ - append:
  field: error.message
  value: "{{ _ingest.on_failure_message }}"
diff --git a/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml b/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml
@@ -24,3 +24,5 @@
  external: ecs
 - name: user.name
  external: ecs
+- name: log.file.path
+ external: ecs
diff --git a/packages/microsoft_dhcp/data_stream/log/sample_event.json b/packages/microsoft_dhcp/data_stream/log/sample_event.json
@@ -1,28 +1,60 @@
 {
- "@timestamp": "2021-07-09T17:20:27.182Z",
+ "@timestamp": "2001-01-01T01:01:01.000-05:00",
+ "agent": {
+ "ephemeral_id": "7b80c5f6-3f5b-436f-aab7-ad35bc17cde9",
+ "hostname": "docker-fleet-agent",
+ "id": "303093f0-28ce-40db-ad0f-05f02e31b666",
+ "name": "docker-fleet-agent",
+ "type": "filebeat",
+ "version": "7.15.0"
+ },
+ "data_stream": {
+ "dataset": "microsoft_dhcp.log",
+ "namespace": "ep",
+ "type": "logs"
+ },
  "ecs": {
  "version": "1.12.0"
  },
+ "elastic_agent": {
+ "id": "303093f0-28ce-40db-ad0f-05f02e31b666",
+ "snapshot": true,
+ "version": "7.15.0"
+ },
  "event": {
- "ingested": "2021-07-22T19:26:33.689669663Z",
+ "agent_id_status": "verified",
+ "category": [
+ "network"
+ ],
+ "code": "35",
+ "dataset": "microsoft_dhcp.log",
+ "ingested": "2021-10-05T12:12:13Z",
  "kind": "event",
- "original": "{\"@level\":\"info\",\"@message\":\"starting listener\",\"@module\":\"core.cluster-listener.tcp\",\"@timestamp\":\"2021-07-09T17:20:27.182327Z\",\"listener_address\":{\"IP\":\"0.0.0.0\",\"Port\":8201,\"Zone\":\"\"}}"
+ "original": "35,01/01/01,01:01:01,DNS update request failed,192.0.2.1,host.test.com,000000000000,",
+ "outcome": "success",
+ "timezone": "America/New_York",
+ "type": [
+ "connection"
+ ]
+ },
+ "host": {
+ "domain": "host.test.com",
+ "ip": "192.0.2.1",
+ "mac": "00-00-00-00-00-00"
  },
- "hashicorp_vault": {
- "log": {
- "listener_address": {
- "IP": "0.0.0.0",
- "Port": 8201,
- "Zone": ""
- }
- }
+ "input": {
+ "type": "log"
  },
  "log": {
- "level": "info",
- "logger": "core.cluster-listener.tcp"
+ "file": {
+ "path": "/tmp/service_logs/test-dhcp.log"
+ },
+ "offset": 646
  },
- "message": "starting listener",
+ "message": "DNS update request failed",
  "tags": [
- "preserve_original_event"
+ "preserve_original_event",
+ "forwarded",
+ "microsoft_dhcp"
  ]
 }
diff --git a/packages/microsoft_dhcp/docs/README.md b/packages/microsoft_dhcp/docs/README.md
@@ -4,7 +4,7 @@ This integration collects logs and metrics from Microsoft DHCP logs.
 
 ## Compatibility
 
-This integration has been made to support the DHCP log format Windows Server 2008 and later.
+This integration has been made to support the DHCP log format from Windows Server 2008 and later.
 
 ### Logs
 
@@ -17,31 +17,63 @@ An example event for `log` looks as following:
 
 ```json
 {
- "@timestamp": "2021-07-09T17:20:27.182Z",
+ "@timestamp": "2001-01-01T01:01:01.000-05:00",
+ "agent": {
+ "ephemeral_id": "7b80c5f6-3f5b-436f-aab7-ad35bc17cde9",
+ "hostname": "docker-fleet-agent",
+ "id": "303093f0-28ce-40db-ad0f-05f02e31b666",
+ "name": "docker-fleet-agent",
+ "type": "filebeat",
+ "version": "7.15.0"
+ },
+ "data_stream": {
+ "dataset": "microsoft_dhcp.log",
+ "namespace": "ep",
+ "type": "logs"
+ },
  "ecs": {
  "version": "1.12.0"
  },
+ "elastic_agent": {
+ "id": "303093f0-28ce-40db-ad0f-05f02e31b666",
+ "snapshot": true,
+ "version": "7.15.0"
+ },
  "event": {
- "ingested": "2021-07-22T19:26:33.689669663Z",
+ "agent_id_status": "verified",
+ "category": [
+ "network"
+ ],
+ "code": "35",
+ "dataset": "microsoft_dhcp.log",
+ "ingested": "2021-10-05T12:12:13Z",
  "kind": "event",
- "original": "{\"@level\":\"info\",\"@message\":\"starting listener\",\"@module\":\"core.cluster-listener.tcp\",\"@timestamp\":\"2021-07-09T17:20:27.182327Z\",\"listener_address\":{\"IP\":\"0.0.0.0\",\"Port\":8201,\"Zone\":\"\"}}"
+ "original": "35,01/01/01,01:01:01,DNS update request failed,192.0.2.1,host.test.com,000000000000,",
+ "outcome": "success",
+ "timezone": "America/New_York",
+ "type": [
+ "connection"
+ ]
+ },
+ "host": {
+ "domain": "host.test.com",
+ "ip": "192.0.2.1",
+ "mac": "00-00-00-00-00-00"
  },
- "hashicorp_vault": {
- "log": {
- "listener_address": {
- "IP": "0.0.0.0",
- "Port": 8201,
- "Zone": ""
- }
- }
+ "input": {
+ "type": "log"
  },
  "log": {
- "level": "info",
- "logger": "core.cluster-listener.tcp"
+ "file": {
+ "path": "/tmp/service_logs/test-dhcp.log"
+ },
+ "offset": 646
  },
- "message": "starting listener",
+ "message": "DNS update request failed",
  "tags": [
- "preserve_original_event"
+ "preserve_original_event",
+ "forwarded",
+ "microsoft_dhcp"
  ]
 }
 ```
@@ -67,6 +99,7 @@ An example event for `log` looks as following:
 | host.ip | Host ip addresses. | ip |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
 | input.type | | keyword |
+| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword |
 | log.offset | | long |
 | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text |
 | microsoft.dhcp.correlation_id | The NAP correlation ID related to the client/server transaction. | keyword |

diff --git a/packages/microsoft_dhcp/manifest.yml b/packages/microsoft_dhcp/manifest.yml
@@ -1,13 +1,13 @@
 format_version: 1.0.0
 name: microsoft_dhcp
 title: Microsoft DHCP
-version: 1.0.0
+version: 0.1.0
 license: basic
 description: "Collect logs from Microsoft DHCP."
 type: integration
 categories:
  - network
-release: ga
+release: beta
 conditions:
  kibana.version: "^7.14.0"
 icons: