diff --git a/packages/microsoft_dhcp/_dev/build/docs/README.md b/packages/microsoft_dhcp/_dev/build/docs/README.md index 3894c9464dfc..8e4b9e0f0a41 100644 --- a/packages/microsoft_dhcp/_dev/build/docs/README.md +++ b/packages/microsoft_dhcp/_dev/build/docs/README.md @@ -4,7 +4,7 @@ This integration collects logs and metrics from Microsoft DHCP logs. ## Compatibility -This integration has been made to support the DHCP log format Windows Server 2008 and later. +This integration has been made to support the DHCP log format from Windows Server 2008 and later. ### Logs diff --git a/packages/microsoft_dhcp/_dev/deploy/docker/docker-compose.yml b/packages/microsoft_dhcp/_dev/deploy/docker/docker-compose.yml index d6333780bb87..3ad0f78df9bd 100644 --- a/packages/microsoft_dhcp/_dev/deploy/docker/docker-compose.yml +++ b/packages/microsoft_dhcp/_dev/deploy/docker/docker-compose.yml @@ -1,4 +1,4 @@ -version: "2.3" +version: "3.0" services: dhcp-logfile: image: alpine diff --git a/packages/microsoft_dhcp/changelog.yml b/packages/microsoft_dhcp/changelog.yml index 3e446e0f5550..af4cf723ad16 100644 --- a/packages/microsoft_dhcp/changelog.yml +++ b/packages/microsoft_dhcp/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "1.0.0" +- version: "0.1.0" changes: - description: Initial release type: enhancement diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json index 798b3d71b19e..ccc4b71c2562 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json @@ -6,7 +6,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-09-29T15:45:12.559563100Z", + "ingested": "2021-10-05T12:22:29.761168700Z", "original": "01,04/19/20,13:11:13,Stopped,,,", "code": "01", "kind": "event", @@ -31,7 +31,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-09-29T15:45:12.559588400Z", + "ingested": "2021-10-05T12:22:29.761216300Z", "original": "00,04/19/20,12:43:06,Started,,,", "code": "00", "kind": "event", @@ -60,7 +60,7 @@ "domain": "057182593757.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559595700Z", + "ingested": "2021-10-05T12:22:29.761225300Z", "original": "30,09/20/21,09:16:15,DNS Update Request,172.28.43.169,057182593757.test.com,,,0,6,,,,,,,,,0", "code": "30", "kind": "event", @@ -95,7 +95,7 @@ "domain": "1-07.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559601400Z", + "ingested": "2021-10-05T12:22:29.761231800Z", "original": "30,09/20/21,09:16:09,DNS Update Request,172.28.53.173,1-07.test.com,,,0,6,,,,,,,,,0", "code": "30", "kind": "event", @@ -130,7 +130,7 @@ "domain": "3-07.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559606600Z", + "ingested": "2021-10-05T12:22:29.761237600Z", "original": "32,09/20/21,09:16:03,DNS Update Successful,172.28.53.36,3-07.test.com,,,0,6,,,,,,,,,0", "code": "32", "kind": "event", @@ -165,7 +165,7 @@ "ip": "172.28.52.0" }, "event": { - "ingested": "2021-09-29T15:45:12.559611700Z", + "ingested": "2021-10-05T12:22:29.761243Z", "original": "36,09/20/21,09:18:01,Packet dropped because of Client ID hash mismatch or standby server.,172.28.52.0,,76691ED45C90,,0,6,,,,,,,,,0", "code": "36", "kind": "event", @@ -200,7 +200,7 @@ "domain": "035856103966.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559617700Z", + "ingested": "2021-10-05T12:22:29.761249200Z", "original": "31,09/20/21,09:18:00,DNS Update Failed,172.28.43.159,035856103966.test.com,,,0,6,,,,,,,,,10054", "code": "31", "kind": "event", @@ -235,7 +235,7 @@ "domain": "001100581357.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559622900Z", + "ingested": "2021-10-05T12:22:29.761254700Z", "original": "31,09/20/21,09:18:01,DNS Update Failed,172.28.40.35,001100581357.test.com,,,0,6,,,,,,,,,10054", "code": "31", "kind": "event", @@ -271,7 +271,7 @@ "domain": "host.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559627800Z", + "ingested": "2021-10-05T12:22:29.761260100Z", "original": "35,01/01/01,01:01:01,DNS update request failed,192.0.2.1,host.test.com,000000000000,", "code": "35", "kind": "event", @@ -300,7 +300,7 @@ "domain": "host.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559632500Z", + "ingested": "2021-10-05T12:22:29.761265100Z", "original": "10,01/01/01,01:01:01,Assign,192.0.2.10,host.test.com,000000000000,,17739,0,,,", "code": "10", "kind": "event", @@ -336,7 +336,7 @@ "domain": "host.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559639Z", + "ingested": "2021-10-05T12:22:29.761271200Z", "original": "10,01/01/01,01:01:01,Assign,192.0.2.20,host.test.com,000000000000,,3096562285,0,,,,0x4D53465420352E30,MSFT 5.0,,,,0", "code": "10", "kind": "event", @@ -372,7 +372,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-09-29T15:45:12.559644400Z", + "ingested": "2021-10-05T12:22:29.761277400Z", "original": "24,11/20/20,00:00:05,Database Cleanup Begin,,,,,0,6,,,,,,,,,0", "code": "24", "kind": "event", @@ -407,7 +407,7 @@ "domain": "hostname.test.com" }, "event": { - "ingested": "2021-09-29T15:45:12.559649600Z", + "ingested": "2021-10-05T12:22:29.761283200Z", "original": "30,11/20/20,00:00:05,DNS Update Request,10.10.10.10,hostname.test.com,,,0,6,,,,,,,,,0", "code": "30", "kind": "event", @@ -441,7 +441,7 @@ "ip": "8.8.8.8" }, "event": { - "ingested": "2021-09-29T15:45:12.559654600Z", + "ingested": "2021-10-05T12:22:29.761288300Z", "original": "17,11/20/20,00:00:05,DNS record not deleted,8.8.8.8,,,,0,6,,,,,,,,,0", "code": "17", "kind": "event", @@ -475,7 +475,7 @@ "domain": "domain.local" }, "event": { - "ingested": "2021-09-29T15:45:12.559659500Z", + "ingested": "2021-10-05T12:22:29.761293300Z", "original": "55,04/19/20,12:43:54,Authorized(servicing),,domain.local,", "code": "55", "kind": "event", @@ -502,7 +502,7 @@ "domain": "domain.local" }, "event": { - "ingested": "2021-09-29T15:45:12.559664200Z", + "ingested": "2021-10-05T12:22:29.761298600Z", "original": "60,04/19/20,12:43:21,No DC is DS Enabled,,domain.local,", "code": "60", "kind": "event", @@ -528,7 +528,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-09-29T15:45:12.559669300Z", + "ingested": "2021-10-05T12:22:29.761303800Z", "original": "63,04/19/20,12:43:28,Restarting rogue detection,,,", "code": "63", "kind": "event", diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/system/test-default-config.yml b/packages/microsoft_dhcp/data_stream/log/_dev/test/system/test-default-config.yml index 253b9a334eaa..4182ffc8d963 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/system/test-default-config.yml +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/system/test-default-config.yml @@ -2,6 +2,7 @@ service: dhcp-logfile input: logfile data_stream: vars: + tz_offset: America/New_York preserve_original_event: true paths: - - "{{SERVICE_LOGS_DIR}}/log*.json" + - "{{SERVICE_LOGS_DIR}}/*.log" diff --git a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 80c9a6d40d78..61cc6dd60c6c 100644 --- a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -50,6 +50,10 @@ processors: formats: - "MM/dd/yy HH:mm:ss" timezone: "{{event.timezone}}" + on_failure: + - append: + field: error.message + value: "date processor failed to convert the timestamp" - convert: field: event.code target_field: _tmp_.code @@ -130,6 +134,6 @@ processors: - _conf ignore_missing: true on_failure: - - set: + - append: field: error.message value: "{{ _ingest.on_failure_message }}" diff --git a/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml b/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml index 59bfd977234d..869c1793b510 100644 --- a/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml +++ b/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml @@ -24,3 +24,5 @@ external: ecs - name: user.name external: ecs +- name: log.file.path + external: ecs diff --git a/packages/microsoft_dhcp/data_stream/log/sample_event.json b/packages/microsoft_dhcp/data_stream/log/sample_event.json index e1f561b7d366..24bafda194cb 100644 --- a/packages/microsoft_dhcp/data_stream/log/sample_event.json +++ b/packages/microsoft_dhcp/data_stream/log/sample_event.json @@ -1,28 +1,60 @@ { - "@timestamp": "2021-07-09T17:20:27.182Z", + "@timestamp": "2001-01-01T01:01:01.000-05:00", + "agent": { + "ephemeral_id": "7b80c5f6-3f5b-436f-aab7-ad35bc17cde9", + "hostname": "docker-fleet-agent", + "id": "303093f0-28ce-40db-ad0f-05f02e31b666", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.15.0" + }, + "data_stream": { + "dataset": "microsoft_dhcp.log", + "namespace": "ep", + "type": "logs" + }, "ecs": { "version": "1.12.0" }, + "elastic_agent": { + "id": "303093f0-28ce-40db-ad0f-05f02e31b666", + "snapshot": true, + "version": "7.15.0" + }, "event": { - "ingested": "2021-07-22T19:26:33.689669663Z", + "agent_id_status": "verified", + "category": [ + "network" + ], + "code": "35", + "dataset": "microsoft_dhcp.log", + "ingested": "2021-10-05T12:12:13Z", "kind": "event", - "original": "{\"@level\":\"info\",\"@message\":\"starting listener\",\"@module\":\"core.cluster-listener.tcp\",\"@timestamp\":\"2021-07-09T17:20:27.182327Z\",\"listener_address\":{\"IP\":\"0.0.0.0\",\"Port\":8201,\"Zone\":\"\"}}" + "original": "35,01/01/01,01:01:01,DNS update request failed,192.0.2.1,host.test.com,000000000000,", + "outcome": "success", + "timezone": "America/New_York", + "type": [ + "connection" + ] + }, + "host": { + "domain": "host.test.com", + "ip": "192.0.2.1", + "mac": "00-00-00-00-00-00" }, - "hashicorp_vault": { - "log": { - "listener_address": { - "IP": "0.0.0.0", - "Port": 8201, - "Zone": "" - } - } + "input": { + "type": "log" }, "log": { - "level": "info", - "logger": "core.cluster-listener.tcp" + "file": { + "path": "/tmp/service_logs/test-dhcp.log" + }, + "offset": 646 }, - "message": "starting listener", + "message": "DNS update request failed", "tags": [ - "preserve_original_event" + "preserve_original_event", + "forwarded", + "microsoft_dhcp" ] } \ No newline at end of file diff --git a/packages/microsoft_dhcp/docs/README.md b/packages/microsoft_dhcp/docs/README.md index 30db6ae76f83..3f21176c306c 100644 --- a/packages/microsoft_dhcp/docs/README.md +++ b/packages/microsoft_dhcp/docs/README.md @@ -4,7 +4,7 @@ This integration collects logs and metrics from Microsoft DHCP logs. ## Compatibility -This integration has been made to support the DHCP log format Windows Server 2008 and later. +This integration has been made to support the DHCP log format from Windows Server 2008 and later. ### Logs @@ -17,31 +17,63 @@ An example event for `log` looks as following: ```json { - "@timestamp": "2021-07-09T17:20:27.182Z", + "@timestamp": "2001-01-01T01:01:01.000-05:00", + "agent": { + "ephemeral_id": "7b80c5f6-3f5b-436f-aab7-ad35bc17cde9", + "hostname": "docker-fleet-agent", + "id": "303093f0-28ce-40db-ad0f-05f02e31b666", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.15.0" + }, + "data_stream": { + "dataset": "microsoft_dhcp.log", + "namespace": "ep", + "type": "logs" + }, "ecs": { "version": "1.12.0" }, + "elastic_agent": { + "id": "303093f0-28ce-40db-ad0f-05f02e31b666", + "snapshot": true, + "version": "7.15.0" + }, "event": { - "ingested": "2021-07-22T19:26:33.689669663Z", + "agent_id_status": "verified", + "category": [ + "network" + ], + "code": "35", + "dataset": "microsoft_dhcp.log", + "ingested": "2021-10-05T12:12:13Z", "kind": "event", - "original": "{\"@level\":\"info\",\"@message\":\"starting listener\",\"@module\":\"core.cluster-listener.tcp\",\"@timestamp\":\"2021-07-09T17:20:27.182327Z\",\"listener_address\":{\"IP\":\"0.0.0.0\",\"Port\":8201,\"Zone\":\"\"}}" + "original": "35,01/01/01,01:01:01,DNS update request failed,192.0.2.1,host.test.com,000000000000,", + "outcome": "success", + "timezone": "America/New_York", + "type": [ + "connection" + ] + }, + "host": { + "domain": "host.test.com", + "ip": "192.0.2.1", + "mac": "00-00-00-00-00-00" }, - "hashicorp_vault": { - "log": { - "listener_address": { - "IP": "0.0.0.0", - "Port": 8201, - "Zone": "" - } - } + "input": { + "type": "log" }, "log": { - "level": "info", - "logger": "core.cluster-listener.tcp" + "file": { + "path": "/tmp/service_logs/test-dhcp.log" + }, + "offset": 646 }, - "message": "starting listener", + "message": "DNS update request failed", "tags": [ - "preserve_original_event" + "preserve_original_event", + "forwarded", + "microsoft_dhcp" ] } ``` @@ -67,6 +99,7 @@ An example event for `log` looks as following: | host.ip | Host ip addresses. | ip | | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | input.type | | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | | long | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | microsoft.dhcp.correlation_id | The NAP correlation ID related to the client/server transaction. | keyword | diff --git a/packages/microsoft_dhcp/manifest.yml b/packages/microsoft_dhcp/manifest.yml index df2850043f77..3b468eb12011 100644 --- a/packages/microsoft_dhcp/manifest.yml +++ b/packages/microsoft_dhcp/manifest.yml @@ -1,13 +1,13 @@ format_version: 1.0.0 name: microsoft_dhcp title: Microsoft DHCP -version: 1.0.0 +version: 0.1.0 license: basic description: "Collect logs from Microsoft DHCP." type: integration categories: - network -release: ga +release: beta conditions: kibana.version: "^7.14.0" icons: