Skip to content

Commit

Permalink
[Filebeat][Gsuite] Transform all dates to timestamp with processor (e…
Browse files Browse the repository at this point in the history 
…lastic#20308)

* Transform all dates to timestamp with processor

* Change parse date function to create a chain
  • Loading branch information
@marc-gr @melchiormoulin
marc-gr authored and melchiormoulin committed Oct 14, 2020
1 parent ef23981 commit 9a2ca19
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 56 deletions.
90 changes: 57 additions & 33 deletions x-pack/filebeat/module/gsuite/admin/config/pipeline.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,11 +421,7 @@ var login = (function () {
return;
}

var millisToNano = 1e6;
var tsStart = Date.parse(start) * millisToNano;
var tsEnd = Date.parse(end) * millisToNano;

evt.Put("event.duration", tsEnd-tsStart);
evt.Put("event.duration", end.UnixNano() - start.UnixNano());
};

var setEventOutcome = function(evt) {
Expand All @@ -451,6 +447,34 @@ var login = (function () {
evt.Delete("gsuite.admin.WHITELISTED_GROUPS");
};

var deleteField = function(field) {
return function(evt) {
evt.Delete(field);
};
};

var parseDate = function(field, targetField) {
return new processor.Chain()
.Add(new processor.Timestamp({
field: field,
target_field: targetField,
timezone: "UTC",
layouts: [
"2006-01-02T15:04:05Z",
"2006-01-02T15:04:05.999Z",
"2006/01/02 15:04:05 UTC",
],
tests: [
"2020-02-05T18:19:23Z",
"2020-02-05T18:19:23.599Z",
"2020/07/28 04:59:59 UTC",
],
ignore_missing: true,
}))
.Add(deleteField(field))
.Build()
};

var pipeline = new processor.Chain()
.Add(categorizeEvent)
.Add(flattenParams)
Expand Down Expand Up @@ -621,22 +645,6 @@ var login = (function () {
from: "gsuite.admin.PRIVILEGE_NAME",
to: "gsuite.admin.privilege.name",
},
{
from: "gsuite.admin.BEGIN_DATE_TIME",
to: "event.start",
},
{
from: "gsuite.admin.END_DATE_TIME",
to: "event.end",
},
{
from: "gsuite.admin.START_DATE",
to: "event.start",
},
{
from: "gsuite.admin.END_DATE",
to: "event.end",
},
{
from: "gsuite.admin.SITE_LOCATION",
to: "url.path",
Expand Down Expand Up @@ -685,10 +693,6 @@ var login = (function () {
from: "gsuite.admin.EMAIL_LOG_SEARCH_MSG_ID",
to: "gsuite.admin.email.log_search_filter.message_id",
},
{
from: "gsuite.admin.EMAIL_LOG_SEARCH_END_DATE",
to: "gsuite.admin.email.log_search_filter.end_date",
},
{
from: "gsuite.admin.EMAIL_LOG_SEARCH_RECIPIENT",
to: "gsuite.admin.email.log_search_filter.recipient.value",
Expand All @@ -707,10 +711,6 @@ var login = (function () {
to: "gsuite.admin.email.log_search_filter.sender.ip",
type: "ip",
},
{
from: "gsuite.admin.EMAIL_LOG_SEARCH_START_DATE",
to: "gsuite.admin.email.log_search_filter.start_date",
},
{
from: "gsuite.admin.QUARANTINE_NAME",
to: "gsuite.admin.email.quarantine_name",
Expand Down Expand Up @@ -847,10 +847,6 @@ var login = (function () {
from: "gsuite.admin.USER_NICKNAME",
to: "gsuite.admin.user.nickname",
},
{
from: "gsuite.admin.BIRTHDATE",
to: "gsuite.admin.user.birthdate",
},
{
from: "gsuite.admin.ACTION_ID",
to: "gsuite.admin.mobile.action.id",
Expand Down Expand Up @@ -905,6 +901,34 @@ var login = (function () {
ignore_missing: true,
fail_on_error: false,
})
.Add(parseDate(
"gsuite.admin.EMAIL_LOG_SEARCH_END_DATE",
"gsuite.admin.email.log_search_filter.end_date"
))
.Add(parseDate(
"gsuite.admin.EMAIL_LOG_SEARCH_START_DATE",
"gsuite.admin.email.log_search_filter.start_date"
))
.Add(parseDate(
"gsuite.admin.BIRTHDATE",
"gsuite.admin.user.birthdate"
))
.Add(parseDate(
"gsuite.admin.BEGIN_DATE_TIME",
"event.start"
))
.Add(parseDate(
"gsuite.admin.START_DATE",
"event.start"
))
.Add(parseDate(
"gsuite.admin.END_DATE",
"event.end"
))
.Add(parseDate(
"gsuite.admin.END_DATE_TIME",
"event.end"
))
.Add(setGroupInfo)
.Add(setRelatedUserInfo)
.Add(setEventDuration)
Expand Down
6 changes: 3 additions & 3 deletions x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-docs-test.json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,12 @@
],
"event.dataset": "gsuite.admin",
"event.duration": 10800000000000,
"event.end": "2002-10-02T15:00:00Z",
"event.end": "2002-10-02T15:00:00.000Z",
"event.id": "1",
"event.module": "gsuite",
"event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"DOCS_SETTINGS\",\"name\":\"DRIVE_DATA_RESTORE\",\"parameters\":[{\"name\":\"BEGIN_DATE_TIME\",\"value\":\"2002-10-02T12:00:00Z\"},{\"name\":\"END_DATE_TIME\",\"value\":\"2002-10-02T15:00:00Z\"},{\"name\":\"USER_EMAIL\",\"value\":\"user@example.com\"}]}}",
"event.provider": "admin",
"event.start": "2002-10-02T12:00:00Z",
"event.start": "2002-10-02T12:00:00.000Z",
"event.type": [
"info"
],
Expand Down Expand Up @@ -157,4 +157,4 @@
"forwarded"
]
}
]
]
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-gmail-test.json.log
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"DROP_FROM_QUARANTINE","parameters":[{"name":"EMAIL_LOG_SEARCH_MSG_ID","value":"id"},{"name":"QUARANTINE_NAME","value":"quarantine"}]}}
{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"EMAIL_LOG_SEARCH","parameters":[{"name":"EMAIL_LOG_SEARCH_END_DATE","value":"2002-10-02T12:00:00Z"},{"name":"EMAIL_LOG_SEARCH_MSG_ID","value":"id"},{"name":"EMAIL_LOG_SEARCH_RECIPIENT","value":"recipient"},{"name":"EMAIL_LOG_SEARCH_SENDER","value":"sender"},{"name":"EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP","value":"1.1.1.1"},{"name":"EMAIL_LOG_SEARCH_SMTP_SENDER_IP","value":"1.1.1.1"},{"name":"EMAIL_LOG_SEARCH_START_DATE","value":"2002-10-02T10:00:00Z"}]}}
{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"EMAIL_LOG_SEARCH","parameters":[{"name":"EMAIL_LOG_SEARCH_END_DATE","value":"2020/07/28 04:59:59 UTC"},{"name":"EMAIL_LOG_SEARCH_MSG_ID","value":"id"},{"name":"EMAIL_LOG_SEARCH_RECIPIENT","value":"recipient"},{"name":"EMAIL_LOG_SEARCH_SENDER","value":"sender"},{"name":"EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP","value":"1.1.1.1"},{"name":"EMAIL_LOG_SEARCH_SMTP_SENDER_IP","value":"1.1.1.1"},{"name":"EMAIL_LOG_SEARCH_START_DATE","value":"2002-10-02T10:00:00Z"}]}}
{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"EMAIL_UNDELETE","parameters":[{"name":"END_DATE","value":"2002-10-02T12:00:00Z"},{"name":"USER_EMAIL","value":"user@example.com"},{"name":"START_DATE","value":"2002-10-02T10:00:00Z"}]}}
{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"CHANGE_EMAIL_SETTING","parameters":[{"name":"DOMAIN_NAME","value":"example.com"},{"name":"GROUP_EMAIL","value":"group@example.com"},{"name":"NEW_VALUE","value":"new"},{"name":"OLD_VALUE","value":"old"},{"name":"ORG_UNIT_NAME","value":"org"},{"name":"SETTING_NAME","value":"setting"}]}}
{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"CHANGE_GMAIL_SETTING","parameters":[{"name":"ORG_UNIT_NAME","value":"org"},{"name":"SETTING_DESCRIPTION","value":"setting description"},{"name":"SETTING_NAME","value":"setting"},{"name":"USER_DEFINED_SETTING_NAME","value":"setting name"}]}}
Expand Down
26 changes: 13 additions & 13 deletions x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-gmail-test.json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,20 @@
"event.dataset": "gsuite.admin",
"event.id": "1",
"event.module": "gsuite",
"event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"EMAIL_SETTINGS\",\"name\":\"EMAIL_LOG_SEARCH\",\"parameters\":[{\"name\":\"EMAIL_LOG_SEARCH_END_DATE\",\"value\":\"2002-10-02T12:00:00Z\"},{\"name\":\"EMAIL_LOG_SEARCH_MSG_ID\",\"value\":\"id\"},{\"name\":\"EMAIL_LOG_SEARCH_RECIPIENT\",\"value\":\"recipient\"},{\"name\":\"EMAIL_LOG_SEARCH_SENDER\",\"value\":\"sender\"},{\"name\":\"EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP\",\"value\":\"1.1.1.1\"},{\"name\":\"EMAIL_LOG_SEARCH_SMTP_SENDER_IP\",\"value\":\"1.1.1.1\"},{\"name\":\"EMAIL_LOG_SEARCH_START_DATE\",\"value\":\"2002-10-02T10:00:00Z\"}]}}",
"event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"EMAIL_SETTINGS\",\"name\":\"EMAIL_LOG_SEARCH\",\"parameters\":[{\"name\":\"EMAIL_LOG_SEARCH_END_DATE\",\"value\":\"2020/07/28 04:59:59 UTC\"},{\"name\":\"EMAIL_LOG_SEARCH_MSG_ID\",\"value\":\"id\"},{\"name\":\"EMAIL_LOG_SEARCH_RECIPIENT\",\"value\":\"recipient\"},{\"name\":\"EMAIL_LOG_SEARCH_SENDER\",\"value\":\"sender\"},{\"name\":\"EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP\",\"value\":\"1.1.1.1\"},{\"name\":\"EMAIL_LOG_SEARCH_SMTP_SENDER_IP\",\"value\":\"1.1.1.1\"},{\"name\":\"EMAIL_LOG_SEARCH_START_DATE\",\"value\":\"2002-10-02T10:00:00Z\"}]}}",
"event.provider": "admin",
"event.type": [
"info"
],
"fileset.name": "admin",
"gsuite.actor.type": "USER",
"gsuite.admin.email.log_search_filter.end_date": "2002-10-02T12:00:00Z",
"gsuite.admin.email.log_search_filter.end_date": "2020-07-28T04:59:59.000Z",
"gsuite.admin.email.log_search_filter.message_id": "id",
"gsuite.admin.email.log_search_filter.recipient.ip": "1.1.1.1",
"gsuite.admin.email.log_search_filter.recipient.value": "recipient",
"gsuite.admin.email.log_search_filter.sender.ip": "1.1.1.1",
"gsuite.admin.email.log_search_filter.sender.value": "sender",
"gsuite.admin.email.log_search_filter.start_date": "2002-10-02T10:00:00Z",
"gsuite.admin.email.log_search_filter.start_date": "2002-10-02T10:00:00.000Z",
"gsuite.event.type": "EMAIL_SETTINGS",
"gsuite.kind": "admin#reports#activity",
"gsuite.organization.domain": "elastic.com",
Expand Down Expand Up @@ -110,12 +110,12 @@
],
"event.dataset": "gsuite.admin",
"event.duration": 7200000000000,
"event.end": "2002-10-02T12:00:00Z",
"event.end": "2002-10-02T12:00:00.000Z",
"event.id": "1",
"event.module": "gsuite",
"event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"EMAIL_SETTINGS\",\"name\":\"EMAIL_UNDELETE\",\"parameters\":[{\"name\":\"END_DATE\",\"value\":\"2002-10-02T12:00:00Z\"},{\"name\":\"USER_EMAIL\",\"value\":\"user@example.com\"},{\"name\":\"START_DATE\",\"value\":\"2002-10-02T10:00:00Z\"}]}}",
"event.provider": "admin",
"event.start": "2002-10-02T10:00:00Z",
"event.start": "2002-10-02T10:00:00.000Z",
"event.type": [
"creation"
],
Expand All @@ -126,7 +126,7 @@
"gsuite.kind": "admin#reports#activity",
"gsuite.organization.domain": "elastic.com",
"input.type": "log",
"log.offset": 1185,
"log.offset": 1188,
"organization.id": "1",
"related.ip": [
"98.235.162.24"
Expand Down Expand Up @@ -182,7 +182,7 @@
"gsuite.kind": "admin#reports#activity",
"gsuite.organization.domain": "elastic.com",
"input.type": "log",
"log.offset": 1668,
"log.offset": 1671,
"organization.id": "1",
"related.ip": [
"98.235.162.24"
Expand Down Expand Up @@ -233,7 +233,7 @@
"gsuite.kind": "admin#reports#activity",
"gsuite.organization.domain": "elastic.com",
"input.type": "log",
"log.offset": 2251,
"log.offset": 2254,
"organization.id": "1",
"related.ip": [
"98.235.162.24"
Expand Down Expand Up @@ -284,7 +284,7 @@
"gsuite.kind": "admin#reports#activity",
"gsuite.organization.domain": "elastic.com",
"input.type": "log",
"log.offset": 2789,
"log.offset": 2792,
"organization.id": "1",
"related.ip": [
"98.235.162.24"
Expand Down Expand Up @@ -335,7 +335,7 @@
"gsuite.kind": "admin#reports#activity",
"gsuite.organization.domain": "elastic.com",
"input.type": "log",
"log.offset": 3327,
"log.offset": 3330,
"organization.id": "1",
"related.ip": [
"98.235.162.24"
Expand Down Expand Up @@ -384,7 +384,7 @@
"gsuite.kind": "admin#reports#activity",
"gsuite.organization.domain": "elastic.com",
"input.type": "log",
"log.offset": 3865,
"log.offset": 3868,
"organization.id": "1",
"related.ip": [
"98.235.162.24"
Expand Down Expand Up @@ -433,7 +433,7 @@
"gsuite.kind": "admin#reports#activity",
"gsuite.organization.domain": "elastic.com",
"input.type": "log",
"log.offset": 4299,
"log.offset": 4302,
"organization.id": "1",
"related.ip": [
"98.235.162.24"
Expand All @@ -460,4 +460,4 @@
"forwarded"
]
}
]
]
12 changes: 6 additions & 6 deletions x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-user-test.json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1389,12 +1389,12 @@
],
"event.dataset": "gsuite.admin",
"event.duration": 3600000000000,
"event.end": "2002-10-02T16:00:00Z",
"event.end": "2002-10-02T16:00:00.000Z",
"event.id": "1",
"event.module": "gsuite",
"event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"USER_SETTINGS\",\"name\":\"CREATE_EMAIL_MONITOR\",\"parameters\":[{\"name\":\"USER_EMAIL\",\"value\":\"user@example.com\"},{\"name\":\"BEGIN_DATE_TIME\",\"value\":\"2002-10-02T15:00:00Z\"},{\"name\":\"EMAIL_MONITOR_DEST_EMAIL\",\"value\":\"dest@example.com\"},{\"name\":\"EMAIL_MONITOR_LEVEL_CHAT\",\"value\":\"info\"},{\"name\":\"EMAIL_MONITOR_LEVEL_DRAFT_EMAIL\",\"value\":\"info\"},{\"name\":\"EMAIL_MONITOR_LEVEL_INCOMING_EMAIL\",\"value\":\"info\"},{\"name\":\"EMAIL_MONITOR_LEVEL_OUTGOING_EMAIL\",\"value\":\"info\"},{\"name\":\"END_DATE_TIME\",\"value\":\"2002-10-02T16:00:00Z\"}]}}",
"event.provider": "admin",
"event.start": "2002-10-02T15:00:00Z",
"event.start": "2002-10-02T15:00:00.000Z",
"event.type": [
"user",
"creation"
Expand Down Expand Up @@ -2361,12 +2361,12 @@
],
"event.dataset": "gsuite.admin",
"event.duration": 3600000000000,
"event.end": "2002-10-02T16:00:00Z",
"event.end": "2002-10-02T16:00:00.000Z",
"event.id": "1",
"event.module": "gsuite",
"event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"USER_SETTINGS\",\"name\":\"REQUEST_MAILBOX_DUMP\",\"parameters\":[{\"name\":\"USER_EMAIL\",\"value\":\"user@example.com\"},{\"name\":\"BEGIN_DATE_TIME\",\"value\":\"2002-10-02T15:00:00Z\"},{\"name\":\"EMAIL_EXPORT_INCLUDE_DELETED\",\"value\":\"true\"},{\"name\":\"EMAIL_EXPORT_PACKAGE_CONTENT\",\"value\":\"contents\"},{\"name\":\"SEARCH_QUERY_FOR_DUMP\",\"value\":\"foo bar\"},{\"name\":\"END_DATE_TIME\",\"value\":\"2002-10-02T16:00:00Z\"}]}}",
"event.provider": "admin",
"event.start": "2002-10-02T15:00:00Z",
"event.start": "2002-10-02T15:00:00.000Z",
"event.type": [
"user",
"info"
Expand Down Expand Up @@ -2929,7 +2929,7 @@
],
"fileset.name": "admin",
"gsuite.actor.type": "USER",
"gsuite.admin.user.birthdate": "2002-10-02T15:00:00Z",
"gsuite.admin.user.birthdate": "2002-10-02T15:00:00.000Z",
"gsuite.admin.user.email": "user@example.com",
"gsuite.event.type": "USER_SETTINGS",
"gsuite.kind": "admin#reports#activity",
Expand Down Expand Up @@ -3763,4 +3763,4 @@
"forwarded"
]
}
]
]

0 comments on commit 9a2ca19

Please sign in to comment.