Skip to content

Commit

Permalink
[Journalbeat] Add ECS fields to journalbeat (elastic#19176)
Browse files Browse the repository at this point in the history 
* Add ECS fields to journalbeat

* Add changelog entry

(cherry picked from commit a9d457b)
  • Loading branch information
Andrew Stucki committed Jun 18, 2020
1 parent d680b8f commit 7b41942
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 63 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,7 @@ field. You can revert this change by configuring tags for the module and omittin

- Added an `id` config option to inputs to allow running multiple inputs on the
same journal. {pull}18467{18467}
- Add basic ECS categorization and `log.syslog` fields. {pull}19176[19176]

*Metricbeat*

Expand Down
116 changes: 58 additions & 58 deletions journalbeat/reader/fields.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,74 +22,74 @@ package reader
import "github.com/coreos/go-systemd/v22/sdjournal"

type fieldConversion struct {
name string
names []string
isInteger bool
dropped bool
}

var (
journaldEventFields = map[string]fieldConversion{
// provided by systemd journal
"COREDUMP_UNIT": fieldConversion{"journald.coredump.unit", false, false},
"COREDUMP_USER_UNIT": fieldConversion{"journald.coredump.user_unit", false, false},
"OBJECT_AUDIT_LOGINUID": fieldConversion{"journald.object.audit.login_uid", true, false},
"OBJECT_AUDIT_SESSION": fieldConversion{"journald.object.audit.session", true, false},
"OBJECT_CMDLINE": fieldConversion{"journald.object.cmd", false, false},
"OBJECT_COMM": fieldConversion{"journald.object.name", false, false},
"OBJECT_EXE": fieldConversion{"journald.object.executable", false, false},
"OBJECT_GID": fieldConversion{"journald.object.gid", true, false},
"OBJECT_PID": fieldConversion{"journald.object.pid", true, false},
"OBJECT_SYSTEMD_OWNER_UID": fieldConversion{"journald.object.systemd.owner_uid", true, false},
"OBJECT_SYSTEMD_SESSION": fieldConversion{"journald.object.systemd.session", false, false},
"OBJECT_SYSTEMD_UNIT": fieldConversion{"journald.object.systemd.unit", false, false},
"OBJECT_SYSTEMD_USER_UNIT": fieldConversion{"journald.object.systemd.user_unit", false, false},
"OBJECT_UID": fieldConversion{"journald.object.uid", true, false},
"_KERNEL_DEVICE": fieldConversion{"journald.kernel.device", false, false},
"_KERNEL_SUBSYSTEM": fieldConversion{"journald.kernel.subsystem", false, false},
"_SYSTEMD_INVOCATION_ID": fieldConversion{"systemd.invocation_id", false, false},
"_SYSTEMD_USER_SLICE": fieldConversion{"systemd.user_slice", false, false},
"_UDEV_DEVLINK": fieldConversion{"journald.kernel.device_symlinks", false, false}, // TODO aggregate multiple elements
"_UDEV_DEVNODE": fieldConversion{"journald.kernel.device_node_path", false, false},
"_UDEV_SYSNAME": fieldConversion{"journald.kernel.device_name", false, false},
sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID: fieldConversion{"process.audit.login_uid", true, false},
sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION: fieldConversion{"process.audit.session", false, false},
sdjournal.SD_JOURNAL_FIELD_BOOT_ID: fieldConversion{"host.boot_id", false, false},
sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE: fieldConversion{"process.capabilites", false, false},
sdjournal.SD_JOURNAL_FIELD_CMDLINE: fieldConversion{"process.cmd", false, false},
sdjournal.SD_JOURNAL_FIELD_CODE_FILE: fieldConversion{"journald.code.file", false, false},
sdjournal.SD_JOURNAL_FIELD_CODE_FUNC: fieldConversion{"journald.code.func", false, false},
sdjournal.SD_JOURNAL_FIELD_CODE_LINE: fieldConversion{"journald.code.line", true, false},
sdjournal.SD_JOURNAL_FIELD_COMM: fieldConversion{"process.name", false, false},
sdjournal.SD_JOURNAL_FIELD_EXE: fieldConversion{"process.executable", false, false},
sdjournal.SD_JOURNAL_FIELD_GID: fieldConversion{"process.uid", true, false},
sdjournal.SD_JOURNAL_FIELD_HOSTNAME: fieldConversion{"host.hostname", false, false},
sdjournal.SD_JOURNAL_FIELD_MACHINE_ID: fieldConversion{"host.id", false, false},
sdjournal.SD_JOURNAL_FIELD_MESSAGE: fieldConversion{"message", false, false},
sdjournal.SD_JOURNAL_FIELD_PID: fieldConversion{"process.pid", true, false},
sdjournal.SD_JOURNAL_FIELD_PRIORITY: fieldConversion{"syslog.priority", true, false},
sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY: fieldConversion{"syslog.facility", true, false},
sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: fieldConversion{"syslog.identifier", false, false},
sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: fieldConversion{"syslog.pid", true, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP: fieldConversion{"systemd.cgroup", false, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_OWNER_UID: fieldConversion{"systemd.owner_uid", true, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SESSION: fieldConversion{"systemd.session", false, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SLICE: fieldConversion{"systemd.slice", false, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT: fieldConversion{"systemd.unit", false, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: fieldConversion{"systemd.user_unit", false, false},
sdjournal.SD_JOURNAL_FIELD_TRANSPORT: fieldConversion{"systemd.transport", false, false},
sdjournal.SD_JOURNAL_FIELD_UID: fieldConversion{"process.uid", true, false},
"COREDUMP_UNIT": fieldConversion{[]string{"journald.coredump.unit"}, false, false},
"COREDUMP_USER_UNIT": fieldConversion{[]string{"journald.coredump.user_unit"}, false, false},
"OBJECT_AUDIT_LOGINUID": fieldConversion{[]string{"journald.object.audit.login_uid"}, true, false},
"OBJECT_AUDIT_SESSION": fieldConversion{[]string{"journald.object.audit.session"}, true, false},
"OBJECT_CMDLINE": fieldConversion{[]string{"journald.object.cmd"}, false, false},
"OBJECT_COMM": fieldConversion{[]string{"journald.object.name"}, false, false},
"OBJECT_EXE": fieldConversion{[]string{"journald.object.executable"}, false, false},
"OBJECT_GID": fieldConversion{[]string{"journald.object.gid"}, true, false},
"OBJECT_PID": fieldConversion{[]string{"journald.object.pid"}, true, false},
"OBJECT_SYSTEMD_OWNER_UID": fieldConversion{[]string{"journald.object.systemd.owner_uid"}, true, false},
"OBJECT_SYSTEMD_SESSION": fieldConversion{[]string{"journald.object.systemd.session"}, false, false},
"OBJECT_SYSTEMD_UNIT": fieldConversion{[]string{"journald.object.systemd.unit"}, false, false},
"OBJECT_SYSTEMD_USER_UNIT": fieldConversion{[]string{"journald.object.systemd.user_unit"}, false, false},
"OBJECT_UID": fieldConversion{[]string{"journald.object.uid"}, true, false},
"_KERNEL_DEVICE": fieldConversion{[]string{"journald.kernel.device"}, false, false},
"_KERNEL_SUBSYSTEM": fieldConversion{[]string{"journald.kernel.subsystem"}, false, false},
"_SYSTEMD_INVOCATION_ID": fieldConversion{[]string{"systemd.invocation_id"}, false, false},
"_SYSTEMD_USER_SLICE": fieldConversion{[]string{"systemd.user_slice"}, false, false},
"_UDEV_DEVLINK": fieldConversion{[]string{"journald.kernel.device_symlinks"}, false, false}, // TODO aggregate multiple elements
"_UDEV_DEVNODE": fieldConversion{[]string{"journald.kernel.device_node_path"}, false, false},
"_UDEV_SYSNAME": fieldConversion{[]string{"journald.kernel.device_name"}, false, false},
sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID: fieldConversion{[]string{"process.audit.login_uid"}, true, false},
sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION: fieldConversion{[]string{"process.audit.session"}, false, false},
sdjournal.SD_JOURNAL_FIELD_BOOT_ID: fieldConversion{[]string{"host.boot_id"}, false, false},
sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE: fieldConversion{[]string{"process.capabilites"}, false, false},
sdjournal.SD_JOURNAL_FIELD_CMDLINE: fieldConversion{[]string{"process.cmd"}, false, false},
sdjournal.SD_JOURNAL_FIELD_CODE_FILE: fieldConversion{[]string{"journald.code.file"}, false, false},
sdjournal.SD_JOURNAL_FIELD_CODE_FUNC: fieldConversion{[]string{"journald.code.func"}, false, false},
sdjournal.SD_JOURNAL_FIELD_CODE_LINE: fieldConversion{[]string{"journald.code.line"}, true, false},
sdjournal.SD_JOURNAL_FIELD_COMM: fieldConversion{[]string{"process.name"}, false, false},
sdjournal.SD_JOURNAL_FIELD_EXE: fieldConversion{[]string{"process.executable"}, false, false},
sdjournal.SD_JOURNAL_FIELD_GID: fieldConversion{[]string{"process.uid"}, true, false},
sdjournal.SD_JOURNAL_FIELD_HOSTNAME: fieldConversion{[]string{"host.hostname"}, false, false},
sdjournal.SD_JOURNAL_FIELD_MACHINE_ID: fieldConversion{[]string{"host.id"}, false, false},
sdjournal.SD_JOURNAL_FIELD_MESSAGE: fieldConversion{[]string{"message"}, false, false},
sdjournal.SD_JOURNAL_FIELD_PID: fieldConversion{[]string{"process.pid"}, true, false},
sdjournal.SD_JOURNAL_FIELD_PRIORITY: fieldConversion{[]string{"syslog.priority", "log.syslog.priority"}, true, false},
sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY: fieldConversion{[]string{"syslog.facility", "log.syslog.facility.name"}, true, false},
sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: fieldConversion{[]string{"syslog.identifier"}, false, false},
sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: fieldConversion{[]string{"syslog.pid"}, true, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP: fieldConversion{[]string{"systemd.cgroup"}, false, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_OWNER_UID: fieldConversion{[]string{"systemd.owner_uid"}, true, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SESSION: fieldConversion{[]string{"systemd.session"}, false, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SLICE: fieldConversion{[]string{"systemd.slice"}, false, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT: fieldConversion{[]string{"systemd.unit"}, false, false},
sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: fieldConversion{[]string{"systemd.user_unit"}, false, false},
sdjournal.SD_JOURNAL_FIELD_TRANSPORT: fieldConversion{[]string{"systemd.transport"}, false, false},
sdjournal.SD_JOURNAL_FIELD_UID: fieldConversion{[]string{"process.uid"}, true, false},

// docker journald fields from: https://docs.docker.com/config/containers/logging/journald/
"CONTAINER_ID": fieldConversion{"container.id_truncated", false, false},
"CONTAINER_ID_FULL": fieldConversion{"container.id", false, false},
"CONTAINER_NAME": fieldConversion{"container.name", false, false},
"CONTAINER_TAG": fieldConversion{"container.log.tag", false, false},
"CONTAINER_PARTIAL_MESSAGE": fieldConversion{"container.partial", false, false},
"CONTAINER_ID": fieldConversion{[]string{"container.id_truncated"}, false, false},
"CONTAINER_ID_FULL": fieldConversion{[]string{"container.id"}, false, false},
"CONTAINER_NAME": fieldConversion{[]string{"container.name"}, false, false},
"CONTAINER_TAG": fieldConversion{[]string{"container.log.tag"}, false, false},
"CONTAINER_PARTIAL_MESSAGE": fieldConversion{[]string{"container.partial"}, false, false},

// dropped fields
sdjournal.SD_JOURNAL_FIELD_MONOTONIC_TIMESTAMP: fieldConversion{"", false, true}, // saved in the registry
sdjournal.SD_JOURNAL_FIELD_SOURCE_REALTIME_TIMESTAMP: fieldConversion{"", false, true}, // saved in the registry
sdjournal.SD_JOURNAL_FIELD_CURSOR: fieldConversion{"", false, true}, // saved in the registry
"_SOURCE_MONOTONIC_TIMESTAMP": fieldConversion{"", false, true}, // received timestamp stored in @timestamp
sdjournal.SD_JOURNAL_FIELD_MONOTONIC_TIMESTAMP: fieldConversion{nil, false, true}, // saved in the registry
sdjournal.SD_JOURNAL_FIELD_SOURCE_REALTIME_TIMESTAMP: fieldConversion{nil, false, true}, // saved in the registry
sdjournal.SD_JOURNAL_FIELD_CURSOR: fieldConversion{nil, false, true}, // saved in the registry
"_SOURCE_MONOTONIC_TIMESTAMP": fieldConversion{nil, false, true}, // received timestamp stored in @timestamp
}
)
18 changes: 13 additions & 5 deletions journalbeat/reader/journal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,10 @@ func setupMatches(j *sdjournal.Journal, matches []string) error {

var p string
for journalKey, eventField := range journaldEventFields {
if elems[0] == eventField.name {
p = journalKey + "=" + elems[1]
for _, name := range eventField.names {
if elems[0] == name {
p = journalKey + "=" + elems[1]
}
}
}

Expand Down Expand Up @@ -240,7 +242,11 @@ func (r *Reader) checkForNewEvents() (bool, error) {

// toEvent creates a beat.Event from journal entries.
func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event {
fields := common.MapStr{}
fields := common.MapStr{
"event": common.MapStr{
"kind": "event",
},
}
custom := common.MapStr{}

for entryKey, v := range entry.Fields {
Expand All @@ -249,7 +255,9 @@ func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event {
custom.Put(normalized, v)
} else if !fieldConversionInfo.dropped {
value := r.convertNamedField(fieldConversionInfo, v)
fields.Put(fieldConversionInfo.name, value)
for _, name := range fieldConversionInfo.names {
fields.Put(name, value)
}
}
}

Expand Down Expand Up @@ -295,7 +303,7 @@ func (r *Reader) convertNamedField(fc fieldConversion, value string) interface{}
s := strings.Split(value, ",")
v, err = strconv.ParseInt(s[0], 10, 64)
if err != nil {
r.logger.Debugf("Failed to convert field: %s \"%v\" to int: %v", fc.name, value, err)
r.logger.Debugf("Failed to convert field: %v \"%v\" to int: %v", fc.names, value, err)
return value
}
}
Expand Down
18 changes: 18 additions & 0 deletions journalbeat/reader/journal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,24 @@ func TestToEvent(t *testing.T) {
},
},
},
// 'syslog.priority' field with junk
ToEventTestCase{
entry: sdjournal.JournalEntry{
Fields: map[string]string{
sdjournal.SD_JOURNAL_FIELD_PRIORITY: "123456, ",
},
},
expectedFields: common.MapStr{
"syslog": common.MapStr{
"priority": int64(123456),
},
"log": common.MapStr{
"syslog": common.MapStr{
"priority": int64(123456),
},
},
},
},
// 'syslog.pid' field with user append
ToEventTestCase{
entry: sdjournal.JournalEntry{
Expand Down

0 comments on commit 7b41942

Please sign in to comment.