diff --git a/receiver/mongodbatlasreceiver/log_decode_test.go b/receiver/mongodbatlasreceiver/log_decode_test.go index 8a54aaeca670..4f0eba14b5b5 100644 --- a/receiver/mongodbatlasreceiver/log_decode_test.go +++ b/receiver/mongodbatlasreceiver/log_decode_test.go @@ -226,3 +226,266 @@ func TestDecode5_0NotGzip(t *testing.T) { func strp(s string) *string { return &s } + +func TestDecodeAudit4_2(t *testing.T) { + b, err := os.ReadFile(filepath.Join("testdata", "logs", "sample-payloads", "4.2_audit.log")) + require.NoError(t, err) + + zippedBuffer := &bytes.Buffer{} + gzipWriter := gzip.NewWriter(zippedBuffer) + + _, err = gzipWriter.Write(b) + require.NoError(t, err) + require.NoError(t, gzipWriter.Close()) + + entries, err := decodeAuditJSON(zippedBuffer) + require.NoError(t, err) + + t.Logf("%#v", entries) + + require.Equal(t, []model.AuditLog{ + { + AuthType: "", + Timestamp: model.LogTimestamp{ + Date: "2022-09-16T01:38:20.034+0000", + }, + ID: model.ID{ + Binary: "", + Type: "", + }, + Local: model.Address{ + IP: "127.0.0.1", + Port: 27017, + }, + Remote: model.Address{ + IP: "127.0.0.1", + Port: 50722, + }, + Result: 0, + Param: model.Param{ + User: "mms-automation", + Database: "admin", + Mechanism: "SCRAM-SHA-1", + }, + }, + { + AuthType: "", + Timestamp: model.LogTimestamp{ + Date: "2022-09-16T02:37:38.714+0000", + }, ID: model.ID{ + Binary: "", + Type: "", + }, Local: model.Address{ + IP: "192.168.248.5", + Port: 27017, + }, Remote: model.Address{ + IP: "192.168.248.6", + Port: 43714, + }, Result: 0, + Param: model.Param{ + User: "mms-automation", + Database: "admin", + Mechanism: "SCRAM-SHA-1", + }, + }, + { + AuthType: "", + Timestamp: model.LogTimestamp{ + Date: "2022-09-16T02:38:20.030+0000", + }, ID: model.ID{ + Binary: "", + Type: "", + }, + Local: model.Address{ + IP: "127.0.0.1", + Port: 27017, + }, + Remote: model.Address{ + IP: "127.0.0.1", + Port: 52216, + }, + Result: 0, + Param: model.Param{ + User: "mms-automation", + Database: "admin", + Mechanism: "SCRAM-SHA-1", + }, + }, + }, entries) +} + +func TestDecodeAudit4_2InvalidLog(t *testing.T) { + b, err := os.ReadFile(filepath.Join("testdata", "logs", "sample-payloads", "4.2_audit_invalid_log.log")) + require.NoError(t, err) + + zippedBuffer := &bytes.Buffer{} + gzipWriter := gzip.NewWriter(zippedBuffer) + + _, err = gzipWriter.Write(b) + require.NoError(t, err) + require.NoError(t, gzipWriter.Close()) + + entries, err := decodeAuditJSON(zippedBuffer) + assert.ErrorContains(t, err, "entry could not be decoded into AuditLog") + + require.Equal(t, []model.AuditLog{ + { + AuthType: "", + Timestamp: model.LogTimestamp{ + Date: "2022-09-16T01:38:20.034+0000", + }, + ID: model.ID{ + Binary: "", + Type: "", + }, + Local: model.Address{ + IP: "127.0.0.1", + Port: 27017, + }, + Remote: model.Address{ + IP: "127.0.0.1", + Port: 50722, + }, + Result: 0, + Param: model.Param{ + User: "mms-automation", + Database: "admin", + Mechanism: "SCRAM-SHA-1", + }, + }, + }, entries) +} + +func TestDecodeAudit5_0(t *testing.T) { + b, err := os.ReadFile(filepath.Join("testdata", "logs", "sample-payloads", "5.0_audit.log")) + require.NoError(t, err) + + zippedBuffer := &bytes.Buffer{} + gzipWriter := gzip.NewWriter(zippedBuffer) + + _, err = gzipWriter.Write(b) + require.NoError(t, err) + require.NoError(t, gzipWriter.Close()) + + entries, err := decodeAuditJSON(zippedBuffer) + require.NoError(t, err) + + t.Logf("%#v", entries) + + require.Equal(t, []model.AuditLog{ + { + AuthType: "", + Timestamp: model.LogTimestamp{ + Date: "2022-09-15T23:56:28.043+00:00", + }, + ID: model.ID{ + Binary: "KXMtAMh9TOOSl9aQBW1Zkg==", + Type: "04", + }, + Local: model.Address{ + IP: "192.168.248.2", + Port: 27017, + }, Remote: model.Address{ + IP: "192.168.248.2", + Port: 34736, + }, + Result: 0, + Param: model.Param{ + User: "", + Database: "", + Mechanism: "", + }, + }, { + AuthType: "", + Timestamp: model.LogTimestamp{ + Date: "2022-09-15T23:56:28.055+00:00", + }, + ID: model.ID{ + Binary: "pWSwWRZvR9CgNsvcDYhiwg==", + Type: "04", + }, + Local: model.Address{ + IP: "192.168.248.2", + Port: 27017, + }, + Remote: model.Address{ + IP: "192.168.248.2", + Port: 34740, + }, + Result: 0, + Param: model.Param{ + User: "", + Database: "", + Mechanism: "", + }, + }, { + AuthType: "", + Timestamp: model.LogTimestamp{ + Date: "2022-09-15T23:56:28.071+00:00", + }, + ID: model.ID{ + Binary: "pWSwWRZvR9CgNsvcDYhiwg==", + Type: "04", + }, Local: model.Address{ + IP: "192.168.248.2", + Port: 27017, + }, Remote: model.Address{ + IP: "192.168.248.2", + Port: 34740, + }, + Result: 0, + Param: model.Param{ + User: "", + Database: "", + Mechanism: "", + }, + }, + }, entries) +} + +func TestDecodeAudit5_0InvalidLog(t *testing.T) { + b, err := os.ReadFile(filepath.Join("testdata", "logs", "sample-payloads", "5.0_audit_invalid_log.log")) + require.NoError(t, err) + + zippedBuffer := &bytes.Buffer{} + gzipWriter := gzip.NewWriter(zippedBuffer) + + _, err = gzipWriter.Write(b) + require.NoError(t, err) + require.NoError(t, gzipWriter.Close()) + + entries, err := decodeAuditJSON(zippedBuffer) + assert.ErrorContains(t, err, "entry could not be decoded into AuditLog") + + require.Equal(t, []model.AuditLog{ + { + AuthType: "", + Timestamp: model.LogTimestamp{ + Date: "2022-09-15T23:56:28.043+00:00", + }, + ID: model.ID{ + Binary: "KXMtAMh9TOOSl9aQBW1Zkg==", + Type: "04", + }, + Local: model.Address{ + IP: "192.168.248.2", + Port: 27017, + }, Remote: model.Address{ + IP: "192.168.248.2", + Port: 34736, + }, + Result: 0, + Param: model.Param{ + User: "", + Database: "", + Mechanism: "", + }, + }, + }, entries) +} + +func TestDecodeAuditNotGzip(t *testing.T) { + entries, err := decodeAuditJSON(bytes.NewBuffer([]byte("Not compressed log"))) + require.ErrorContains(t, err, "gzip: invalid header") + require.Nil(t, entries) +} diff --git a/receiver/mongodbatlasreceiver/log_decoder.go b/receiver/mongodbatlasreceiver/log_decoder.go index f57ec8b538b9..05a895fe0f0a 100644 --- a/receiver/mongodbatlasreceiver/log_decoder.go +++ b/receiver/mongodbatlasreceiver/log_decoder.go @@ -102,3 +102,28 @@ func decode4_2(logger *zap.Logger, r io.Reader) ([]model.LogEntry, error) { entries = append(entries, entry) } } + +func decodeAuditJSON(r io.Reader) ([]model.AuditLog, error) { + // Pass this into a gzip reader for decoding + reader, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + + // Logs are in JSON format so create a JSON decoder to process them + dec := json.NewDecoder(reader) + + var entries []model.AuditLog + for { + var entry model.AuditLog + err := dec.Decode(&entry) + if errors.Is(err, io.EOF) { + return entries, nil + } + if err != nil { + return entries, fmt.Errorf("entry could not be decoded into AuditLog: %w", err) + } + + entries = append(entries, entry) + } +} diff --git a/receiver/mongodbatlasreceiver/logs.go b/receiver/mongodbatlasreceiver/logs.go index a24604ba75d8..ab2d247d0854 100644 --- a/receiver/mongodbatlasreceiver/logs.go +++ b/receiver/mongodbatlasreceiver/logs.go @@ -15,9 +15,7 @@ package mongodbatlasreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mongodbatlasreceiver" import ( - "compress/gzip" "context" - "encoding/json" "errors" "io" "net" @@ -219,26 +217,8 @@ func (s *logsReceiver) getHostAuditLogs(groupID, hostname, logName string) ([]mo if err != nil { return nil, err } - reader, err := gzip.NewReader(buf) - if err != nil { - return nil, err - } - dec := json.NewDecoder(reader) - - var entries []model.AuditLog - for { - var entry model.AuditLog - err := dec.Decode(&entry) - if errors.Is(err, io.EOF) { - return entries, nil - } - if err != nil { - s.log.Error("Entry could not be decoded into LogEntry", zap.Error(err)) - } - - entries = append(entries, entry) - } + return decodeAuditJSON(buf) } func (s *logsReceiver) collectLogs(pc ProjectContext, hostname, logName, clusterName, clusterMajorVersion string) { diff --git a/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/4.2_audit.log b/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/4.2_audit.log new file mode 100644 index 000000000000..3060b4722b4c --- /dev/null +++ b/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/4.2_audit.log @@ -0,0 +1,3 @@ +{ "atype" : "authenticate", "ts" : { "$date" : "2022-09-16T01:38:20.034+0000" }, "local" : { "ip" : "127.0.0.1", "port" : 27017 }, "remote" : { "ip" : "127.0.0.1", "port" : 50722 }, "users" : [ { "user" : "mms-automation", "db" : "admin" } ], "roles" : [ { "role" : "clusterAdmin", "db" : "admin" }, { "role" : "backup", "db" : "admin" }, { "role" : "dbAdminAnyDatabase", "db" : "admin" }, { "role" : "restore", "db" : "admin" }, { "role" : "userAdminAnyDatabase", "db" : "admin" }, { "role" : "readWriteAnyDatabase", "db" : "admin" } ], "param" : { "user" : "mms-automation", "db" : "admin", "mechanism" : "SCRAM-SHA-1" }, "result" : 0 } +{ "atype" : "authenticate", "ts" : { "$date" : "2022-09-16T02:37:38.714+0000" }, "local" : { "ip" : "192.168.248.5", "port" : 27017 }, "remote" : { "ip" : "192.168.248.6", "port" : 43714 }, "users" : [ { "user" : "mms-automation", "db" : "admin" } ], "roles" : [ { "role" : "clusterAdmin", "db" : "admin" }, { "role" : "backup", "db" : "admin" }, { "role" : "dbAdminAnyDatabase", "db" : "admin" }, { "role" : "restore", "db" : "admin" }, { "role" : "userAdminAnyDatabase", "db" : "admin" }, { "role" : "readWriteAnyDatabase", "db" : "admin" } ], "param" : { "user" : "mms-automation", "db" : "admin", "mechanism" : "SCRAM-SHA-1" }, "result" : 0 } +{ "atype" : "authenticate", "ts" : { "$date" : "2022-09-16T02:38:20.030+0000" }, "local" : { "ip" : "127.0.0.1", "port" : 27017 }, "remote" : { "ip" : "127.0.0.1", "port" : 52216 }, "users" : [ { "user" : "mms-automation", "db" : "admin" } ], "roles" : [ { "role" : "clusterAdmin", "db" : "admin" }, { "role" : "backup", "db" : "admin" }, { "role" : "dbAdminAnyDatabase", "db" : "admin" }, { "role" : "restore", "db" : "admin" }, { "role" : "userAdminAnyDatabase", "db" : "admin" }, { "role" : "readWriteAnyDatabase", "db" : "admin" } ], "param" : { "user" : "mms-automation", "db" : "admin", "mechanism" : "SCRAM-SHA-1" }, "result" : 0 } diff --git a/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/4.2_audit_invalid_log.log b/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/4.2_audit_invalid_log.log new file mode 100644 index 000000000000..0d1f1b10aa19 --- /dev/null +++ b/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/4.2_audit_invalid_log.log @@ -0,0 +1,3 @@ +{ "atype" : "authenticate", "ts" : { "$date" : "2022-09-16T01:38:20.034+0000" }, "local" : { "ip" : "127.0.0.1", "port" : 27017 }, "remote" : { "ip" : "127.0.0.1", "port" : 50722 }, "users" : [ { "user" : "mms-automation", "db" : "admin" } ], "roles" : [ { "role" : "clusterAdmin", "db" : "admin" }, { "role" : "backup", "db" : "admin" }, { "role" : "dbAdminAnyDatabase", "db" : "admin" }, { "role" : "restore", "db" : "admin" }, { "role" : "userAdminAnyDatabase", "db" : "admin" }, { "role" : "readWriteAnyDatabase", "db" : "admin" } ], "param" : { "user" : "mms-automation", "db" : "admin", "mechanism" : "SCRAM-SHA-1" }, "result" : 0 } +"atype" : "authenticate", "ts" : { "$date" : "2022-09-16T02:37:38.714+0000" }, "local" : { "ip" : "192.168.248.5", "port" : 27017 }, "remote" : { "ip" : "192.168.248.6", "port" : 43714 }, "users" : [ { "user" : "mms-automation", "db" : "admin" } ], "roles" : [ { "role" : "clusterAdmin", "db" : "admin" }, { "role" : "backup", "db" : "admin" }, { "role" : "dbAdminAnyDatabase", "db" : "admin" }, { "role" : "restore", "db" : "admin" }, { "role" : "userAdminAnyDatabase", "db" : "admin" }, { "role" : "readWriteAnyDatabase", "db" : "admin" } ], "param" : { "user" : "mms-automation", "db" : "admin", "mechanism" : "SCRAM-SHA-1" }, "result" : 0 } +{ "atype" : "authenticate", "ts" : { "$date" : "2022-09-16T02:38:20.030+0000" }, "local" : { "ip" : "127.0.0.1", "port" : 27017 }, "remote" : { "ip" : "127.0.0.1", "port" : 52216 }, "users" : [ { "user" : "mms-automation", "db" : "admin" } ], "roles" : [ { "role" : "clusterAdmin", "db" : "admin" }, { "role" : "backup", "db" : "admin" }, { "role" : "dbAdminAnyDatabase", "db" : "admin" }, { "role" : "restore", "db" : "admin" }, { "role" : "userAdminAnyDatabase", "db" : "admin" }, { "role" : "readWriteAnyDatabase", "db" : "admin" } ], "param" : { "user" : "mms-automation", "db" : "admin", "mechanism" : "SCRAM-SHA-1" }, "result" : 0 } diff --git a/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/5.0_audit.log b/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/5.0_audit.log new file mode 100644 index 000000000000..59906e159601 --- /dev/null +++ b/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/5.0_audit.log @@ -0,0 +1,3 @@ +{ "atype" : "clientMetadata", "ts" : { "$date" : "2022-09-15T23:56:28.043+00:00" }, "uuid" : { "$binary" : "KXMtAMh9TOOSl9aQBW1Zkg==", "$type" : "04" }, "local" : { "ip" : "192.168.248.2", "port" : 27017 }, "remote" : { "ip" : "192.168.248.2", "port" : 34736 }, "users" : [], "roles" : [], "param" : { "localEndpoint" : { "ip" : "192.168.248.2", "port" : 27017 }, "clientMetadata" : { "driver" : { "name" : "mongo-go-driver", "version" : "v1.7.2+prerelease" }, "os" : { "type" : "linux", "architecture" : "amd64" }, "platform" : "go1.18.2", "application" : { "name" : "MongoDB Automation Agent v12.3.4.7674 (git: 4c7df3ac1d15ef3269d44aa38b17376ca00147eb)" } } }, "result" : 0 } +{ "atype" : "clientMetadata", "ts" : { "$date" : "2022-09-15T23:56:28.055+00:00" }, "uuid" : { "$binary" : "pWSwWRZvR9CgNsvcDYhiwg==", "$type" : "04" }, "local" : { "ip" : "192.168.248.2", "port" : 27017 }, "remote" : { "ip" : "192.168.248.2", "port" : 34740 }, "users" : [], "roles" : [], "param" : { "localEndpoint" : { "ip" : "192.168.248.2", "port" : 27017 }, "clientMetadata" : { "driver" : { "name" : "mongo-go-driver", "version" : "v1.7.2+prerelease" }, "os" : { "type" : "linux", "architecture" : "amd64" }, "platform" : "go1.18.2", "application" : { "name" : "MongoDB Automation Agent v12.3.4.7674 (git: 4c7df3ac1d15ef3269d44aa38b17376ca00147eb)" } } }, "result" : 0 } +{ "atype" : "logout", "ts" : { "$date" : "2022-09-15T23:56:28.071+00:00" }, "uuid" : { "$binary" : "pWSwWRZvR9CgNsvcDYhiwg==", "$type" : "04" }, "local" : { "ip" : "192.168.248.2", "port" : 27017 }, "remote" : { "ip" : "192.168.248.2", "port" : 34740 }, "users" : [], "roles" : [], "param" : { "reason" : "Client has disconnected", "initialUsers" : [ { "user" : "__system", "db" : "local" } ], "updatedUsers" : [] }, "result" : 0 } diff --git a/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/5.0_audit_invalid_log.log b/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/5.0_audit_invalid_log.log new file mode 100644 index 000000000000..d8ed8bad4822 --- /dev/null +++ b/receiver/mongodbatlasreceiver/testdata/logs/sample-payloads/5.0_audit_invalid_log.log @@ -0,0 +1,3 @@ +{ "atype" : "clientMetadata", "ts" : { "$date" : "2022-09-15T23:56:28.043+00:00" }, "uuid" : { "$binary" : "KXMtAMh9TOOSl9aQBW1Zkg==", "$type" : "04" }, "local" : { "ip" : "192.168.248.2", "port" : 27017 }, "remote" : { "ip" : "192.168.248.2", "port" : 34736 }, "users" : [], "roles" : [], "param" : { "localEndpoint" : { "ip" : "192.168.248.2", "port" : 27017 }, "clientMetadata" : { "driver" : { "name" : "mongo-go-driver", "version" : "v1.7.2+prerelease" }, "os" : { "type" : "linux", "architecture" : "amd64" }, "platform" : "go1.18.2", "application" : { "name" : "MongoDB Automation Agent v12.3.4.7674 (git: 4c7df3ac1d15ef3269d44aa38b17376ca00147eb)" } } }, "result" : 0 } +"atype" : "clientMetadata", "ts" : { "$date" : "2022-09-15T23:56:28.055+00:00" }, "uuid" : { "$binary" : "pWSwWRZvR9CgNsvcDYhiwg==", "$type" : "04" }, "local" : { "ip" : "192.168.248.2", "port" : 27017 }, "remote" : { "ip" : "192.168.248.2", "port" : 34740 }, "users" : [], "roles" : [], "param" : { "localEndpoint" : { "ip" : "192.168.248.2", "port" : 27017 }, "clientMetadata" : { "driver" : { "name" : "mongo-go-driver", "version" : "v1.7.2+prerelease" }, "os" : { "type" : "linux", "architecture" : "amd64" }, "platform" : "go1.18.2", "application" : { "name" : "MongoDB Automation Agent v12.3.4.7674 (git: 4c7df3ac1d15ef3269d44aa38b17376ca00147eb)" } } }, "result" : 0 } +{ "atype" : "logout", "ts" : { "$date" : "2022-09-15T23:56:28.071+00:00" }, "uuid" : { "$binary" : "pWSwWRZvR9CgNsvcDYhiwg==", "$type" : "04" }, "local" : { "ip" : "192.168.248.2", "port" : 27017 }, "remote" : { "ip" : "192.168.248.2", "port" : 34740 }, "users" : [], "roles" : [], "param" : { "reason" : "Client has disconnected", "initialUsers" : [ { "user" : "__system", "db" : "local" } ], "updatedUsers" : [] }, "result" : 0 } diff --git a/unreleased/mongodb-atlas-strengthen-audit-log-parsing.yaml b/unreleased/mongodb-atlas-strengthen-audit-log-parsing.yaml new file mode 100755 index 000000000000..a29dd706cb51 --- /dev/null +++ b/unreleased/mongodb-atlas-strengthen-audit-log-parsing.yaml @@ -0,0 +1,11 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: mongodbatlasreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Fix potential infinite loop when scraping audit logs + +# One or more tracking issues related to the change +issues: [14169]