diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5cf98b41f5b..cd01556a321 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -339,6 +339,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add more DNS error codes to the Sysmon module. {issue}15685[15685] - Add Audit and Log Management, Computer Object Management, and Distribution Group related events to the Security module. {pull}15217[15217] +- Add experimental event log reader implementation that should be faster in most cases. {issue}6585[6585] {pull}16849[16849] ==== Deprecated diff --git a/Vagrantfile b/Vagrantfile index b7cc6f03523..eba786adda2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -164,6 +164,11 @@ Vagrant.configure(2) do |config| config.vm.define "win2019", primary: true do |c| c.vm.box = "StefanScherer/windows_2019" c.vm.provision "shell", inline: $winPsProvision, privileged: false + + c.vm.provider :virtualbox do |vbox| + vbox.memory = 4096 + vbox.cpus = 4 + end end # Solaris 11.2 diff --git a/winlogbeat/docs/winlogbeat-options.asciidoc b/winlogbeat/docs/winlogbeat-options.asciidoc index 88cb61a553f..f5a7520e824 100644 --- a/winlogbeat/docs/winlogbeat-options.asciidoc +++ b/winlogbeat/docs/winlogbeat-options.asciidoc @@ -410,3 +410,29 @@ stopped. *{vista_and_newer}* Setting `no_more_events` to `stop` is useful when reading from archived event log files where you want to read the whole file then exit. There's a complete example of how to read from an `.evtx` file in the <>. + +[float] +==== `event_logs.api` + +experimental[] + +This selects the event log reader implementation that is used to read events +from the Windows APIs. You should only set this option when testing experimental +features. When the value is set to `wineventlog-experimental` Winlogbeat will +replace the default event log reader with the experimental implementation. +We are evaluating this implementation to see if it can provide increased +performance and reduce CPU usage. *{vista_and_newer}* + +[source,yaml] +-------------------------------------------------------------------------------- +winlogbeat.event_logs: + - name: ForwardedEvents + api: wineventlog-experimental +-------------------------------------------------------------------------------- + +There are a few notable differences in the events: + +* Events that contained data under `winlog.user_data` will now have it under + `winlog.event_data`. +* Setting `include_xml: true` has no effect. + diff --git a/winlogbeat/eventlog/bench_test.go b/winlogbeat/eventlog/bench_test.go index e665f01bbbd..ec680c5fb5d 100644 --- a/winlogbeat/eventlog/bench_test.go +++ b/winlogbeat/eventlog/bench_test.go @@ -22,95 +22,112 @@ package eventlog import ( "bytes" "flag" + "fmt" "math/rand" - "os/exec" "strconv" + "strings" "testing" - "time" - elog "github.com/andrewkroh/sys/windows/svc/eventlog" - "github.com/dustin/go-humanize" + "golang.org/x/sys/windows/svc/eventlog" + + "github.com/elastic/beats/v7/libbeat/common" ) -// Benchmark tests with customized output. (`go test -v -benchtime 10s -benchtest .`) +const gigabyte = 1 << 30 var ( - benchTest = flag.Bool("benchtest", false, "Run benchmarks for the eventlog package") - injectAmount = flag.Int("inject", 50000, "Number of events to inject before running benchmarks") + benchTest = flag.Bool("benchtest", false, "Run benchmarks for the eventlog package.") + injectAmount = flag.Int("inject", 1E6, "Number of events to inject before running benchmarks.") ) -// TestBenchmarkBatchReadSize tests the performance of different -// batch_read_size values. -func TestBenchmarkBatchReadSize(t *testing.T) { +// TestBenchmarkRead benchmarks each event log reader implementation with +// different batch sizes. +// +// Recommended usage: +// go test -run TestBenchmarkRead -benchmem -benchtime 10s -benchtest -v . +func TestBenchmarkRead(t *testing.T) { if !*benchTest { t.Skip("-benchtest not enabled") } - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t) + defer teardown() - // Increase the log size so that it can hold these large events. - output, err := exec.Command("wevtutil.exe", "sl", "/ms:1073741824", providerName).CombinedOutput() - if err != nil { - t.Fatal(err, string(output)) - } + setLogSize(t, providerName, gigabyte) // Publish test messages: for i := 0; i < *injectAmount; i++ { - err = log.Report(elog.Info, uint32(rand.Int63()%1000), []string{strconv.Itoa(i) + " " + randomSentence(256)}) + err := writer.Report(eventlog.Info, uint32(rand.Int63()%1000), []string{strconv.Itoa(i) + " " + randomSentence(256)}) if err != nil { - t.Fatal("ReportEvent error", err) + t.Fatal(err) } } - benchTest := func(batchSize int) { - var err error - result := testing.Benchmark(func(b *testing.B) { - eventlog, tearDown := setupWinEventLog(t, 0, map[string]interface{}{ - "name": providerName, - "batch_read_size": batchSize, - }) - defer tearDown() - b.ResetTimer() - - // Each iteration reads one batch. - for i := 0; i < b.N; i++ { - _, err = eventlog.Read() - if err != nil { - return - } + for _, api := range []string{winEventLogAPIName, winEventLogExpAPIName} { + t.Run("api="+api, func(t *testing.T) { + for _, batchSize := range []int{10, 100, 500, 1000} { + t.Run(fmt.Sprintf("batch_size=%d", batchSize), func(t *testing.T) { + result := testing.Benchmark(benchmarkEventLog(api, batchSize)) + outputBenchmarkResults(t, result) + }) } }) + } - if err != nil { - t.Fatal(err) - return + t.Run("api="+eventLoggingAPIName, func(t *testing.T) { + result := testing.Benchmark(benchmarkEventLog(eventLoggingAPIName, -1)) + outputBenchmarkResults(t, result) + }) +} + +func benchmarkEventLog(api string, batchSize int) func(b *testing.B) { + return func(b *testing.B) { + conf := common.MapStr{ + "name": providerName, + } + if strings.HasPrefix(api, "wineventlog") { + conf.Put("batch_read_size", batchSize) + conf.Put("no_more_events", "stop") } - t.Logf("batch_size=%v, total_events=%v, batch_time=%v, events_per_sec=%v, bytes_alloced_per_event=%v, total_allocs=%v", - batchSize, - result.N*batchSize, - time.Duration(result.NsPerOp()), - float64(batchSize)/time.Duration(result.NsPerOp()).Seconds(), - humanize.Bytes(result.MemBytes/(uint64(result.N)*uint64(batchSize))), - result.MemAllocs) - } + log := openLog(b, api, nil, conf) + defer log.Close() + + events := 0 + b.ResetTimer() + + // Each iteration reads one batch. + for i := 0; i < b.N; i++ { + records, err := log.Read() + if err != nil { + b.Fatal(err) + return + } + events += len(records) + } + + b.StopTimer() - benchTest(10) - benchTest(100) - benchTest(500) - benchTest(1000) + b.ReportMetric(float64(events), "events") + b.ReportMetric(float64(batchSize), "batch_size") + } } -// Utility Functions +func outputBenchmarkResults(t testing.TB, result testing.BenchmarkResult) { + totalBatches := result.N + totalEvents := int(result.Extra["events"]) + totalBytes := result.MemBytes + totalAllocs := result.MemAllocs + + eventsPerSec := float64(totalEvents) / result.T.Seconds() + bytesPerEvent := float64(totalBytes) / float64(totalEvents) + bytesPerBatch := float64(totalBytes) / float64(totalBatches) + allocsPerEvent := float64(totalAllocs) / float64(totalEvents) + allocsPerBatch := float64(totalAllocs) / float64(totalBatches) + + t.Logf("%.2f events/sec\t %d B/event\t %d B/batch\t %d allocs/event\t %d allocs/batch", + eventsPerSec, int(bytesPerEvent), int(bytesPerBatch), int(allocsPerEvent), int(allocsPerBatch)) +} var randomWords = []string{ "recover", diff --git a/winlogbeat/eventlog/eventlogging.go b/winlogbeat/eventlog/eventlogging.go index 3e9494c91b6..963797264b2 100644 --- a/winlogbeat/eventlog/eventlogging.go +++ b/winlogbeat/eventlog/eventlogging.go @@ -27,6 +27,7 @@ import ( "github.com/joeshaw/multierror" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/winlogbeat/checkpoint" "github.com/elastic/beats/v7/winlogbeat/sys" @@ -277,6 +278,8 @@ func (l *eventLogging) ignoreOlder(r *Record) bool { // newEventLogging creates and returns a new EventLog for reading event logs // using the Event Logging API. func newEventLogging(options *common.Config) (EventLog, error) { + cfgwarn.Deprecate("8.0", "The eventlogging API reader is deprecated.") + c := eventLoggingConfig{ ReadBufferSize: win.MaxEventBufferSize, FormatBufferSize: win.MaxFormatMessageBufferSize, diff --git a/winlogbeat/eventlog/eventlogging_test.go b/winlogbeat/eventlog/eventlogging_test.go index 7ed4ea78772..ba8524cad09 100644 --- a/winlogbeat/eventlog/eventlogging_test.go +++ b/winlogbeat/eventlog/eventlogging_test.go @@ -21,14 +21,11 @@ package eventlog import ( "fmt" - "os/exec" - "strconv" "strings" "sync" "testing" - elog "github.com/andrewkroh/sys/windows/svc/eventlog" - "github.com/joeshaw/multierror" + "github.com/andrewkroh/sys/windows/svc/eventlog" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" @@ -54,37 +51,33 @@ const ( netEventMsgFile = "%SystemRoot%\\System32\\netevent.dll" ) -const allLevels = elog.Success | elog.AuditFailure | elog.AuditSuccess | elog.Error | elog.Info | elog.Warning - -const gigabyte = 1 << 30 - // Test messages. var messages = map[uint32]struct { eventType uint16 message string }{ 1: { - eventType: elog.Info, + eventType: eventlog.Info, message: "Hmmmm.", }, 2: { - eventType: elog.Success, + eventType: eventlog.Success, message: "I am so blue I'm greener than purple.", }, 3: { - eventType: elog.Warning, + eventType: eventlog.Warning, message: "I stepped on a Corn Flake, now I'm a Cereal Killer.", }, 4: { - eventType: elog.Error, + eventType: eventlog.Error, message: "The quick brown fox jumps over the lazy dog.", }, 5: { - eventType: elog.AuditSuccess, + eventType: eventlog.AuditSuccess, message: "Where do random thoughts come from?", }, 6: { - eventType: elog.AuditFailure, + eventType: eventlog.AuditFailure, message: "Login failure for user xyz!", }, } @@ -100,107 +93,27 @@ func configureLogp() { } else { logp.DevelopmentSetup(logp.WithLevel(logp.WarnLevel)) } - - // Clear the event log before starting. - log, _ := elog.Open(sourceName) - eventlogging.ClearEventLog(eventlogging.Handle(log.Handle), "") - log.Close() }) } -// initLog initializes an event logger. It registers the source name with -// the registry if it does not already exist. -func initLog(provider, source, msgFile string) (*elog.Log, error) { - // Install entry to registry: - _, err := elog.Install(providerName, sourceName, msgFile, true, allLevels) - if err != nil { - return nil, err - } - - // Open a new logger for writing events: - log, err := elog.Open(sourceName) - if err != nil { - var errs multierror.Errors - errs = append(errs, err) - err := elog.RemoveSource(providerName, sourceName) - if err != nil { - errs = append(errs, err) - } - err = elog.RemoveProvider(providerName) - if err != nil { - errs = append(errs, err) - } - return nil, errs.Err() - } - - return log, nil -} - -// uninstallLog unregisters the event logger from the registry and closes the -// log's handle if it is open. -func uninstallLog(provider, source string, log *elog.Log) error { - var errs multierror.Errors - - if log != nil { - err := eventlogging.ClearEventLog(eventlogging.Handle(log.Handle), "") - if err != nil { - errs = append(errs, err) - } - - err = log.Close() - if err != nil { - errs = append(errs, err) - } - } - - err := elog.RemoveSource(providerName, sourceName) - if err != nil { - errs = append(errs, err) - } - - err = elog.RemoveProvider(providerName) - if err != nil { - errs = append(errs, err) - } - - return errs.Err() -} - -// setLogSize set the maximum number of bytes that an event log can hold. -func setLogSize(t testing.TB, provider string, sizeBytes int) { - output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), providerName).CombinedOutput() - if err != nil { - t.Fatal("failed to set log size", err, string(output)) - } -} - // Verify that all messages are read from the event log. func TestRead(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t) + defer teardown() // Publish test messages: for k, m := range messages { - err = log.Report(m.eventType, k, []string{m.message}) - if err != nil { + if err := writer.Report(m.eventType, k, []string{m.message}); err != nil { t.Fatal(err) } } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -219,7 +132,7 @@ func TestRead(t *testing.T) { } // Validate getNumberOfEventLogRecords returns the correct number of messages. - numMessages, err := eventlogging.GetNumberOfEventLogRecords(eventlogging.Handle(log.Handle)) + numMessages, err := eventlogging.GetNumberOfEventLogRecords(eventlogging.Handle(writer.Handle)) assert.NoError(t, err) assert.Equal(t, len(messages), int(numMessages)) } @@ -229,36 +142,27 @@ func TestRead(t *testing.T) { // possible buffer so this error should not occur. func TestFormatMessageWithLargeMessage(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t) + defer teardown() - message := "Hello" - err = log.Report(elog.Info, 1, []string{message}) - if err != nil { + const message = "Hello" + if err := writer.Report(eventlog.Info, 1, []string{message}); err != nil { t.Fatal(err) } // Messages are received as UTF-16 so we must have enough space in the read // buffer for the message, a windows newline, and a null-terminator. - requiredBufferSize := len(message+"\r\n")*2 + 2 + const requiredBufferSize = len(message+"\r\n")*2 + 2 // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{ + log := openEventLogging(t, 0, map[string]interface{}{ "name": providerName, // Use a buffer smaller than what is required. "format_buffer_size": requiredBufferSize / 2, }) - defer teardown() + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -275,29 +179,20 @@ func TestFormatMessageWithLargeMessage(t *testing.T) { // insert strings (the message parameters) is returned. func TestReadUnknownEventId(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, servicesMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, servicesMsgFile) + defer teardown() - var eventID uint32 = 1000 - msg := "Test Message" - err = log.Success(eventID, msg) - if err != nil { + const eventID uint32 = 1000 + const msg = "Test Message" + if err := writer.Success(eventID, msg); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -319,30 +214,20 @@ func TestReadUnknownEventId(t *testing.T) { // of the files then the next file should be checked. func TestReadTriesMultipleEventMsgFiles(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, - servicesMsgFile+";"+eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, servicesMsgFile, eventCreateMsgFile) + defer teardown() - var eventID uint32 = 1000 - msg := "Test Message" - err = log.Success(eventID, msg) - if err != nil { + const eventID uint32 = 1000 + const msg = "Test Message" + if err := writer.Success(eventID, msg); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -359,34 +244,25 @@ func TestReadTriesMultipleEventMsgFiles(t *testing.T) { // Test event messages that require more than one message parameter. func TestReadMultiParameterMsg(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, servicesMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, servicesMsgFile) + defer teardown() // EventID observed by exporting system event log to XML and doing calculation. // 7036 // 1073748860 = 16384 << 16 + 7036 // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385206(v=vs.85).aspx - var eventID uint32 = 1073748860 - template := "The %s service entered the %s state." + const eventID uint32 = 1073748860 + const template = "The %s service entered the %s state." msgs := []string{"Windows Update", "running"} - err = log.Report(elog.Info, eventID, msgs) - if err != nil { + if err := writer.Report(eventlog.Info, eventID, msgs); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -406,40 +282,31 @@ func TestReadMultiParameterMsg(t *testing.T) { func TestOpenInvalidProvider(t *testing.T) { configureLogp() - el := newTestEventLogging(t, map[string]interface{}{"name": "nonExistentProvider"}) - assert.NoError(t, el.Open(checkpoint.EventLogState{}), "Calling Open() on an unknown provider "+ - "should automatically open Application.") - _, err := el.Read() + log := openEventLogging(t, 0, map[string]interface{}{"name": "nonExistentProvider"}) + defer log.Close() + + _, err := log.Read() assert.NoError(t, err) } // Test event messages that require no parameters. func TestReadNoParameterMsg(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, netEventMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, netEventMsgFile) + defer teardown() - var eventID uint32 = 2147489654 // 1<<31 + 6006 - template := "The Event log service was stopped." + const eventID uint32 = 2147489654 // 1<<31 + 6006 + const template = "The Event log service was stopped." msgs := []string{} - err = log.Report(elog.Info, eventID, msgs) - if err != nil { + if err := writer.Report(eventlog.Info, eventID, msgs); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -458,33 +325,25 @@ func TestReadNoParameterMsg(t *testing.T) { // being cleared or reset while reading. func TestReadWhileCleared(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() - - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) + writer, teardown := createLog(t) defer teardown() - log.Info(1, "Message 1") - log.Info(2, "Message 2") - lr, err := eventlog.Read() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() + + writer.Info(1, "Message 1") + writer.Info(2, "Message 2") + lr, err := log.Read() assert.NoError(t, err, "Expected 2 messages but received error") assert.Len(t, lr, 2, "Expected 2 messages") - assert.NoError(t, eventlogging.ClearEventLog(eventlogging.Handle(log.Handle), "")) - lr, err = eventlog.Read() + assert.NoError(t, eventlogging.ClearEventLog(eventlogging.Handle(writer.Handle), "")) + lr, err = log.Read() assert.NoError(t, err, "Expected 0 messages but received error") assert.Len(t, lr, 0, "Expected 0 message") - log.Info(3, "Message 3") - lr, err = eventlog.Read() + writer.Info(3, "Message 3") + lr, err = log.Read() assert.NoError(t, err, "Expected 1 message but received error") assert.Len(t, lr, 1, "Expected 1 message") if len(lr) > 0 { @@ -493,34 +352,25 @@ func TestReadWhileCleared(t *testing.T) { } // Test event messages that include less parameters than required for message -// formating (caused a crash in previous versions) +// formatting (caused a crash in previous versions) func TestReadMissingParameters(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, servicesMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, servicesMsgFile) + defer teardown() - var eventID uint32 = 1073748860 + const eventID uint32 = 1073748860 // Missing parameters will be substituted by "(null)" - template := "The %s service entered the (null) state." + const template = "The %s service entered the (null) state." msgs := []string{"Windows Update"} - err = log.Report(elog.Info, eventID, msgs) - if err != nil { + if err := writer.Report(eventlog.Info, eventID, msgs); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -535,22 +385,7 @@ func TestReadMissingParameters(t *testing.T) { strings.TrimRight(records[0].Message, "\r\n")) } -func newTestEventLogging(t *testing.T, options map[string]interface{}) EventLog { - return newTestEventLog(t, newEventLogging, options) +func openEventLogging(t *testing.T, recordID uint64, options map[string]interface{}) EventLog { + t.Helper() + return openLog(t, eventLoggingAPIName, &checkpoint.EventLogState{RecordNumber: recordID}, options) } - -func setupEventLogging(t *testing.T, recordID uint64, options map[string]interface{}) (EventLog, func()) { - return setupEventLog(t, newEventLogging, recordID, options) -} - -// TODO: Add more test cases: -// - Record number rollover (there may be an issue with this if ++ is used anywhere) -// - Reading from a source name instead of provider name (can't be done according to docs). -// - Persistent read mode shall support specifying a record number (or not specifying a record number). -// -- Invalid record number based on range (should start at first record). -// -- Invalid record number based on range timestamp match check (should start at first record). -// -- Valid record number -// --- Do not replay first record (it was already reported) -// -- First read (no saved state) should return the first record (send first reported record). -// - NewOnly read mode shall seek to end and ignore first. -// - ReadThenExit read mode shall seek to end, read backwards, honor the EOF, then exit. diff --git a/winlogbeat/eventlog/wineventlog_expirimental.go b/winlogbeat/eventlog/wineventlog_expirimental.go new file mode 100644 index 00000000000..5952aad0f5a --- /dev/null +++ b/winlogbeat/eventlog/wineventlog_expirimental.go @@ -0,0 +1,298 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package eventlog + +import ( + "io" + "os" + "path/filepath" + + "github.com/pkg/errors" + "go.uber.org/multierr" + "golang.org/x/sys/windows" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/checkpoint" + win "github.com/elastic/beats/v7/winlogbeat/sys/wineventlog" +) + +const ( + // winEventLogExpApiName is the name used to identify the Windows Event Log API + // as both an event type and an API. + winEventLogExpAPIName = "wineventlog-experimental" +) + +// winEventLogExp implements the EventLog interface for reading from the Windows +// Event Log API. +type winEventLogExp struct { + config winEventLogConfig + query string + channelName string // Name of the channel from which to read. + file bool // Reading from file rather than channel. + maxRead int // Maximum number returned in one Read. + lastRead checkpoint.EventLogState // Record number of the last read event. + log *logp.Logger + + iterator *win.EventIterator + renderer *win.Renderer +} + +// Name returns the name of the event log (i.e. Application, Security, etc.). +func (l *winEventLogExp) Name() string { + return l.channelName +} + +func (l *winEventLogExp) Open(state checkpoint.EventLogState) error { + l.lastRead = state + + var err error + l.iterator, err = win.NewEventIterator( + win.WithSubscriptionFactory(func() (handle win.EvtHandle, err error) { + return l.open(l.lastRead) + }), + win.WithBatchSize(l.maxRead)) + return err +} + +func (l *winEventLogExp) open(state checkpoint.EventLogState) (win.EvtHandle, error) { + var bookmark win.Bookmark + if len(state.Bookmark) > 0 { + var err error + bookmark, err = win.NewBookmarkFromXML(state.Bookmark) + if err != nil { + return win.NilHandle, err + } + defer bookmark.Close() + } + + if l.file { + return l.openFile(state, bookmark) + } + return l.openChannel(bookmark) +} + +func (l *winEventLogExp) openChannel(bookmark win.Bookmark) (win.EvtHandle, error) { + // Using a pull subscription to receive events. See: + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull + signalEvent, err := windows.CreateEvent(nil, 0, 0, nil) + if err != nil { + return win.NilHandle, err + } + defer windows.CloseHandle(signalEvent) + + var flags win.EvtSubscribeFlag + if bookmark > 0 { + flags = win.EvtSubscribeStartAfterBookmark + } else { + flags = win.EvtSubscribeStartAtOldestRecord + } + + l.log.Debugw("Using subscription query.", "winlog.query", l.query) + return win.Subscribe( + 0, // Session - nil for localhost + signalEvent, + "", // Channel - empty b/c channel is in the query + l.query, // Query - nil means all events + win.EvtHandle(bookmark), // Bookmark - for resuming from a specific event + flags) +} + +func (l *winEventLogExp) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) { + path := l.channelName + + h, err := win.EvtQuery(0, path, "", win.EvtQueryFilePath|win.EvtQueryForwardDirection) + if err != nil { + return win.NilHandle, errors.Wrapf(err, "failed to get handle to event log file %v", path) + } + + if bookmark > 0 { + l.log.Debugf("Seeking to bookmark. timestamp=%v bookmark=%v", + state.Timestamp, state.Bookmark) + + // This seeks to the last read event and strictly validates that the + // bookmarked record number exists. + if err = win.EvtSeek(h, 0, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil { + // Then we advance past the last read event to avoid sending that + // event again. This won't fail if we're at the end of the file. + err = errors.Wrap( + win.EvtSeek(h, 1, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark), + "failed to seek past bookmarked position") + } else { + l.log.Warnf("s Failed to seek to bookmarked location in %v (error: %v). "+ + "Recovering by reading the log from the beginning. (Did the file "+ + "change since it was last read?)", path, err) + err = errors.Wrap( + win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst), + "failed to seek to beginning of log") + } + + if err != nil { + return win.NilHandle, err + } + } + + return h, err +} + +func (l *winEventLogExp) Read() ([]Record, error) { + var records []Record + + for h, ok := l.iterator.Next(); ok; h, ok = l.iterator.Next() { + record, err := l.processHandle(h) + if err != nil { + l.log.Warnw("Dropping event due to rendering error.", "error", err) + incrementMetric(dropReasons, err) + continue + } + records = append(records, *record) + + // It has read the maximum requested number of events. + if len(records) >= l.maxRead { + return records, nil + } + } + + // An error occurred while retrieving more events. + if err := l.iterator.Err(); err != nil { + return records, err + } + + // Reader is configured to stop when there are no more events. + if Stop == l.config.NoMoreEvents { + return records, io.EOF + } + + return records, nil +} + +func (l *winEventLogExp) processHandle(h win.EvtHandle) (*Record, error) { + defer h.Close() + + // NOTE: Render can return an error and a partial event. + evt, err := l.renderer.Render(h) + if evt == nil { + return nil, err + } + if err != nil { + evt.RenderErr = append(evt.RenderErr, err.Error()) + } + + // TODO: Need to add XML when configured. + + r := &Record{ + API: winEventLogExpAPIName, + Event: *evt, + } + + if l.file { + r.File = l.channelName + } + + r.Offset = checkpoint.EventLogState{ + Name: l.channelName, + RecordNumber: r.RecordID, + Timestamp: r.TimeCreated.SystemTime, + } + if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { + l.log.Warnw("Failed creating bookmark.", "error", err) + } + l.lastRead = r.Offset + return r, nil +} + +func (l *winEventLogExp) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) { + bookmark, err := win.NewBookmarkFromEvent(evtHandle) + if err != nil { + return "", errors.Wrap(err, "failed to create new bookmark from event handle") + } + defer bookmark.Close() + + return bookmark.XML() +} + +func (l *winEventLogExp) Close() error { + l.log.Debug("Closing event log reader handles.") + return multierr.Combine( + l.iterator.Close(), + l.renderer.Close(), + ) +} + +// newWinEventLogExp creates and returns a new EventLog for reading event logs +// using the Windows Event Log. +func newWinEventLogExp(options *common.Config) (EventLog, error) { + cfgwarn.Experimental("The %s event log reader is experimental.", winEventLogExpAPIName) + + c := winEventLogConfig{BatchReadSize: 512} + if err := readConfig(options, &c, winEventLogConfigKeys); err != nil { + return nil, err + } + + queryLog := c.Name + isFile := false + if info, err := os.Stat(c.Name); err == nil && info.Mode().IsRegular() { + path, err := filepath.Abs(c.Name) + if err != nil { + return nil, err + } + isFile = true + queryLog = "file://" + path + } + + query, err := win.Query{ + Log: queryLog, + IgnoreOlder: c.SimpleQuery.IgnoreOlder, + Level: c.SimpleQuery.Level, + EventID: c.SimpleQuery.EventID, + Provider: c.SimpleQuery.Provider, + }.Build() + if err != nil { + return nil, err + } + + log := logp.NewLogger("wineventlog").With("channel", c.Name) + + renderer, err := win.NewRenderer(win.NilHandle, log) + if err != nil { + return nil, err + } + + l := &winEventLogExp{ + config: c, + query: query, + channelName: c.Name, + file: isFile, + maxRead: c.BatchReadSize, + renderer: renderer, + log: log, + } + + return l, nil +} + +func init() { + // Register wineventlog API if it is available. + available, _ := win.IsAvailable() + if available { + Register(winEventLogExpAPIName, 10, newWinEventLogExp, win.Channels) + } +} diff --git a/winlogbeat/eventlog/wineventlog_test.go b/winlogbeat/eventlog/wineventlog_test.go index c37615551a7..a9205113f78 100644 --- a/winlogbeat/eventlog/wineventlog_test.go +++ b/winlogbeat/eventlog/wineventlog_test.go @@ -20,122 +20,191 @@ package eventlog import ( - "expvar" + "io" + "os/exec" "path/filepath" "strconv" + "strings" "testing" - elog "github.com/andrewkroh/sys/windows/svc/eventlog" + "github.com/andrewkroh/sys/windows/svc/eventlog" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/winlogbeat/checkpoint" + "github.com/elastic/beats/v7/winlogbeat/sys/wineventlog" ) -func TestWinEventLogBatchReadSize(t *testing.T) { - configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() +func TestWindowsEventLogAPI(t *testing.T) { + testWindowsEventLog(t, winEventLogAPIName) +} - // Publish test messages: - for k, m := range messages { - err = log.Report(m.eventType, k, []string{m.message}) +func TestWindowsEventLogAPIExperimental(t *testing.T) { + testWindowsEventLog(t, winEventLogExpAPIName) +} + +func testWindowsEventLog(t *testing.T, api string) { + writer, teardown := createLog(t) + defer teardown() + + setLogSize(t, providerName, gigabyte) + + // Publish large test messages. + const totalEvents = 1000 + for i := 0; i < totalEvents; i++ { + err := writer.Report(eventlog.Info, uint32(i%1000), []string{strconv.Itoa(i) + " " + randomSentence(31800)}) if err != nil { t.Fatal(err) } } - batchReadSize := 2 - eventlog, teardown := setupWinEventLog(t, 0, map[string]interface{}{"name": providerName, "batch_read_size": batchReadSize}) - defer teardown() - - records, err := eventlog.Read() - if err != nil { - t.Fatal(err) + openLog := func(t testing.TB, config map[string]interface{}) EventLog { + return openLog(t, api, nil, config) } - assert.Len(t, records, batchReadSize) -} + t.Run("batch_read_size_config", func(t *testing.T) { + const batchReadSize = 2 -// TestReadLargeBatchSize tests reading from an event log using a large -// read_batch_size parameter. When combined with large messages this causes -// EvtNext (wineventlog.EventRecords) to fail with RPC_S_INVALID_BOUND error. -func TestReadLargeBatchSize(t *testing.T) { - configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) + log := openLog(t, map[string]interface{}{"name": providerName, "batch_read_size": batchReadSize}) + defer log.Close() + + records, err := log.Read() if err != nil { t.Fatal(err) } - }() - setLogSize(t, providerName, gigabyte) + assert.Len(t, records, batchReadSize) + }) - // Publish large test messages. - totalEvents := 1000 - for i := 0; i < totalEvents; i++ { - err = log.Report(elog.Info, uint32(i%1000), []string{strconv.Itoa(i) + " " + randomSentence(31800)}) - if err != nil { - t.Fatal("ReportEvent error", err) + // Test reading from an event log using a large batch_read_size parameter. + // When combined with large messages this causes EvtNext to fail with + // RPC_S_INVALID_BOUND error. The reader should recover from the error. + t.Run("large_batch_read", func(t *testing.T) { + log := openLog(t, map[string]interface{}{"name": providerName, "batch_read_size": 1024}) + defer log.Close() + + var eventCount int + + for eventCount < totalEvents { + records, err := log.Read() + if err != nil { + t.Fatal("read error", err) + } + if len(records) == 0 { + t.Fatal("read returned 0 records") + } + + t.Logf("Read() returned %d events.", len(records)) + eventCount += len(records) } - } - eventlog, teardown := setupWinEventLog(t, 0, map[string]interface{}{"name": providerName, "batch_read_size": 1024}) - defer teardown() + assert.Equal(t, totalEvents, eventCount) + }) - var eventCount int - for eventCount < totalEvents { - records, err := eventlog.Read() + t.Run("evtx_file", func(t *testing.T) { + path, err := filepath.Abs("../sys/wineventlog/testdata/sysmon-9.01.evtx") if err != nil { - t.Fatal("read error", err) - } - if len(records) == 0 { - t.Fatal("read returned 0 records") + t.Fatal(err) } - eventCount += len(records) - } - t.Logf("number of records returned: %v", eventCount) + log := openLog(t, map[string]interface{}{ + "name": path, + "no_more_events": "stop", + }) + defer log.Close() - wineventlog := eventlog.(*winEventLog) - assert.Equal(t, 1024, wineventlog.maxRead) + records, err := log.Read() + + // This implementation returns the EOF on the next call. + if err == nil && api == winEventLogAPIName { + _, err = log.Read() + } - expvar.Do(func(kv expvar.KeyValue) { - if kv.Key == "read_errors" { - t.Log(kv) + if assert.Error(t, err, "no_more_events=stop requires io.EOF to be returned") { + assert.Equal(t, io.EOF, err) } + + assert.Len(t, records, 32) }) } -func TestReadEvtxFile(t *testing.T) { - path, err := filepath.Abs("../sys/wineventlog/testdata/sysmon-9.01.evtx") +// ---- Utility Functions ----- + +// createLog creates a new event log and returns a handle for writing events +// to the log. +func createLog(t testing.TB, messageFiles ...string) (log *eventlog.Log, tearDown func()) { + const name = providerName + const source = sourceName + + messageFile := eventCreateMsgFile + if len(messageFiles) > 0 { + messageFile = strings.Join(messageFiles, ";") + } + + existed, err := eventlog.Install(name, source, messageFile, true, eventlog.Error|eventlog.Warning|eventlog.Info) if err != nil { t.Fatal(err) } - configureLogp() - eventlog, teardown := setupWinEventLog(t, 0, map[string]interface{}{ - "name": path, - }) - defer teardown() + if existed { + wineventlog.EvtClearLog(wineventlog.NilHandle, name, "") + } - records, err := eventlog.Read() + log, err = eventlog.Open(source) if err != nil { + eventlog.RemoveSource(name, source) + eventlog.RemoveProvider(name) t.Fatal(err) } - assert.Len(t, records, 32) + tearDown = func() { + log.Close() + wineventlog.EvtClearLog(wineventlog.NilHandle, name, "") + eventlog.RemoveSource(name, source) + eventlog.RemoveProvider(name) + } + + return log, tearDown } -func setupWinEventLog(t *testing.T, recordID uint64, options map[string]interface{}) (EventLog, func()) { - return setupEventLog(t, newWinEventLog, recordID, options) +// setLogSize set the maximum number of bytes that an event log can hold. +func setLogSize(t testing.TB, provider string, sizeBytes int) { + output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), provider).CombinedOutput() + if err != nil { + t.Fatal("Failed to set log size", err, string(output)) + } +} + +func openLog(t testing.TB, api string, state *checkpoint.EventLogState, config map[string]interface{}) EventLog { + cfg, err := common.NewConfigFrom(config) + if err != nil { + t.Fatal(err) + } + + var log EventLog + switch api { + case winEventLogAPIName: + log, err = newWinEventLog(cfg) + case winEventLogExpAPIName: + log, err = newWinEventLogExp(cfg) + case eventLoggingAPIName: + log, err = newEventLogging(cfg) + default: + t.Fatalf("Unknown API name: '%s'", api) + } + if err != nil { + t.Fatal(err) + } + + var eventLogState checkpoint.EventLogState + if state != nil { + eventLogState = *state + } + + if err = log.Open(eventLogState); err != nil { + log.Close() + t.Fatal(err) + } + + return log } diff --git a/winlogbeat/sys/event.go b/winlogbeat/sys/event.go index 50df9ec18d3..d88617d8925 100644 --- a/winlogbeat/sys/event.go +++ b/winlogbeat/sys/event.go @@ -41,6 +41,7 @@ type Event struct { LevelRaw uint8 `xml:"System>Level"` TaskRaw uint16 `xml:"System>Task"` OpcodeRaw uint8 `xml:"System>Opcode"` + KeywordsRaw HexInt64 `xml:"System>Keywords"` TimeCreated TimeCreated `xml:"System>TimeCreated"` RecordID uint64 `xml:"System>EventRecordID"` Correlation Correlation `xml:"System>Correlation"` @@ -96,7 +97,7 @@ type Execution struct { ProcessorTime uint32 `xml:"ProcessorTime,attr"` } -// EventIdentifier is the identifer that the provider uses to identify a +// EventIdentifier is the identifier that the provider uses to identify a // specific event type. type EventIdentifier struct { Qualifiers uint16 `xml:"Qualifiers,attr"` @@ -225,3 +226,21 @@ func (v *Version) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { *v = Version(version) return nil } + +type HexInt64 uint64 + +func (v *HexInt64) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var s string + if err := d.DecodeElement(&s, &start); err != nil { + return err + } + + num, err := strconv.ParseInt(s, 0, 64) + if err != nil { + // Ignore invalid version values. + return nil + } + + *v = HexInt64(num) + return nil +} diff --git a/winlogbeat/sys/event_test.go b/winlogbeat/sys/event_test.go index 0684e99b473..8d0f6ee04f8 100644 --- a/winlogbeat/sys/event_test.go +++ b/winlogbeat/sys/event_test.go @@ -94,6 +94,7 @@ func TestXML(t *testing.T) { EventIdentifier: EventIdentifier{ID: 91}, LevelRaw: 4, TaskRaw: 9, + KeywordsRaw: 0x4000000000000004, TimeCreated: TimeCreated{allXMLTimeCreated}, RecordID: 100, Correlation: Correlation{"{A066CCF1-8AB3-459B-B62F-F79F957A5036}", "{85FC0930-9C49-42DA-804B-A7368104BD1B}"}, diff --git a/winlogbeat/sys/wineventlog/bookmark.go b/winlogbeat/sys/wineventlog/bookmark.go new file mode 100644 index 00000000000..fa806aa2c34 --- /dev/null +++ b/winlogbeat/sys/wineventlog/bookmark.go @@ -0,0 +1,81 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "syscall" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +// Bookmark is a handle to an event log bookmark. +type Bookmark EvtHandle + +// Close closes the bookmark handle. +func (b Bookmark) Close() error { + return EvtHandle(b).Close() +} + +// XML returns the bookmark's value as XML. +func (b Bookmark) XML() (string, error) { + var bufferUsed uint32 + + err := _EvtRender(NilHandle, EvtHandle(b), EvtRenderBookmark, 0, nil, &bufferUsed, nil) + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { + return "", errors.Wrap(err, "failed to determine necessary buffer size for EvtRender") + } + + bb := newByteBuffer() + bb.Reserve(int(bufferUsed * 2)) + defer bb.free() + + err = _EvtRender(NilHandle, EvtHandle(b), EvtRenderBookmark, uint32(len(bb.buf)), &bb.buf[0], &bufferUsed, nil) + if err != nil { + return "", errors.Wrap(err, "failed to render bookmark XML") + } + + return UTF16BytesToString(bb.buf) +} + +// NewBookmarkFromEvent returns a Bookmark pointing to the given event record. +// The returned handle must be closed. +func NewBookmarkFromEvent(eventHandle EvtHandle) (Bookmark, error) { + h, err := _EvtCreateBookmark(nil) + if err != nil { + return 0, err + } + if err = _EvtUpdateBookmark(h, eventHandle); err != nil { + h.Close() + return 0, err + } + return Bookmark(h), nil +} + +// NewBookmarkFromXML returns a Bookmark created from an XML bookmark. +// The returned handle must be closed. +func NewBookmarkFromXML(xml string) (Bookmark, error) { + utf16, err := syscall.UTF16PtrFromString(xml) + if err != nil { + return 0, err + } + h, err := _EvtCreateBookmark(utf16) + return Bookmark(h), err +} diff --git a/winlogbeat/sys/wineventlog/bookmark_test.go b/winlogbeat/sys/wineventlog/bookmark_test.go new file mode 100644 index 00000000000..34a443a4184 --- /dev/null +++ b/winlogbeat/sys/wineventlog/bookmark_test.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBookmark(t *testing.T) { + log := openLog(t, security4752File) + defer log.Close() + + evtHandle := mustNextHandle(t, log) + defer evtHandle.Close() + + t.Run("NewBookmarkFromEvent", func(t *testing.T) { + bookmark, err := NewBookmarkFromEvent(evtHandle) + if err != nil { + t.Fatal(err) + } + defer func() { + assert.NoError(t, bookmark.Close()) + }() + + xml, err := bookmark.XML() + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, xml, "") + }) + + t.Run("NewBookmarkFromXML", func(t *testing.T) { + const savedBookmarkXML = ` + + +` + + bookmark, err := NewBookmarkFromXML(savedBookmarkXML) + if err != nil { + t.Fatal(err) + } + defer func() { + assert.NoError(t, bookmark.Close()) + }() + + xml, err := bookmark.XML() + if err != nil { + t.Fatal(err) + } + + // Ignore whitespace differences. + normalizer := strings.NewReplacer(" ", "", "\r\n", "", "\n", "") + assert.Equal(t, normalizer.Replace(savedBookmarkXML), normalizer.Replace(xml)) + }) + + t.Run("NewBookmarkFromEvent_invalid", func(t *testing.T) { + bookmark, err := NewBookmarkFromEvent(NilHandle) + assert.Error(t, err) + assert.Zero(t, bookmark) + }) + + t.Run("NewBookmarkFromXML_invalid", func(t *testing.T) { + bookmark, err := NewBookmarkFromXML("{Not XML}") + assert.Error(t, err) + assert.Zero(t, bookmark) + }) +} diff --git a/winlogbeat/sys/wineventlog/bufferpool.go b/winlogbeat/sys/wineventlog/bufferpool.go new file mode 100644 index 00000000000..104d45f938a --- /dev/null +++ b/winlogbeat/sys/wineventlog/bufferpool.go @@ -0,0 +1,113 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package wineventlog + +import ( + "sync" + + "github.com/elastic/beats/v7/winlogbeat/sys" +) + +// bufferPool contains a pool of byteBuffer objects. +var bufferPool = sync.Pool{ + New: func() interface{} { return &byteBuffer{buf: make([]byte, 1024)} }, +} + +// byteBuffer is an expandable buffer backed by a byte slice. +type byteBuffer struct { + buf []byte + offset int +} + +// newByteBuffer return a byteBuffer from the pool. The returned value must +// be released with free(). +func newByteBuffer() *byteBuffer { + b := bufferPool.Get().(*byteBuffer) + b.Reset() + return b +} + +// free returns the byteBuffer to the pool. +func (b *byteBuffer) free() { + if b == nil { + return + } + bufferPool.Put(b) +} + +// Write appends the contents of p to the buffer, growing the buffer as needed. +// The return value is the length of p; err is always nil. This implements +// io.Writer. +func (b *byteBuffer) Write(p []byte) (int, error) { + if len(b.buf) < b.offset+len(p) { + // Create a buffer larger than needed so we don't spend lots of time + // allocating and copying. + spaceNeeded := len(b.buf) - b.offset + len(p) + largerBuf := make([]byte, 2*len(b.buf)+spaceNeeded) + copy(largerBuf, b.buf[:b.offset]) + b.buf = largerBuf + } + n := copy(b.buf[b.offset:], p) + b.offset += n + return n, nil +} + +// Reset resets the buffer to be empty. It retains the same underlying storage +// capacity. +func (b *byteBuffer) Reset() { + b.offset = 0 + b.buf = b.buf[:cap(b.buf)] +} + +// Bytes returns a slice of length b.Len() holding the bytes that have been +// written to the buffer. +func (b *byteBuffer) Bytes() []byte { + return b.buf[:b.offset] +} + +// Len returns the number of bytes that have been written to the buffer. +func (b *byteBuffer) Len() int { + return b.offset +} + +// Reserve reserves n bytes by increasing the buffer's length. It may allocate +// a new underlying buffer discarding any existing contents. +func (b *byteBuffer) Reserve(n int) { + b.offset = n + + if n > cap(b.buf) { + // Allocate new larger buffer with len=n. + b.buf = make([]byte, n) + } else { + b.buf = b.buf[:n] + } +} + +// UTF16BytesToString converts the given UTF-16 bytes to a string. +func UTF16BytesToString(b []byte) (string, error) { + // Use space from the byteBuffer pool as working memory for the conversion. + bb := newByteBuffer() + defer bb.free() + + if err := sys.UTF16ToUTF8Bytes(b, bb); err != nil { + return "", err + } + + // This copies the UTF-8 bytes to create a string. + return string(bb.Bytes()), nil +} diff --git a/winlogbeat/sys/wineventlog/doc.go b/winlogbeat/sys/wineventlog/doc.go index 7c3936a0fb7..09c8685c331 100644 --- a/winlogbeat/sys/wineventlog/doc.go +++ b/winlogbeat/sys/wineventlog/doc.go @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -/* -Package wineventlog provides access to the Windows Event Log API used in -all versions of Windows since Vista (i.e. Windows 7+ and Windows Server 2008+). -This is distinct from the Event Logging API that was used in Windows XP, -Windows Server 2003, and Windows 2000. -*/ +// Package wineventlog provides access to the Windows Event Log API used in +// all versions of Windows since Vista (i.e. Windows 7+ and Windows Server 2008+). +// This is distinct from the Event Logging API that was used in Windows XP, +// Windows Server 2003, and Windows 2000. package wineventlog + +// Add -trace to enable debug prints around syscalls. +//go:generate go get golang.org/x/sys/windows/mkwinsyscall +//go:generate $GOPATH/bin/mkwinsyscall.exe -systemdll -output zsyscall_windows.go syscall_windows.go diff --git a/winlogbeat/sys/wineventlog/format_message.go b/winlogbeat/sys/wineventlog/format_message.go new file mode 100644 index 00000000000..e8befcdeae3 --- /dev/null +++ b/winlogbeat/sys/wineventlog/format_message.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "unsafe" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +// getMessageStringFromHandle returns the message for the given eventHandle. +func getMessageStringFromHandle(metadata *PublisherMetadata, eventHandle EvtHandle, values []EvtVariant) (string, error) { + return getMessageString(metadata, eventHandle, 0, values) +} + +// getMessageStringFromMessageID returns the message associated with the given +// message ID. +func getMessageStringFromMessageID(metadata *PublisherMetadata, messageID uint32, values []EvtVariant) (string, error) { + return getMessageString(metadata, NilHandle, messageID, values) +} + +// getMessageString returns an event's message. Don't use this directly. Instead +// use either getMessageStringFromHandle or getMessageStringFromMessageID. +func getMessageString(metadata *PublisherMetadata, eventHandle EvtHandle, messageID uint32, values []EvtVariant) (string, error) { + var flags EvtFormatMessageFlag + if eventHandle > 0 { + flags = EvtFormatMessageEvent + } else if messageID > 0 { + flags = EvtFormatMessageId + } + + metadataHandle := NilHandle + if metadata != nil { + metadataHandle = metadata.Handle + } + + return evtFormatMessage(metadataHandle, eventHandle, messageID, values, flags) +} + +// getEventXML returns all data in the event as XML. +func getEventXML(metadata *PublisherMetadata, eventHandle EvtHandle) (string, error) { + metadataHandle := NilHandle + if metadata != nil { + metadataHandle = metadata.Handle + } + return evtFormatMessage(metadataHandle, eventHandle, 0, nil, EvtFormatMessageXml) +} + +// evtFormatMessage uses EvtFormatMessage to generate a string. +func evtFormatMessage(metadataHandle EvtHandle, eventHandle EvtHandle, messageID uint32, values []EvtVariant, messageFlag EvtFormatMessageFlag) (string, error) { + var ( + valuesCount = uint32(len(values)) + valuesPtr uintptr + ) + if len(values) > 0 { + valuesPtr = uintptr(unsafe.Pointer(&values[0])) + } + + // Determine the buffer size needed (given in WCHARs). + var bufferUsed uint32 + err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, 0, nil, &bufferUsed) + if err != windows.ERROR_INSUFFICIENT_BUFFER { + return "", errors.Wrap(err, "failed in EvtFormatMessage") + } + + // Get a buffer from the pool and adjust its length. + bb := newByteBuffer() + defer bb.free() + bb.Reserve(int(bufferUsed * 2)) + + err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, uint32(len(bb.buf)/2), &bb.buf[0], &bufferUsed) + if err != nil { + switch err { + // Ignore some errors so it can tolerate missing or mismatched parameter values. + case windows.ERROR_EVT_UNRESOLVED_VALUE_INSERT: + case windows.ERROR_EVT_UNRESOLVED_PARAMETER_INSERT: + case windows.ERROR_EVT_MAX_INSERTS_REACHED: + default: + return "", errors.Wrap(err, "failed in EvtFormatMessage") + } + } + + return UTF16BytesToString(bb.buf) +} diff --git a/winlogbeat/sys/wineventlog/format_message_test.go b/winlogbeat/sys/wineventlog/format_message_test.go new file mode 100644 index 00000000000..492061f7a28 --- /dev/null +++ b/winlogbeat/sys/wineventlog/format_message_test.go @@ -0,0 +1,153 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFormatMessage(t *testing.T) { + log := openLog(t, security4752File) + defer log.Close() + + evtHandle := mustNextHandle(t, log) + defer evtHandle.Close() + + publisherMetadata, err := NewPublisherMetadata(NilHandle, "Microsoft-Windows-Security-Auditing") + if err != nil { + t.Fatal(err) + } + defer publisherMetadata.Close() + + t.Run("getMessageStringFromHandle", func(t *testing.T) { + t.Run("no_metadata", func(t *testing.T) { + // Metadata is required unless the events were forwarded with "RenderedText". + _, err := getMessageStringFromHandle(nil, evtHandle, nil) + assert.Error(t, err) + }) + + t.Run("with_metadata", func(t *testing.T) { + // When no values are passed in then event data from the event is + // substituted into the message. + msg, err := getMessageStringFromHandle(publisherMetadata, evtHandle, nil) + if err != nil { + t.Fatal(err) + } + assert.Contains(t, msg, "CN=Administrator,CN=Users,DC=TEST,DC=SAAS") + }) + + t.Run("custom_values", func(t *testing.T) { + // Substitute custom values into the message. + msg, err := getMessageStringFromHandle(publisherMetadata, evtHandle, templateInserts.Slice()) + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, msg, `{{eventParam $ 2}}`) + + // NOTE: In this test case I noticed the messages contains + // "Logon ID: 0x0" + // but it should contain + // "Logon ID: {{eventParam $ 9}}" + // + // This may mean that certain windows.GUID values cannot be + // substituted with string values. So we shouldn't rely on this + // method to create text/templates. Instead we can use the + // getMessageStringFromMessageID (see test below) that works as + // expected. + assert.NotContains(t, msg, `{{eventParam $ 9}}`) + }) + }) + + t.Run("getMessageStringFromMessageID", func(t *testing.T) { + // Get the message ID for event 4752. + itr, err := publisherMetadata.EventMetadataIterator() + if err != nil { + t.Fatal(err) + } + defer itr.Close() + + var messageID uint32 + for itr.Next() { + id, err := itr.EventID() + if err != nil { + t.Fatal(err) + } + if id == 4752 { + messageID, err = itr.MessageID() + if err != nil { + t.Fatal(err) + } + } + } + + if messageID == 0 { + t.Fatal("message ID for event 4752 not found") + } + + t.Run("no_metadata", func(t *testing.T) { + // Metadata is required to find the message file. + _, err := getMessageStringFromMessageID(nil, messageID, nil) + assert.Error(t, err) + }) + + t.Run("with_metadata", func(t *testing.T) { + // When no values are passed in then the raw message is returned + // with place-holders like %1 and %2. + msg, err := getMessageStringFromMessageID(publisherMetadata, messageID, nil) + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, msg, "%9") + }) + + t.Run("custom_values", func(t *testing.T) { + msg, err := getMessageStringFromMessageID(publisherMetadata, messageID, templateInserts.Slice()) + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, msg, `{{eventParam $ 2}}`) + assert.Contains(t, msg, `{{eventParam $ 9}}`) + }) + }) + + t.Run("getEventXML", func(t *testing.T) { + t.Run("no_metadata", func(t *testing.T) { + // It needs the metadata handle to add the message to the XML. + _, err := getEventXML(nil, evtHandle) + assert.Error(t, err) + }) + + t.Run("with_metadata", func(t *testing.T) { + xml, err := getEventXML(publisherMetadata, evtHandle) + if err != nil { + t.Fatal(err) + } + + assert.True(t, strings.HasPrefix(xml, "")) + }) + }) +} diff --git a/winlogbeat/sys/wineventlog/iterator.go b/winlogbeat/sys/wineventlog/iterator.go new file mode 100644 index 00000000000..26492fe96b0 --- /dev/null +++ b/winlogbeat/sys/wineventlog/iterator.go @@ -0,0 +1,207 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "sync" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +const ( + evtNextMaxHandles = 1024 + evtNextDefaultHandles = 512 +) + +// EventIterator provides an iterator to read events from a log. It takes the +// place of calling EvtNext directly. +type EventIterator struct { + subscriptionFactory SubscriptionFactory // Factory for producing a new subscription handle. + subscription EvtHandle // Handle from EvtQuery or EvtSubscribe. + batchSize uint32 // Number of handles to request by default. + handles [evtNextMaxHandles]EvtHandle // Handles returned by EvtNext. + lastErr error // Last error returned by EvtNext. + active []EvtHandle // Slice of the handles array containing the valid unread handles. + mutex sync.Mutex // Mutex to enable parallel iteration. + + // For testing purposes to be able to mock EvtNext. + evtNext func(resultSet EvtHandle, eventArraySize uint32, eventArray *EvtHandle, timeout uint32, flags uint32, numReturned *uint32) (err error) +} + +// SubscriptionFactory produces a handle from EvtQuery or EvtSubscribe that +// points to the next unread event. Provide a factory to enable automatic +// recover of certain errors. +type SubscriptionFactory func() (EvtHandle, error) + +// EventIteratorOption represents a configuration of for the construction of +// the EventIterator. +type EventIteratorOption func(*EventIterator) + +// WithSubscriptionFactory configures a SubscriptionFactory for the iterator to +// use to create a subscription handle. +func WithSubscriptionFactory(factory SubscriptionFactory) EventIteratorOption { + return func(itr *EventIterator) { + itr.subscriptionFactory = factory + } +} + +// WithSubscription configures the iterator with an existing subscription handle. +func WithSubscription(subscription EvtHandle) EventIteratorOption { + return func(itr *EventIterator) { + itr.subscription = subscription + } +} + +// WithBatchSize configures the number of handles the iterator will request +// when calling EvtNext. Valid batch sizes range on [1, 1024]. +func WithBatchSize(size int) EventIteratorOption { + return func(itr *EventIterator) { + if size > 0 { + itr.batchSize = uint32(size) + } + if size > evtNextMaxHandles { + itr.batchSize = evtNextMaxHandles + } + } +} + +// NewEventIterator creates an iterator to read event handles from a subscription. +// The iterator is thread-safe. +func NewEventIterator(opts ...EventIteratorOption) (*EventIterator, error) { + itr := &EventIterator{ + batchSize: evtNextDefaultHandles, + evtNext: _EvtNext, + } + + for _, opt := range opts { + opt(itr) + } + + if itr.subscriptionFactory == nil && itr.subscription == NilHandle { + return nil, errors.New("either a subscription or subscription factory is required") + } + + if itr.subscription == NilHandle { + handle, err := itr.subscriptionFactory() + if err != nil { + return nil, err + } + itr.subscription = handle + } + + return itr, nil +} + +// Next advances the iterator to the next handle. After Next returns false, the +// Err() method will return any error that occurred during iteration, except +// that if it was windows.ERROR_NO_MORE_ITEMS, Err() will return nil and you +// may call Next() again later to check if new events are available. +func (itr *EventIterator) Next() (EvtHandle, bool) { + itr.mutex.Lock() + defer itr.mutex.Unlock() + + if itr.lastErr != nil { + return NilHandle, false + } + + if !itr.empty() { + itr.active = itr.active[1:] + } + + if itr.empty() && !itr.moreHandles() { + return NilHandle, false + } + + return itr.active[0], true +} + +// empty returns true when there are no more handles left to read from memory. +func (itr *EventIterator) empty() bool { + return len(itr.active) == 0 +} + +// moreHandles fetches more handles using EvtNext. It returns true if it +// successfully fetched more handles. +func (itr *EventIterator) moreHandles() bool { + batchSize := itr.batchSize + + for batchSize > 0 { + var numReturned uint32 + + err := itr.evtNext(itr.subscription, batchSize, &itr.handles[0], 0, 0, &numReturned) + switch err { + case nil: + itr.lastErr = nil + itr.active = itr.handles[:numReturned] + case windows.ERROR_NO_MORE_ITEMS, windows.ERROR_INVALID_OPERATION: + case windows.RPC_S_INVALID_BOUND: + // Attempt automated recovery if we have a factory. + if itr.subscriptionFactory != nil { + itr.subscription.Close() + itr.subscription, err = itr.subscriptionFactory() + if err != nil { + itr.lastErr = errors.Wrap(err, "failed in EvtNext while trying to "+ + "recover from RPC_S_INVALID_BOUND error") + return false + } + + // Reduce batch size and try again. + batchSize = batchSize / 2 + continue + } else { + itr.lastErr = errors.Wrap(err, "failed in EvtNext (try "+ + "reducing the batch size or providing a subscription "+ + "factory for automatic recovery)") + } + default: + itr.lastErr = err + } + + break + } + + return !itr.empty() +} + +// Err returns the first non-ERROR_NO_MORE_ITEMS error encountered by the +// EventIterator. +// +// Some Windows versions will fail with windows.RPC_S_INVALID_BOUND when the +// batch size is too large. If this occurs you can recover by closing the +// iterator, creating a new subscription, seeking to the next unread event, and +// creating a new EventIterator with a smaller batch size. +func (itr *EventIterator) Err() error { + itr.mutex.Lock() + defer itr.mutex.Unlock() + + return itr.lastErr +} + +// Close closes the subscription handle and any unread event handles. +func (itr *EventIterator) Close() error { + itr.mutex.Lock() + defer itr.mutex.Unlock() + + for _, h := range itr.active { + h.Close() + } + return itr.subscription.Close() +} diff --git a/winlogbeat/sys/wineventlog/iterator_test.go b/winlogbeat/sys/wineventlog/iterator_test.go new file mode 100644 index 00000000000..fee6553f36c --- /dev/null +++ b/winlogbeat/sys/wineventlog/iterator_test.go @@ -0,0 +1,271 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strconv" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestEventIterator(t *testing.T) { + logp.TestingSetup() + + writer, tearDown := createLog(t) + defer tearDown() + + const eventCount = 1500 + for i := 0; i < eventCount; i++ { + if err := writer.Info(1, "Test message "+strconv.Itoa(i+1)); err != nil { + t.Fatal(err) + } + } + + // Validate the assumption that 1024 is the max number of handles supported + // by EvtNext. + t.Run("max_handles_assumption", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + var ( + numReturned uint32 + handles = [evtNextMaxHandles + 1]EvtHandle{} + ) + + // Too many handles. + err := _EvtNext(log, uint32(len(handles)), &handles[0], 0, 0, &numReturned) + assert.Equal(t, windows.ERROR_INVALID_PARAMETER, err) + + // The max number of handles. + err = _EvtNext(log, evtNextMaxHandles, &handles[0], 0, 0, &numReturned) + if assert.NoError(t, err) { + for _, h := range handles[:numReturned] { + h.Close() + } + } + }) + + t.Run("no_subscription", func(t *testing.T) { + _, err := NewEventIterator() + assert.Error(t, err) + }) + + t.Run("with_subscription", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + itr, err := NewEventIterator(WithSubscription(log)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + assert.Nil(t, itr.subscriptionFactory) + assert.NotEqual(t, NilHandle, itr.subscription) + }) + + t.Run("with_subscription_factory", func(t *testing.T) { + factory := func() (handle EvtHandle, err error) { + return openLog(t, winlogbeatTestLogName), nil + } + itr, err := NewEventIterator(WithSubscriptionFactory(factory)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + assert.NotNil(t, itr.subscriptionFactory) + assert.NotEqual(t, NilHandle, itr.subscription) + }) + + t.Run("with_batch_size", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + t.Run("default", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log)) + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, evtNextDefaultHandles, itr.batchSize) + }) + + t.Run("custom", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(128)) + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, 128, itr.batchSize) + }) + + t.Run("too_small", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(0)) + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, evtNextDefaultHandles, itr.batchSize) + }) + + t.Run("too_big", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(evtNextMaxHandles+1)) + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, evtNextMaxHandles, itr.batchSize) + }) + }) + + t.Run("iterate", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(13)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + var iterateCount int + for h, ok := itr.Next(); ok; h, ok = itr.Next() { + h.Close() + + if !assert.NotZero(t, h) { + return + } + + iterateCount++ + } + if err := itr.Err(); err != nil { + t.Fatal(err) + } + + assert.EqualValues(t, eventCount, iterateCount) + }) + + // Check for regressions of https://github.com/elastic/beats/issues/3076 + // where EvtNext fails reading batch of large events. + // + // Note: As of 2020-03 Windows 2019 no longer exhibits this behavior. + // Instead EvtNext simply returns fewer handles that the requested size. + t.Run("rpc_error", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + // Mock the behavior to simplify testing since it's not reproducible + // on all Windows versions. + mockEvtNext := func(resultSet EvtHandle, eventArraySize uint32, eventArray *EvtHandle, timeout uint32, flags uint32, numReturned *uint32) (err error) { + if eventArraySize > 3 { + return windows.RPC_S_INVALID_BOUND + } + return _EvtNext(resultSet, eventArraySize, eventArray, timeout, flags, numReturned) + } + + // If you create the iterator with only a subscription handle then + // no recovery is possible without data loss. + t.Run("no_recovery", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + itr.evtNext = mockEvtNext + + h, ok := itr.Next() + assert.False(t, ok) + assert.Zero(t, h) + if assert.Error(t, itr.Err()) { + assert.Contains(t, itr.Err().Error(), "try reducing the batch size") + assert.Equal(t, windows.RPC_S_INVALID_BOUND, errors.Cause(itr.Err())) + } + }) + + t.Run("automated_recovery", func(t *testing.T) { + var numFactoryInvocations int + var bookmark Bookmark + + // Create a proper subscription factor that resumes from the last + // read position by using bookmarks. + factory := func() (handle EvtHandle, err error) { + numFactoryInvocations++ + log := openLog(t, winlogbeatTestLogName) + + if bookmark != 0 { + // Seek to bookmark location. + err := EvtSeek(log, 0, EvtHandle(bookmark), EvtSeekRelativeToBookmark|EvtSeekStrict) + if err != nil { + t.Fatal(err) + } + + // Seek to one event after bookmark (unread position). + if err = EvtSeek(log, 1, NilHandle, EvtSeekRelativeToCurrent); err != nil { + t.Fatal(err) + } + } + + return log, err + } + + itr, err := NewEventIterator(WithSubscriptionFactory(factory), WithBatchSize(10)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + // Mock the EvtNext to cause the the RPC_S_INVALID_BOUND error. + itr.evtNext = mockEvtNext + + var iterateCount int + for h, ok := itr.Next(); ok; h, ok = itr.Next() { + func() { + defer h.Close() + + if !assert.NotZero(t, h) { + t.FailNow() + } + + // Store last read position. + if bookmark != 0 { + bookmark.Close() + } + bookmark, err = NewBookmarkFromEvent(h) + if err != nil { + t.Fatal(err) + } + + iterateCount++ + }() + } + if err := itr.Err(); err != nil { + t.Fatal(err) + } + + // Validate that the factory has been used to recover and + // that we received all the events. + assert.Greater(t, numFactoryInvocations, 1) + assert.EqualValues(t, eventCount, iterateCount) + }) + }) +} diff --git a/winlogbeat/sys/wineventlog/metadata_store.go b/winlogbeat/sys/wineventlog/metadata_store.go new file mode 100644 index 00000000000..e59294f6276 --- /dev/null +++ b/winlogbeat/sys/wineventlog/metadata_store.go @@ -0,0 +1,463 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strconv" + "strings" + "sync" + "text/template" + + "github.com/pkg/errors" + "go.uber.org/multierr" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys" +) + +var ( + // eventDataNameTransform removes spaces from parameter names. + eventDataNameTransform = strings.NewReplacer(" ", "_") + + // eventMessageTemplateFuncs contains functions for use in message templates. + eventMessageTemplateFuncs = template.FuncMap{ + "eventParam": eventParam, + } +) + +// publisherMetadataStore stores metadata from a publisher. +type publisherMetadataStore struct { + Metadata *PublisherMetadata // Handle to the publisher metadata. May be nil. + Keywords map[int64]string // Keyword bit mask to keyword name. + Opcodes map[uint8]string // Opcode value to name. + Levels map[uint8]string // Level value to name. + Tasks map[uint16]string // Task value to name. + + // Event ID to event metadata (message and event data param names). + Events map[uint16]*eventMetadata + // Event ID to map of fingerprints to event metadata. The fingerprint value + // is hash of the event data parameters count and types. + EventFingerprints map[uint16]map[uint64]*eventMetadata + + mutex sync.RWMutex + log *logp.Logger +} + +func newPublisherMetadataStore(session EvtHandle, provider string, log *logp.Logger) (*publisherMetadataStore, error) { + md, err := NewPublisherMetadata(session, provider) + if err != nil { + return nil, err + } + store := &publisherMetadataStore{ + Metadata: md, + EventFingerprints: map[uint16]map[uint64]*eventMetadata{}, + log: log.With("publisher", provider), + } + + // Query the provider metadata to build an in-memory cache of the + // information to optimize event reading. + err = multierr.Combine( + store.initKeywords(), + store.initOpcodes(), + store.initLevels(), + store.initTasks(), + store.initEvents(), + ) + if err != nil { + return nil, err + } + + return store, nil +} + +// newEmptyPublisherMetadataStore creates an empty metadata store for cases +// where no local publisher metadata exists. +func newEmptyPublisherMetadataStore(provider string, log *logp.Logger) *publisherMetadataStore { + return &publisherMetadataStore{ + Keywords: map[int64]string{}, + Opcodes: map[uint8]string{}, + Levels: map[uint8]string{}, + Tasks: map[uint16]string{}, + Events: map[uint16]*eventMetadata{}, + EventFingerprints: map[uint16]map[uint64]*eventMetadata{}, + log: log.With("publisher", provider, "empty", true), + } +} + +func (s *publisherMetadataStore) initKeywords() error { + keywords, err := s.Metadata.Keywords() + if err != nil { + return err + } + + s.Keywords = make(map[int64]string, len(keywords)) + for _, keywordMeta := range keywords { + val := keywordMeta.Name + if val == "" { + val = keywordMeta.Message + } + s.Keywords[int64(keywordMeta.Mask)] = val + } + return nil +} + +func (s *publisherMetadataStore) initOpcodes() error { + opcodes, err := s.Metadata.Opcodes() + if err != nil { + return err + } + s.Opcodes = make(map[uint8]string, len(opcodes)) + for _, opcodeMeta := range opcodes { + val := opcodeMeta.Message + if val == "" { + val = opcodeMeta.Name + } + s.Opcodes[uint8(opcodeMeta.Mask)] = val + } + return nil +} + +func (s *publisherMetadataStore) initLevels() error { + levels, err := s.Metadata.Levels() + if err != nil { + return err + } + + s.Levels = make(map[uint8]string, len(levels)) + for _, levelMeta := range levels { + val := levelMeta.Name + if val == "" { + val = levelMeta.Message + } + s.Levels[uint8(levelMeta.Mask)] = val + } + return nil +} + +func (s *publisherMetadataStore) initTasks() error { + tasks, err := s.Metadata.Tasks() + if err != nil { + return err + } + s.Tasks = make(map[uint16]string, len(tasks)) + for _, taskMeta := range tasks { + val := taskMeta.Message + if val == "" { + val = taskMeta.Name + } + s.Tasks[uint16(taskMeta.Mask)] = val + } + return nil +} + +func (s *publisherMetadataStore) initEvents() error { + itr, err := s.Metadata.EventMetadataIterator() + if err != nil { + return err + } + defer itr.Close() + + s.Events = map[uint16]*eventMetadata{} + for itr.Next() { + evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata) + if err != nil { + s.log.Warnw("Failed to read event metadata from publisher. Continuing to next event.", + "error", err) + continue + } + s.Events[evt.EventID] = evt + } + return itr.Err() +} + +func (s *publisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *eventMetadata { + // Use a read lock to get a cached value. + s.mutex.RLock() + fingerprints, found := s.EventFingerprints[eventID] + if found { + em, found := fingerprints[eventDataFingerprint] + if found { + s.mutex.RUnlock() + return em + } + } + + // Elevate to write lock. + s.mutex.RUnlock() + s.mutex.Lock() + defer s.mutex.Unlock() + + fingerprints, found = s.EventFingerprints[eventID] + if !found { + fingerprints = map[uint64]*eventMetadata{} + s.EventFingerprints[eventID] = fingerprints + } + + em, found := fingerprints[eventDataFingerprint] + if found { + return em + } + + // To ensure we always match the correct event data parameter names to + // values we will rely a fingerprint made of the number of event data + // properties and each of their EvtVariant type values. + // + // The first time we observe a new fingerprint value we get the XML + // representation of the event in order to know the parameter names. + // If they turn out to match the values that we got from the provider's + // metadata then we just associate the fingerprint with a pointer to the + // providers metadata for the event ID. + + defaultEM := s.Events[eventID] + + // Use XML to get the parameters names. + em, err := newEventMetadataFromEventHandle(s.Metadata, eventHandle) + if err != nil { + s.log.Debugw("Failed to make event metadata from event handle. Will "+ + "use default event metadata from the publisher.", + "event_id", eventID, + "fingerprint", eventDataFingerprint, + "error", err) + + if defaultEM != nil { + fingerprints[eventDataFingerprint] = defaultEM + } + return defaultEM + } + + // Are the parameters the same as what the provider metadata listed? + // (This ignores the message values.) + if em.equal(defaultEM) { + fingerprints[eventDataFingerprint] = defaultEM + return defaultEM + } + + // If we couldn't get a message from the event handle use the one + // from the installed provider metadata. + if defaultEM != nil && em.MsgStatic == "" && em.MsgTemplate == nil { + em.MsgStatic = defaultEM.MsgStatic + em.MsgTemplate = defaultEM.MsgTemplate + } + + s.log.Debugw("Obtained unique event metadata from event handle. "+ + "It differed from what was listed in the publisher's metadata.", + "event_id", eventID, + "fingerprint", eventDataFingerprint, + "default_event_metadata", defaultEM, + "event_metadata", em) + + fingerprints[eventDataFingerprint] = em + return em +} + +func (s *publisherMetadataStore) Close() error { + if s.Metadata != nil { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.Metadata.Close() + } + return nil +} + +type eventMetadata struct { + EventID uint16 // Event ID. + Version uint8 // Event format version. + MsgStatic string // Used when the message has no parameters. + MsgTemplate *template.Template `json:"-"` // Template that expects an array of values as its data. + EventData []eventData // Names of parameters from XML template. +} + +// newEventMetadataFromEventHandle collects metadata about an event type using +// the handle of an event. +func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle EvtHandle) (*eventMetadata, error) { + xml, err := getEventXML(publisher, eventHandle) + if err != nil { + return nil, err + } + + // By parsing the XML we can get the names of the parameters even if the + // publisher metadata is unavailable or is out of sync with the events. + event, err := sys.UnmarshalEventXML([]byte(xml)) + if err != nil { + return nil, errors.Wrap(err, "failed to unmarshal XML") + } + + em := &eventMetadata{ + EventID: uint16(event.EventIdentifier.ID), + Version: uint8(event.Version), + } + if len(event.EventData.Pairs) > 0 { + for _, pair := range event.EventData.Pairs { + em.EventData = append(em.EventData, eventData{Name: pair.Key}) + } + } else { + for _, pair := range event.UserData.Pairs { + em.EventData = append(em.EventData, eventData{Name: pair.Key}) + } + } + + // The message template is only available from the publisher metadata. This + // message template may not match up with the event data we got from the + // event's XML, but it's the only option available. Even forwarded events + // with "RenderedText" won't help because their messages are already + // rendered. + if publisher != nil { + msg, err := getMessageStringFromHandle(publisher, eventHandle, templateInserts.Slice()) + if err != nil { + return nil, err + } + if err = em.setMessage(msg); err != nil { + return nil, err + } + } + + return em, nil +} + +// newEventMetadataFromPublisherMetadata collects metadata about an event type +// using the publisher metadata. +func newEventMetadataFromPublisherMetadata(itr *EventMetadataIterator, publisher *PublisherMetadata) (*eventMetadata, error) { + em := &eventMetadata{} + err := multierr.Combine( + em.initEventID(itr), + em.initVersion(itr), + em.initEventDataTemplate(itr), + em.initEventMessage(itr, publisher), + ) + if err != nil { + return nil, err + } + return em, nil +} + +func (em *eventMetadata) initEventID(itr *EventMetadataIterator) error { + id, err := itr.EventID() + if err != nil { + return err + } + // The upper 16 bits are the qualifier and lower 16 are the ID. + em.EventID = uint16(0xFFFF & id) + return nil +} + +func (em *eventMetadata) initVersion(itr *EventMetadataIterator) error { + version, err := itr.Version() + if err != nil { + return err + } + em.Version = uint8(version) + return nil +} + +func (em *eventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error { + xml, err := itr.Template() + if err != nil { + return err + } + // Some events do not have templates. + if xml == "" { + return nil + } + + tmpl := &eventTemplate{} + if err = tmpl.Unmarshal([]byte(xml)); err != nil { + return err + } + + for _, kv := range tmpl.Data { + kv.Name = eventDataNameTransform.Replace(kv.Name) + } + + em.EventData = tmpl.Data + return nil +} + +func (em *eventMetadata) initEventMessage(itr *EventMetadataIterator, publisher *PublisherMetadata) error { + messageID, err := itr.MessageID() + if err != nil { + return err + } + + msg, err := getMessageString(publisher, NilHandle, messageID, templateInserts.Slice()) + if err != nil { + return err + } + + return em.setMessage(msg) +} + +func (em *eventMetadata) setMessage(msg string) error { + msg = sys.RemoveWindowsLineEndings(msg) + tmplID := strconv.Itoa(int(em.EventID)) + + tmpl, err := template.New(tmplID).Funcs(eventMessageTemplateFuncs).Parse(msg) + if err != nil { + return err + } + + // One node means there were no parameters so this will optimize that case + // by using a static string rather than a text/template. + if len(tmpl.Root.Nodes) == 1 { + em.MsgStatic = msg + } else { + em.MsgTemplate = tmpl + } + return nil +} + +func (em *eventMetadata) equal(other *eventMetadata) bool { + if em == other { + return true + } + if em == nil || other == nil { + return false + } + + eventDataNamesEqual := func(a, b []eventData) bool { + if len(a) != len(b) { + return false + } + for n, v := range a { + if v.Name != b[n].Name { + return false + } + } + return true + } + + return em.EventID == other.EventID && + em.Version == other.Version && + eventDataNamesEqual(em.EventData, other.EventData) +} + +// --- Template Funcs + +// eventParam return an event data value inside a text/template. +func eventParam(items []interface{}, paramNumber int) (interface{}, error) { + // Windows parameter values start at %1 so adjust index value by -1. + index := paramNumber - 1 + if index < len(items) { + return items[index], nil + } + // Windows Event Viewer leaves the original placeholder (e.g. %22) in the + // rendered message when no value provided. + return "%" + strconv.Itoa(paramNumber), nil +} diff --git a/winlogbeat/sys/wineventlog/metadata_store_test.go b/winlogbeat/sys/wineventlog/metadata_store_test.go new file mode 100644 index 00000000000..0c89251bb7a --- /dev/null +++ b/winlogbeat/sys/wineventlog/metadata_store_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestPublisherMetadataStore(t *testing.T) { + logp.TestingSetup() + + s, err := newPublisherMetadataStore( + NilHandle, + "Microsoft-Windows-Security-Auditing", + logp.NewLogger("metadata")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + assert.NotEmpty(t, s.Events) + assert.Empty(t, s.EventFingerprints) + + t.Run("event_metadata_from_handle", func(t *testing.T) { + log := openLog(t, security4752File) + defer log.Close() + + h := mustNextHandle(t, log) + defer h.Close() + + em, err := newEventMetadataFromEventHandle(s.Metadata, h) + if err != nil { + t.Fatal(err) + } + + assert.EqualValues(t, 4752, em.EventID) + assert.EqualValues(t, 0, em.Version) + assert.Empty(t, em.MsgStatic) + assert.NotNil(t, em.MsgTemplate) + assert.NotEmpty(t, em.EventData) + }) +} diff --git a/winlogbeat/sys/wineventlog/publisher_metadata.go b/winlogbeat/sys/wineventlog/publisher_metadata.go new file mode 100644 index 00000000000..73be91e849c --- /dev/null +++ b/winlogbeat/sys/wineventlog/publisher_metadata.go @@ -0,0 +1,663 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "os" + "syscall" + + "github.com/pkg/errors" + "go.uber.org/multierr" + "golang.org/x/sys/windows" +) + +// PublisherMetadata provides methods to query metadata from an event log +// publisher. +type PublisherMetadata struct { + Name string // Name of the publisher/provider. + Handle EvtHandle // Handle to the publisher metadata from EvtOpenPublisherMetadata. +} + +// Close releases the publisher metadata handle. +func (m *PublisherMetadata) Close() error { + return m.Handle.Close() +} + +// NewPublisherMetadata opens the publisher's metadata. Close must be called on +// the returned PublisherMetadata to release its handle. +func NewPublisherMetadata(session EvtHandle, name string) (*PublisherMetadata, error) { + var publisherName, logFile *uint16 + if info, err := os.Stat(name); err == nil && info.Mode().IsRegular() { + logFile, err = syscall.UTF16PtrFromString(name) + if err != nil { + return nil, err + } + } else { + publisherName, err = syscall.UTF16PtrFromString(name) + if err != nil { + return nil, err + } + } + + handle, err := _EvtOpenPublisherMetadata(session, publisherName, logFile, 0, 0) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtOpenPublisherMetadata") + } + + return &PublisherMetadata{ + Name: name, + Handle: handle, + }, nil +} + +func (m *PublisherMetadata) stringProperty(propertyID EvtPublisherMetadataPropertyID) (string, error) { + v, err := EvtGetPublisherMetadataProperty(m.Handle, propertyID) + if err != nil { + return "", err + } + switch t := v.(type) { + case string: + return t, nil + case nil: + return "", nil + default: + return "", errors.Errorf("unexpected data type: %T", v) + } +} + +func (m *PublisherMetadata) PublisherGUID() (windows.GUID, error) { + v, err := EvtGetPublisherMetadataProperty(m.Handle, EvtPublisherMetadataPublisherGuid) + if err != nil { + return windows.GUID{}, err + } + switch t := v.(type) { + case windows.GUID: + return t, nil + case nil: + return windows.GUID{}, nil + default: + return windows.GUID{}, errors.Errorf("unexpected data type: %T", v) + } +} + +func (m *PublisherMetadata) ResourceFilePath() (string, error) { + return m.stringProperty(EvtPublisherMetadataResourceFilePath) +} + +func (m *PublisherMetadata) ParameterFilePath() (string, error) { + return m.stringProperty(EvtPublisherMetadataParameterFilePath) +} + +func (m *PublisherMetadata) MessageFilePath() (string, error) { + return m.stringProperty(EvtPublisherMetadataMessageFilePath) +} + +func (m *PublisherMetadata) HelpLink() (string, error) { + return m.stringProperty(EvtPublisherMetadataHelpLink) +} + +func (m *PublisherMetadata) PublisherMessageID() (uint32, error) { + v, err := EvtGetPublisherMetadataProperty(m.Handle, EvtPublisherMetadataPublisherMessageID) + if err != nil { + return 0, err + } + return v.(uint32), nil +} + +func (m *PublisherMetadata) PublisherMessage() (string, error) { + messageID, err := m.PublisherMessageID() + if err != nil { + return "", err + } + if int32(messageID) == -1 { + return "", nil + } + return getMessageStringFromMessageID(m, messageID, nil) +} + +func (m *PublisherMetadata) Keywords() ([]MetadataKeyword, error) { + return NewMetadataKeywords(m.Handle) +} + +func (m *PublisherMetadata) Opcodes() ([]MetadataOpcode, error) { + return NewMetadataOpcodes(m.Handle) +} + +func (m *PublisherMetadata) Levels() ([]MetadataLevel, error) { + return NewMetadataLevels(m.Handle) +} + +func (m *PublisherMetadata) Tasks() ([]MetadataTask, error) { + return NewMetadataTasks(m.Handle) +} + +func (m *PublisherMetadata) Channels() ([]MetadataChannel, error) { + return NewMetadataChannels(m.Handle) +} + +func (m *PublisherMetadata) EventMetadataIterator() (*EventMetadataIterator, error) { + return NewEventMetadataIterator(m) +} + +type MetadataKeyword struct { + Name string + Mask uint64 + Message string + MessageID uint32 +} + +func NewMetadataKeywords(publisherMetadataHandle EvtHandle) ([]MetadataKeyword, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataKeywords) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get keyword array length") + } + + var values []MetadataKeyword + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataKeyword(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get keyword at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataKeyword(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataKeyword, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataKeywordMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the keyword did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataKeywordName, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataKeywordValue, index) + if err != nil { + return nil, err + } + valueMask := v.(uint64) + + return &MetadataKeyword{ + Name: name, + Mask: valueMask, + MessageID: messageID, + Message: message, + }, nil +} + +type MetadataOpcode struct { + Name string + Mask uint32 + MessageID uint32 + Message string +} + +func NewMetadataOpcodes(publisherMetadataHandle EvtHandle) ([]MetadataOpcode, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataOpcodes) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get opcode array length") + } + + var values []MetadataOpcode + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataOpcode(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get opcode at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataOpcode(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataOpcode, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataOpcodeMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the opcode did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataOpcodeName, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataOpcodeValue, index) + if err != nil { + return nil, err + } + valueMask := v.(uint32) + + return &MetadataOpcode{ + Name: name, + Mask: valueMask, + MessageID: messageID, + Message: message, + }, nil +} + +type MetadataLevel struct { + Name string + Mask uint32 + MessageID uint32 + Message string +} + +func NewMetadataLevels(publisherMetadataHandle EvtHandle) ([]MetadataLevel, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataLevels) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get level array length") + } + + var values []MetadataLevel + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataLevel(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get level at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataLevel(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataLevel, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataLevelMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the level did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataLevelName, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataLevelValue, index) + if err != nil { + return nil, err + } + valueMask := v.(uint32) + + return &MetadataLevel{ + Name: name, + Mask: valueMask, + MessageID: messageID, + Message: message, + }, nil +} + +type MetadataTask struct { + Name string + Mask uint32 + MessageID uint32 + Message string + EventGUID windows.GUID +} + +func NewMetadataTasks(publisherMetadataHandle EvtHandle) ([]MetadataTask, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataTasks) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get task array length") + } + + var values []MetadataTask + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataTask(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get task at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataTask(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataTask, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataTaskMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the task did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataTaskName, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataTaskValue, index) + if err != nil { + return nil, err + } + valueMask := v.(uint32) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataTaskEventGuid, index) + if err != nil { + return nil, err + } + guid := v.(windows.GUID) + + return &MetadataTask{ + Name: name, + Mask: valueMask, + MessageID: messageID, + Message: message, + EventGUID: guid, + }, nil +} + +type MetadataChannel struct { + Name string + Index uint32 + ID uint32 + Message string + MessageID uint32 +} + +func NewMetadataChannels(publisherMetadataHandle EvtHandle) ([]MetadataChannel, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataChannelReferences) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get task array length") + } + + var values []MetadataChannel + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataChannel(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get task at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataChannel(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataChannel, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataChannelReferenceMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the task did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + // Channel name. + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataChannelReferencePath, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataChannelReferenceIndex, index) + if err != nil { + return nil, err + } + channelIndex := v.(uint32) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataChannelReferenceID, index) + if err != nil { + return nil, err + } + id := v.(uint32) + + return &MetadataChannel{ + Name: name, + Index: channelIndex, + ID: id, + MessageID: messageID, + Message: message, + }, nil +} + +type EventMetadataIterator struct { + Publisher *PublisherMetadata + eventMetadataEnumHandle EvtHandle + currentEvent EvtHandle + lastErr error +} + +func NewEventMetadataIterator(publisher *PublisherMetadata) (*EventMetadataIterator, error) { + eventMetadataEnumHandle, err := _EvtOpenEventMetadataEnum(publisher.Handle, 0) + if err != nil { + return nil, errors.Wrap(err, "failed to open event metadata enumerator with EvtOpenEventMetadataEnum") + } + + return &EventMetadataIterator{ + Publisher: publisher, + eventMetadataEnumHandle: eventMetadataEnumHandle, + }, nil +} + +func (itr *EventMetadataIterator) Close() error { + return multierr.Combine( + _EvtClose(itr.eventMetadataEnumHandle), + _EvtClose(itr.currentEvent), + ) +} + +// Next advances to the next event handle. It returns false when there are +// no more items or an error occurred. You should call Err() to check for an +// error. +func (itr *EventMetadataIterator) Next() bool { + // Close existing handle. + itr.currentEvent.Close() + + var err error + itr.currentEvent, err = _EvtNextEventMetadata(itr.eventMetadataEnumHandle, 0) + if err != nil { + if err != windows.ERROR_NO_MORE_ITEMS { + itr.lastErr = errors.Wrap(err, "failed advancing to next event metadata handle") + } + return false + } + return true +} + +// Err returns an error if Next() failed due to an error. +func (itr *EventMetadataIterator) Err() error { + return itr.lastErr +} + +func (itr *EventMetadataIterator) uint32Property(propertyID EvtEventMetadataPropertyID) (uint32, error) { + v, err := GetEventMetadataProperty(itr.currentEvent, propertyID) + if err != nil { + return 0, err + } + return v.(uint32), nil +} + +func (itr *EventMetadataIterator) uint64Property(propertyID EvtEventMetadataPropertyID) (uint64, error) { + v, err := GetEventMetadataProperty(itr.currentEvent, propertyID) + if err != nil { + return 0, err + } + return v.(uint64), nil +} + +func (itr *EventMetadataIterator) stringProperty(propertyID EvtEventMetadataPropertyID) (string, error) { + v, err := GetEventMetadataProperty(itr.currentEvent, propertyID) + if err != nil { + return "", err + } + return v.(string), nil +} + +func (itr *EventMetadataIterator) EventID() (uint32, error) { + return itr.uint32Property(EventMetadataEventID) +} + +func (itr *EventMetadataIterator) Version() (uint32, error) { + return itr.uint32Property(EventMetadataEventVersion) +} + +func (itr *EventMetadataIterator) Channel() (uint32, error) { + return itr.uint32Property(EventMetadataEventVersion) +} + +func (itr *EventMetadataIterator) Level() (uint32, error) { + return itr.uint32Property(EventMetadataEventLevel) +} + +func (itr *EventMetadataIterator) Opcode() (uint32, error) { + return itr.uint32Property(EventMetadataEventOpcode) +} + +func (itr *EventMetadataIterator) Task() (uint32, error) { + return itr.uint32Property(EventMetadataEventTask) +} + +func (itr *EventMetadataIterator) Keyword() (uint64, error) { + return itr.uint64Property(EventMetadataEventKeyword) +} + +func (itr *EventMetadataIterator) MessageID() (uint32, error) { + return itr.uint32Property(EventMetadataEventMessageID) +} + +func (itr *EventMetadataIterator) Template() (string, error) { + return itr.stringProperty(EventMetadataEventTemplate) +} + +// Message returns the raw event description without doing any substitutions +// (e.g. the message will contain %1, %2, etc. as parameter placeholders). +func (itr *EventMetadataIterator) Message() (string, error) { + messageID, err := itr.MessageID() + if err != nil { + return "", err + } + // If the event definition does not specify a message, the value is –1. + if int32(messageID) == -1 { + return "", nil + } + + return getMessageStringFromMessageID(itr.Publisher, messageID, nil) +} diff --git a/winlogbeat/sys/wineventlog/publisher_metadata_test.go b/winlogbeat/sys/wineventlog/publisher_metadata_test.go new file mode 100644 index 00000000000..1f5f5a2b85c --- /dev/null +++ b/winlogbeat/sys/wineventlog/publisher_metadata_test.go @@ -0,0 +1,230 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" +) + +func TestPublisherMetadata(t *testing.T) { + // Modern Application + testPublisherMetadata(t, "Microsoft-Windows-PowerShell") + // Modern Application that uses UserData in XML + testPublisherMetadata(t, "Microsoft-Windows-Eventlog") + // Classic with messages (no event-data XML templates). + testPublisherMetadata(t, "Microsoft-Windows-Security-SPP") + // Classic without message metadata (no event-data XML templates). + testPublisherMetadata(t, "Windows Error Reporting") +} + +func testPublisherMetadata(t *testing.T, provider string) { + t.Run(provider, func(t *testing.T) { + md, err := NewPublisherMetadata(NilHandle, provider) + if err != nil { + t.Fatalf("%+v", err) + } + defer md.Close() + + t.Run("publisher_guid", func(t *testing.T) { + v, err := md.PublisherGUID() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("PublisherGUID: %v", v) + }) + + t.Run("resource_file_path", func(t *testing.T) { + v, err := md.ResourceFilePath() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("ResourceFilePath: %v", v) + }) + + t.Run("parameter_file_path", func(t *testing.T) { + v, err := md.ParameterFilePath() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("ParameterFilePath: %v", v) + }) + + t.Run("message_file_path", func(t *testing.T) { + v, err := md.MessageFilePath() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("MessageFilePath: %v", v) + }) + + t.Run("help_link", func(t *testing.T) { + v, err := md.HelpLink() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("HelpLink: %v", v) + }) + + t.Run("publisher_message_id", func(t *testing.T) { + v, err := md.PublisherMessageID() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("PublisherMessageID: %v", v) + }) + + t.Run("publisher_message", func(t *testing.T) { + v, err := md.PublisherMessage() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("PublisherMessage: %v", v) + }) + + t.Run("keywords", func(t *testing.T) { + values, err := md.Keywords() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("opcodes", func(t *testing.T) { + values, err := md.Opcodes() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("levels", func(t *testing.T) { + values, err := md.Levels() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("tasks", func(t *testing.T) { + values, err := md.Tasks() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("channels", func(t *testing.T) { + values, err := md.Channels() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("event_metadata", func(t *testing.T) { + itr, err := md.EventMetadataIterator() + if err != nil { + t.Fatalf("%+v", err) + } + defer itr.Close() + + for itr.Next() { + eventID, err := itr.EventID() + assert.NoError(t, err) + t.Logf("eventID=%v (id=%v, qualifier=%v)", eventID, + 0xFFFF&eventID, // Lower 16 bits are the event ID. + (0xFFFF0000&eventID)>>16) // Upper 16 bits are the qualifier. + + version, err := itr.Version() + assert.NoError(t, err) + t.Logf("version=%v", version) + + channel, err := itr.Channel() + assert.NoError(t, err) + t.Logf("channel=%v", channel) + + level, err := itr.Level() + assert.NoError(t, err) + t.Logf("level=%v", level) + + opcode, err := itr.Opcode() + assert.NoError(t, err) + t.Logf("opcode=%v", opcode) + + task, err := itr.Task() + assert.NoError(t, err) + t.Logf("task=%v", task) + + keyword, err := itr.Keyword() + assert.NoError(t, err) + t.Logf("keyword=%v", keyword) + + messageID, err := itr.MessageID() + assert.NoError(t, err) + t.Logf("messageID=%v", messageID) + + template, err := itr.Template() + assert.NoError(t, err) + t.Logf("template=%v", template) + + message, err := itr.Message() + assert.NoError(t, err) + t.Logf("message=%v", message) + } + if err = itr.Err(); err != nil { + t.Fatalf("%+v", err) + } + }) + }) +} + +func TestNewPublisherMetadataUnknown(t *testing.T) { + _, err := NewPublisherMetadata(NilHandle, "Fake-Publisher") + assert.Equal(t, windows.ERROR_FILE_NOT_FOUND, errors.Cause(err)) +} diff --git a/winlogbeat/sys/wineventlog/query.go b/winlogbeat/sys/wineventlog/query.go index 76512747c44..8b822600425 100644 --- a/winlogbeat/sys/wineventlog/query.go +++ b/winlogbeat/sys/wineventlog/query.go @@ -49,8 +49,8 @@ var ( // Query that identifies the source of the events and one or more selectors or // suppressors. type Query struct { - // Name of the channel or the path to the log file that contains the events - // to query. + // Name of the channel or the URI path to the log file that contains the + // events to query. The path to files must be a URI like file://C:/log.evtx. Log string IgnoreOlder time.Duration // Ignore records older than this time period. @@ -209,7 +209,7 @@ func (qp *queryParams) providerSelect(q Query) error { return nil } - var selects []string + selects := make([]string, 0, len(q.Provider)) for _, p := range q.Provider { selects = append(selects, fmt.Sprintf("@Name='%s'", p)) } diff --git a/winlogbeat/sys/wineventlog/renderer.go b/winlogbeat/sys/wineventlog/renderer.go new file mode 100644 index 00000000000..4a6fcc2fef1 --- /dev/null +++ b/winlogbeat/sys/wineventlog/renderer.go @@ -0,0 +1,437 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "encoding/binary" + "fmt" + "strconv" + "sync" + "text/template" + "time" + "unsafe" + + "github.com/cespare/xxhash/v2" + "github.com/pkg/errors" + "go.uber.org/multierr" + "golang.org/x/sys/windows" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys" +) + +const ( + // keywordClassic indicates the log was published with the "classic" event + // logging API. + // https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.eventing.reader.standardeventkeywords?view=netframework-4.8 + keywordClassic = 0x80000000000000 +) + +// Renderer is used for converting event log handles into complete events. +type Renderer struct { + // Cache of publisher metadata. Maps publisher names to stored metadata. + metadataCache map[string]*publisherMetadataStore + // Mutex to guard the metadataCache. The other members are immutable. + mutex sync.RWMutex + + session EvtHandle // Session handle if working with remote log. + systemContext EvtHandle // Render context for system values. + userContext EvtHandle // Render context for user values (event data). + log *logp.Logger +} + +// NewRenderer returns a new Renderer. +func NewRenderer(session EvtHandle, log *logp.Logger) (*Renderer, error) { + systemContext, err := _EvtCreateRenderContext(0, 0, EvtRenderContextSystem) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtCreateRenderContext for system context") + } + + userContext, err := _EvtCreateRenderContext(0, 0, EvtRenderContextUser) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtCreateRenderContext for user context") + } + + return &Renderer{ + metadataCache: map[string]*publisherMetadataStore{}, + session: session, + systemContext: systemContext, + userContext: userContext, + log: log.Named("renderer"), + }, nil +} + +// Close closes all handles held by the Renderer. +func (r *Renderer) Close() error { + r.mutex.Lock() + defer r.mutex.Unlock() + + errs := []error{r.systemContext.Close(), r.userContext.Close()} + for _, md := range r.metadataCache { + if err := md.Close(); err != nil { + errs = append(errs, err) + } + } + return multierr.Combine(errs...) +} + +// Render renders the event handle into an Event. +func (r *Renderer) Render(handle EvtHandle) (*sys.Event, error) { + event := &sys.Event{} + + if err := r.renderSystem(handle, event); err != nil { + return nil, errors.Wrap(err, "failed to render system properties") + } + + // From this point on it will return both the event and any errors. It's + // critical to not drop data. + var errs []error + + // This always returns a non-nil value (even on error). + md, err := r.getPublisherMetadata(event.Provider.Name) + if err != nil { + errs = append(errs, err) + } + + // Associate raw system properties to names (e.g. level=2 to Error). + enrichRawValuesWithNames(md, event) + + eventData, fingerprint, err := r.renderUser(handle, event) + if err != nil { + errs = append(errs, errors.Wrap(err, "failed to render event data")) + } + + // Load cached event metadata or try to bootstrap it from the event's XML. + eventMeta := md.getEventMetadata(uint16(event.EventIdentifier.ID), fingerprint, handle) + + // Associate key names with the event data values. + r.addEventData(eventMeta, eventData, event) + + if event.Message, err = r.formatMessage(md, eventMeta, handle, eventData, uint16(event.EventIdentifier.ID)); err != nil { + errs = append(errs, errors.Wrap(err, "failed to get the event message string")) + } + + if len(errs) > 0 { + return event, multierr.Combine(errs...) + } + return event, nil +} + +// getPublisherMetadata return a publisherMetadataStore for the provider. It +// never returns nil, but may return an error if it couldn't open a publisher. +func (r *Renderer) getPublisherMetadata(publisher string) (*publisherMetadataStore, error) { + var err error + + // NOTE: This code uses double-check locking to elevate to a write-lock + // when a cache value needs initialized. + r.mutex.RLock() + + // Lookup cached value. + md, found := r.metadataCache[publisher] + if !found { + // Elevate to write lock. + r.mutex.RUnlock() + r.mutex.Lock() + defer r.mutex.Unlock() + + // Double-check if the condition changed while upgrading the lock. + md, found = r.metadataCache[publisher] + if found { + return md, nil + } + + // Load metadata from the publisher. + md, err = newPublisherMetadataStore(r.session, publisher, r.log) + if err != nil { + // Return an empty store on error (can happen in cases where the + // log was forwarded and the provider doesn't exist on collector). + md = newEmptyPublisherMetadataStore(publisher, r.log) + err = errors.Wrapf(err, "failed to load publisher metadata for %v "+ + "(returning an empty metadata store)", publisher) + } + r.metadataCache[publisher] = md + } else { + r.mutex.RUnlock() + } + + return md, err +} + +// renderSystem writes all the system context properties into the event. +func (r *Renderer) renderSystem(handle EvtHandle, event *sys.Event) error { + bb, propertyCount, err := r.render(r.systemContext, handle) + if err != nil { + return errors.Wrap(err, "failed to get system values") + } + defer bb.free() + + for i := 0; i < int(propertyCount); i++ { + property := EvtSystemPropertyID(i) + offset := i * int(sizeofEvtVariant) + evtVar := (*EvtVariant)(unsafe.Pointer(&bb.buf[offset])) + + data, err := evtVar.Data(bb.buf) + if err != nil || data == nil { + continue + } + + switch property { + case EvtSystemProviderName: + event.Provider.Name = data.(string) + case EvtSystemProviderGuid: + event.Provider.GUID = data.(windows.GUID).String() + case EvtSystemEventID: + event.EventIdentifier.ID = uint32(data.(uint16)) + case EvtSystemQualifiers: + event.EventIdentifier.Qualifiers = data.(uint16) + case EvtSystemLevel: + event.LevelRaw = data.(uint8) + case EvtSystemTask: + event.TaskRaw = data.(uint16) + case EvtSystemOpcode: + event.OpcodeRaw = data.(uint8) + case EvtSystemKeywords: + event.KeywordsRaw = sys.HexInt64(data.(hexInt64)) + case EvtSystemTimeCreated: + event.TimeCreated.SystemTime = data.(time.Time) + case EvtSystemEventRecordId: + event.RecordID = data.(uint64) + case EvtSystemActivityID: + event.Correlation.ActivityID = data.(windows.GUID).String() + case EvtSystemRelatedActivityID: + event.Correlation.RelatedActivityID = data.(windows.GUID).String() + case EvtSystemProcessID: + event.Execution.ProcessID = data.(uint32) + case EvtSystemThreadID: + event.Execution.ThreadID = data.(uint32) + case EvtSystemChannel: + event.Channel = data.(string) + case EvtSystemComputer: + event.Computer = data.(string) + case EvtSystemUserID: + sid := data.(*windows.SID) + event.User.Identifier = sid.String() + var accountType uint32 + event.User.Name, event.User.Domain, accountType, _ = sid.LookupAccount("") + event.User.Type = sys.SIDType(accountType) + case EvtSystemVersion: + event.Version = sys.Version(data.(uint8)) + } + } + + return nil +} + +// renderUser returns the event/user data values. This does not provide the +// parameter names. It computes a fingerprint of the values types to help the +// caller match the correct names to the returned values. +func (r *Renderer) renderUser(handle EvtHandle, event *sys.Event) (values []interface{}, fingerprint uint64, err error) { + bb, propertyCount, err := r.render(r.userContext, handle) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to get user values") + } + defer bb.free() + + if propertyCount == 0 { + return nil, 0, nil + } + + // Fingerprint the argument types to help ensure we match these values with + // the correct event data parameter names. + argumentHash := xxhash.New() + binary.Write(argumentHash, binary.LittleEndian, propertyCount) + + values = make([]interface{}, propertyCount) + for i := 0; i < propertyCount; i++ { + offset := i * int(sizeofEvtVariant) + evtVar := (*EvtVariant)(unsafe.Pointer(&bb.buf[offset])) + binary.Write(argumentHash, binary.LittleEndian, uint32(evtVar.Type)) + + values[i], err = evtVar.Data(bb.buf) + if err != nil { + r.log.Warnw("Failed to read event/user data value. Using nil.", + "provider", event.Provider.Name, + "event_id", event.EventIdentifier.ID, + "value_index", i, + "value_type", evtVar.Type.String(), + "error", err, + ) + } + } + + return values, argumentHash.Sum64(), nil +} + +// render uses EvtRender to event data. The caller must free() the returned when +// done accessing the bytes. +func (r *Renderer) render(context EvtHandle, eventHandle EvtHandle) (*byteBuffer, int, error) { + var bufferUsed, propertyCount uint32 + + err := _EvtRender(context, eventHandle, EvtRenderEventValues, 0, nil, &bufferUsed, &propertyCount) + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { + return nil, 0, errors.Wrap(err, "failed in EvtRender") + } + + if propertyCount == 0 { + return nil, 0, nil + } + + bb := newByteBuffer() + bb.Reserve(int(bufferUsed)) + + err = _EvtRender(context, eventHandle, EvtRenderEventValues, uint32(len(bb.buf)), &bb.buf[0], &bufferUsed, &propertyCount) + if err != nil { + bb.free() + return nil, 0, errors.Wrap(err, "failed in EvtRender") + } + + return bb, int(propertyCount), nil +} + +// addEventData adds the event/user data values to the event. +func (r *Renderer) addEventData(evtMeta *eventMetadata, values []interface{}, event *sys.Event) { + if len(values) == 0 { + return + } + + if evtMeta == nil { + r.log.Warnw("Event metadata not found.", + "provider", event.Provider.Name, + "event_id", event.EventIdentifier.ID) + } else if len(values) != len(evtMeta.EventData) { + r.log.Warnw("The number of event data parameters doesn't match the number "+ + "of parameters in the template.", + "provider", event.Provider.Name, + "event_id", event.EventIdentifier.ID, + "event_parameter_count", len(values), + "template_parameter_count", len(evtMeta.EventData), + "template_version", evtMeta.Version, + "event_version", event.Version) + } + + // Fallback to paramN naming when the value does not exist in event data. + // This can happen for legacy providers without manifests. This can also + // happen if the installed provider manifest doesn't match the version that + // produced the event (forwarded events, reading from evtx, or software was + // updated). If software was updated it could also be that this cached + // template is now stale. + paramName := func(idx int) string { + if evtMeta != nil && idx < len(evtMeta.EventData) { + return evtMeta.EventData[idx].Name + } + return "param" + strconv.Itoa(idx) + } + + for i, v := range values { + var strVal string + switch t := v.(type) { + case string: + strVal = t + case *windows.SID: + strVal = t.String() + default: + strVal = fmt.Sprintf("%v", v) + } + + event.EventData.Pairs = append(event.EventData.Pairs, sys.KeyValue{ + Key: paramName(i), + Value: strVal, + }) + } + + return +} + +// formatMessage adds the message to the event. +func (r *Renderer) formatMessage(publisherMeta *publisherMetadataStore, + eventMeta *eventMetadata, eventHandle EvtHandle, values []interface{}, + eventID uint16) (string, error) { + + if eventMeta != nil { + if eventMeta.MsgStatic != "" { + return eventMeta.MsgStatic, nil + } else if eventMeta.MsgTemplate != nil { + return r.formatMessageFromTemplate(eventMeta.MsgTemplate, values) + } + } + + // Fallback to the trying EvtFormatMessage mechanism. + // This is the path for forwarded events in RenderedText mode where the + // local publisher metadata is not present. NOTE that if the local publisher + // metadata exists it will be preferred over the RenderedText. A config + // option might be desirable to control this behavior. + r.log.Debugf("Falling back to EvtFormatMessage for event ID %d.", eventID) + return getMessageString(publisherMeta.Metadata, eventHandle, 0, nil) +} + +// formatMessageFromTemplate creates the message by executing the stored Go +// text/template with the event/user data values. +func (r *Renderer) formatMessageFromTemplate(msgTmpl *template.Template, values []interface{}) (string, error) { + bb := newByteBuffer() + defer bb.free() + + if err := msgTmpl.Execute(bb, values); err != nil { + return "", errors.Wrapf(err, "failed to execute template with data=%#v template=%v", values, msgTmpl.Root.String()) + } + + return string(bb.Bytes()), nil +} + +// enrichRawValuesWithNames adds the names associated with the raw system +// property values. It enriches the event with keywords, opcode, level, and +// task. The search order is defined in the EvtFormatMessage documentation. +func enrichRawValuesWithNames(publisherMeta *publisherMetadataStore, event *sys.Event) { + // Keywords. Each bit in the value can represent a keyword. + rawKeyword := int64(event.KeywordsRaw) + isClassic := keywordClassic&rawKeyword > 0 + for mask, keyword := range winMeta.Keywords { + if rawKeyword&mask > 0 { + event.Keywords = append(event.Keywords, keyword) + rawKeyword -= mask + } + } + for mask, keyword := range publisherMeta.Keywords { + if rawKeyword&mask > 0 { + event.Keywords = append(event.Keywords, keyword) + rawKeyword -= mask + } + } + + // Opcode (search in winmeta first). + var found bool + if !isClassic { + event.Opcode, found = winMeta.Opcodes[event.OpcodeRaw] + if !found { + event.Opcode = publisherMeta.Opcodes[event.OpcodeRaw] + } + } + + // Level (search in winmeta first). + event.Level, found = winMeta.Levels[event.LevelRaw] + if !found { + event.Level = publisherMeta.Levels[event.LevelRaw] + } + + // Task (fall-back to winmeta if not found). + event.Task, found = publisherMeta.Tasks[event.TaskRaw] + if !found { + event.Task = winMeta.Tasks[event.TaskRaw] + } +} diff --git a/winlogbeat/sys/wineventlog/renderer_test.go b/winlogbeat/sys/wineventlog/renderer_test.go new file mode 100644 index 00000000000..3dd80343803 --- /dev/null +++ b/winlogbeat/sys/wineventlog/renderer_test.go @@ -0,0 +1,291 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "bytes" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "text/template" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys" +) + +func TestRenderer(t *testing.T) { + logp.TestingSetup() + + t.Run(filepath.Base(sysmon9File), func(t *testing.T) { + log := openLog(t, sysmon9File) + defer log.Close() + + r, err := NewRenderer(NilHandle, logp.L()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + events := renderAllEvents(t, log, r, true) + assert.NotEmpty(t, events) + + if t.Failed() { + logAsJSON(t, events) + } + }) + + t.Run(filepath.Base(security4752File), func(t *testing.T) { + log := openLog(t, security4752File) + defer log.Close() + + r, err := NewRenderer(NilHandle, logp.L()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + events := renderAllEvents(t, log, r, false) + if !assert.Len(t, events, 1) { + return + } + e := events[0] + + assert.EqualValues(t, 4752, e.EventIdentifier.ID) + assert.Equal(t, "Microsoft-Windows-Security-Auditing", e.Provider.Name) + assertEqualIgnoreCase(t, "{54849625-5478-4994-a5ba-3e3b0328c30d}", e.Provider.GUID) + assert.Equal(t, "DC_TEST2k12.TEST.SAAS", e.Computer) + assert.Equal(t, "Security", e.Channel) + assert.EqualValues(t, 3707686, e.RecordID) + + assert.Equal(t, e.Keywords, []string{"Audit Success"}) + + assert.EqualValues(t, 0, e.OpcodeRaw) + assert.Equal(t, "Info", e.Opcode) + + assert.EqualValues(t, 0, e.LevelRaw) + assert.Equal(t, "Information", e.Level) + + assert.EqualValues(t, 13827, e.TaskRaw) + assert.Equal(t, "Distribution Group Management", e.Task) + + assert.EqualValues(t, 492, e.Execution.ProcessID) + assert.EqualValues(t, 1076, e.Execution.ThreadID) + assert.Len(t, e.EventData.Pairs, 10) + + assert.NotEmpty(t, e.Message) + + if t.Failed() { + logAsJSON(t, events) + } + }) + + t.Run(filepath.Base(winErrorReportingFile), func(t *testing.T) { + log := openLog(t, winErrorReportingFile) + defer log.Close() + + r, err := NewRenderer(NilHandle, logp.L()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + events := renderAllEvents(t, log, r, false) + if !assert.Len(t, events, 1) { + return + } + e := events[0] + + assert.EqualValues(t, 1001, e.EventIdentifier.ID) + assert.Equal(t, "Windows Error Reporting", e.Provider.Name) + assert.Empty(t, e.Provider.GUID) + assert.Equal(t, "vagrant", e.Computer) + assert.Equal(t, "Application", e.Channel) + assert.EqualValues(t, 420107, e.RecordID) + + assert.Equal(t, e.Keywords, []string{"Classic"}) + + assert.EqualValues(t, 0, e.OpcodeRaw) + assert.Equal(t, "", e.Opcode) + + assert.EqualValues(t, 4, e.LevelRaw) + assert.Equal(t, "Information", e.Level) + + assert.EqualValues(t, 0, e.TaskRaw) + assert.Equal(t, "None", e.Task) + + assert.EqualValues(t, 0, e.Execution.ProcessID) + assert.EqualValues(t, 0, e.Execution.ThreadID) + assert.Len(t, e.EventData.Pairs, 23) + + assert.NotEmpty(t, e.Message) + + if t.Failed() { + logAsJSON(t, events) + } + }) +} + +func TestTemplateFunc(t *testing.T) { + tmpl := template.Must(template.New(""). + Funcs(eventMessageTemplateFuncs). + Parse(`Hello {{ eventParam $ 1 }}! Foo {{ eventParam $ 2 }}.`)) + + buf := new(bytes.Buffer) + err := tmpl.Execute(buf, []interface{}{"world"}) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "Hello world! Foo %2.", buf.String()) +} + +// renderAllEvents reads all events and renders them. +func renderAllEvents(t *testing.T, log EvtHandle, renderer *Renderer, ignoreMissingMetadataError bool) []*sys.Event { + t.Helper() + + var events []*sys.Event + for { + h, done := nextHandle(t, log) + if done { + break + } + + func() { + defer h.Close() + + evt, err := renderer.Render(h) + if err != nil { + md := renderer.metadataCache[evt.Provider.Name] + if !ignoreMissingMetadataError || md.Metadata != nil { + t.Fatalf("Render failed: %+v", err) + } + } + + events = append(events, evt) + }() + } + + return events +} + +// setLogSize set the maximum number of bytes that an event log can hold. +func setLogSize(t testing.TB, provider string, sizeBytes int) { + output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), provider).CombinedOutput() + if err != nil { + t.Fatal("failed to set log size", err, string(output)) + } +} + +func BenchmarkRenderer(b *testing.B) { + writer, teardown := createLog(b) + defer teardown() + + const totalEvents = 1000000 + msg := strings.Repeat("Hello world! ", 21) + for i := 0; i < totalEvents; i++ { + writer.Info(10, msg) + } + + setup := func() (*EventIterator, *Renderer) { + log := openLog(b, winlogbeatTestLogName) + + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(1024)) + if err != nil { + log.Close() + b.Fatal(err) + } + + r, err := NewRenderer(NilHandle, logp.NewLogger("bench")) + if err != nil { + log.Close() + itr.Close() + b.Fatal(err) + } + + return itr, r + } + + b.Run("single_thread", func(b *testing.B) { + itr, r := setup() + defer itr.Close() + defer r.Close() + + count := atomic.NewUint64(0) + start := time.Now() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // Get next handle. + h, ok := itr.Next() + if !ok { + b.Fatal("Ran out of events before benchmark was done.", itr.Err()) + } + + // Render it. + _, err := r.Render(h) + if err != nil { + b.Fatal(err) + } + + count.Inc() + } + + elapsed := time.Since(start) + b.ReportMetric(float64(count.Load())/elapsed.Seconds(), "events/sec") + }) + + b.Run("parallel8", func(b *testing.B) { + itr, r := setup() + defer itr.Close() + defer r.Close() + + count := atomic.NewUint64(0) + start := time.Now() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + // Get next handle. + h, ok := itr.Next() + if !ok { + b.Fatal("Ran out of events before benchmark was done.", itr.Err()) + } + + // Render it. + _, err := r.Render(h) + if err != nil { + b.Fatal(err) + } + count.Inc() + } + }) + + elapsed := time.Since(start) + b.ReportMetric(float64(count.Load())/elapsed.Seconds(), "events/sec") + b.ReportMetric(float64(runtime.GOMAXPROCS(0)), "gomaxprocs") + }) +} diff --git a/winlogbeat/sys/wineventlog/stringinserts.go b/winlogbeat/sys/wineventlog/stringinserts.go new file mode 100644 index 00000000000..347e478b9bd --- /dev/null +++ b/winlogbeat/sys/wineventlog/stringinserts.go @@ -0,0 +1,85 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strconv" + "unsafe" + + "golang.org/x/sys/windows" +) + +const ( + // maxInsertStrings is the maximum number of parameters supported in a + // Windows event message. + maxInsertStrings = 99 +) + +// templateInserts contains EvtVariant values that can be used to substitute +// Go text/template expressions into a Windows event message. +var templateInserts = newTemplateStringInserts() + +// stringsInserts holds EvtVariant values with type EvtVarTypeString. +type stringInserts struct { + // insertStrings are slices holding the strings in the EvtVariant (this must + // keep a reference to these to prevent GC of the strings as there is + // an unsafe reference to them in the evtVariants). + insertStrings [maxInsertStrings][]uint16 + evtVariants [maxInsertStrings]EvtVariant +} + +// Pointer returns a pointer the EvtVariant array. +func (si *stringInserts) Slice() []EvtVariant { + return si.evtVariants[:] +} + +// clear clears the pointers (and unsafe pointers) so that the memory can be +// garbage collected. +func (si *stringInserts) clear() { + for i := 0; i < len(si.evtVariants); i++ { + si.evtVariants[i] = EvtVariant{} + si.insertStrings[i] = nil + } +} + +// newTemplateStringInserts returns a stringInserts where each value is a +// Go text/template expression that references an event data parameter. +func newTemplateStringInserts() *stringInserts { + si := &stringInserts{} + + for i := 0; i < len(si.evtVariants); i++ { + // Use i+1 to keep our inserts numbered the same as Window's inserts. + strSlice, err := windows.UTF16FromString(`{{eventParam $ ` + strconv.Itoa(i+1) + `}}`) + if err != nil { + // This will never happen. + panic(err) + } + + si.insertStrings[i] = strSlice + si.evtVariants[i] = EvtVariant{ + Value: uintptr(unsafe.Pointer(&strSlice[0])), + Count: uint32(len(strSlice)), + Type: EvtVarTypeString, + } + si.evtVariants[i].Type = EvtVarTypeString + } + + return si +} diff --git a/winlogbeat/eventlog/common_test.go b/winlogbeat/sys/wineventlog/stringinserts_test.go similarity index 50% rename from winlogbeat/eventlog/common_test.go rename to winlogbeat/sys/wineventlog/stringinserts_test.go index a7f3f8b2cf1..ffeb2a473d6 100644 --- a/winlogbeat/eventlog/common_test.go +++ b/winlogbeat/sys/wineventlog/stringinserts_test.go @@ -15,36 +15,32 @@ // specific language governing permissions and limitations // under the License. -package eventlog +// +build windows + +package wineventlog import ( "testing" + "unsafe" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/winlogbeat/checkpoint" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" ) -type factory func(*common.Config) (EventLog, error) -type teardown func() +func TestStringInserts(t *testing.T) { + assert.NotNil(t, templateInserts) -func fatalErr(t *testing.T, err error) { - if err != nil { - t.Fatal(err) - } -} + si := newTemplateStringInserts() + defer si.clear() -func newTestEventLog(t *testing.T, factory factory, options map[string]interface{}) EventLog { - config, err := common.NewConfigFrom(options) - fatalErr(t, err) - eventLog, err := factory(config) - fatalErr(t, err) - return eventLog -} + // "The value of n can be a number between 1 and 99." + // https://docs.microsoft.com/en-us/windows/win32/eventlog/message-text-files + assert.Contains(t, windows.UTF16ToString(si.insertStrings[0]), " 1}") + assert.Contains(t, windows.UTF16ToString(si.insertStrings[maxInsertStrings-1]), " 99}") -func setupEventLog(t *testing.T, factory factory, recordID uint64, options map[string]interface{}) (EventLog, teardown) { - eventLog := newTestEventLog(t, factory, options) - fatalErr(t, eventLog.Open(checkpoint.EventLogState{ - RecordNumber: recordID, - })) - return eventLog, func() { fatalErr(t, eventLog.Close()) } + for i, evtVariant := range si.evtVariants { + assert.EqualValues(t, uintptr(unsafe.Pointer(&si.insertStrings[i][0])), evtVariant.Value) + assert.Len(t, si.insertStrings[i], int(evtVariant.Count)) + assert.Equal(t, evtVariant.Type, EvtVarTypeString) + } } diff --git a/winlogbeat/sys/wineventlog/syscall_windows.go b/winlogbeat/sys/wineventlog/syscall_windows.go index ed7dfa67224..13beb04fd85 100644 --- a/winlogbeat/sys/wineventlog/syscall_windows.go +++ b/winlogbeat/sys/wineventlog/syscall_windows.go @@ -18,24 +18,31 @@ package wineventlog import ( + "fmt" "syscall" + "time" + "unsafe" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" ) // EvtHandle is a handle to the event log. type EvtHandle uintptr +func (h EvtHandle) Close() error { + return _EvtClose(h) +} + +const NilHandle EvtHandle = 0 + // Event log error codes. // https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx const ( - ERROR_INSUFFICIENT_BUFFER syscall.Errno = 122 - ERROR_NO_MORE_ITEMS syscall.Errno = 259 - ERROR_NONE_MAPPED syscall.Errno = 1332 - RPC_S_INVALID_BOUND syscall.Errno = 1734 - ERROR_INVALID_OPERATION syscall.Errno = 4317 - ERROR_EVT_MESSAGE_NOT_FOUND syscall.Errno = 15027 - ERROR_EVT_MESSAGE_ID_NOT_FOUND syscall.Errno = 15028 - ERROR_EVT_UNRESOLVED_VALUE_INSERT syscall.Errno = 15029 - ERROR_EVT_UNRESOLVED_PARAMETER_INSERT syscall.Errno = 15030 + ERROR_INSUFFICIENT_BUFFER syscall.Errno = 122 + ERROR_NO_MORE_ITEMS syscall.Errno = 259 + RPC_S_INVALID_BOUND syscall.Errno = 1734 + ERROR_INVALID_OPERATION syscall.Errno = 4317 ) // EvtSubscribeFlag defines the possible values that specify when to start subscribing to events. @@ -184,7 +191,7 @@ const ( var evtSystemMap = map[EvtSystemPropertyID]string{ EvtSystemProviderName: "Provider Name", - EvtSystemProviderGuid: "Provider GUID", + EvtSystemProviderGuid: "Provider PublisherGUID", EvtSystemEventID: "Event ID", EvtSystemQualifiers: "Qualifiers", EvtSystemLevel: "Level", @@ -299,11 +306,308 @@ const ( EvtSeekStrict EvtSeekFlag = 0x10000 ) -// Add -trace to enable debug prints around syscalls. -//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go +type EvtVariantType uint32 + +const ( + EvtVarTypeNull EvtVariantType = iota + EvtVarTypeString + EvtVarTypeAnsiString + EvtVarTypeSByte + EvtVarTypeByte + EvtVarTypeInt16 + EvtVarTypeUInt16 + EvtVarTypeInt32 + EvtVarTypeUInt32 + EvtVarTypeInt64 + EvtVarTypeUInt64 + EvtVarTypeSingle + EvtVarTypeDouble + EvtVarTypeBoolean + EvtVarTypeBinary + EvtVarTypeGuid + EvtVarTypeSizeT + EvtVarTypeFileTime + EvtVarTypeSysTime + EvtVarTypeSid + EvtVarTypeHexInt32 + EvtVarTypeHexInt64 + EvtVarTypeEvtHandle EvtVariantType = 32 + EvtVarTypeEvtXml EvtVariantType = 35 +) + +var evtVariantTypeNames = map[EvtVariantType]string{ + EvtVarTypeNull: "null", + EvtVarTypeString: "string", + EvtVarTypeAnsiString: "ansi_string", + EvtVarTypeSByte: "signed_byte", + EvtVarTypeByte: "unsigned byte", + EvtVarTypeInt16: "int16", + EvtVarTypeUInt16: "uint16", + EvtVarTypeInt32: "int32", + EvtVarTypeUInt32: "uint32", + EvtVarTypeInt64: "int64", + EvtVarTypeUInt64: "uint64", + EvtVarTypeSingle: "float32", + EvtVarTypeDouble: "float64", + EvtVarTypeBoolean: "boolean", + EvtVarTypeBinary: "binary", + EvtVarTypeGuid: "guid", + EvtVarTypeSizeT: "size_t", + EvtVarTypeFileTime: "filetime", + EvtVarTypeSysTime: "systemtime", + EvtVarTypeSid: "sid", + EvtVarTypeHexInt32: "hex_int32", + EvtVarTypeHexInt64: "hex_int64", + EvtVarTypeEvtHandle: "evt_handle", + EvtVarTypeEvtXml: "evt_xml", +} + +func (t EvtVariantType) Mask() EvtVariantType { + return t & EvtVariantTypeMask +} + +func (t EvtVariantType) IsArray() bool { + return t&EvtVariantTypeArray > 0 +} + +func (t EvtVariantType) String() string { + return evtVariantTypeNames[t.Mask()] +} + +const ( + EvtVariantTypeMask = 0x7f + EvtVariantTypeArray = 128 +) + +type EvtVariant struct { + Value uintptr + Count uint32 + Type EvtVariantType +} + +var sizeofEvtVariant = unsafe.Sizeof(EvtVariant{}) + +type hexInt32 int32 + +func (n hexInt32) String() string { + return fmt.Sprintf("%#x", uint32(n)) +} + +type hexInt64 int64 + +func (n hexInt64) String() string { + return fmt.Sprintf("%#x", uint64(n)) +} + +func (v EvtVariant) Data(buf []byte) (interface{}, error) { + typ := v.Type.Mask() + switch typ { + case EvtVarTypeNull: + return nil, nil + case EvtVarTypeString: + addr := unsafe.Pointer(&buf[0]) + offset := v.Value - uintptr(addr) + s, err := UTF16BytesToString(buf[offset:]) + return s, err + case EvtVarTypeSByte: + return int8(v.Value), nil + case EvtVarTypeByte: + return uint8(v.Value), nil + case EvtVarTypeInt16: + return int16(v.Value), nil + case EvtVarTypeInt32: + return int32(v.Value), nil + case EvtVarTypeHexInt32: + return hexInt32(v.Value), nil + case EvtVarTypeInt64: + return int64(v.Value), nil + case EvtVarTypeHexInt64: + return hexInt64(v.Value), nil + case EvtVarTypeUInt16: + return uint16(v.Value), nil + case EvtVarTypeUInt32: + return uint32(v.Value), nil + case EvtVarTypeUInt64: + return uint64(v.Value), nil + case EvtVarTypeSingle: + return float32(v.Value), nil + case EvtVarTypeDouble: + return float64(v.Value), nil + case EvtVarTypeBoolean: + if v.Value == 0 { + return false, nil + } + return true, nil + case EvtVarTypeGuid: + addr := unsafe.Pointer(&buf[0]) + offset := v.Value - uintptr(addr) + guid := (*windows.GUID)(unsafe.Pointer(&buf[offset])) + copy := *guid + return copy, nil + case EvtVarTypeFileTime: + ft := (*windows.Filetime)(unsafe.Pointer(&v.Value)) + return time.Unix(0, ft.Nanoseconds()).UTC(), nil + case EvtVarTypeSid: + addr := unsafe.Pointer(&buf[0]) + offset := v.Value - uintptr(addr) + sidPtr := (*windows.SID)(unsafe.Pointer(&buf[offset])) + return sidPtr.Copy() + case EvtVarTypeEvtHandle: + return EvtHandle(v.Value), nil + default: + return nil, errors.Errorf("unhandled type: %d", typ) + } +} + +type EvtEventMetadataPropertyID uint32 + +const ( + EventMetadataEventID EvtEventMetadataPropertyID = iota + EventMetadataEventVersion + EventMetadataEventChannel + EventMetadataEventLevel + EventMetadataEventOpcode + EventMetadataEventTask + EventMetadataEventKeyword + EventMetadataEventMessageID + EventMetadataEventTemplate +) + +type EvtPublisherMetadataPropertyID uint32 + +const ( + EvtPublisherMetadataPublisherGuid EvtPublisherMetadataPropertyID = iota + EvtPublisherMetadataResourceFilePath + EvtPublisherMetadataParameterFilePath + EvtPublisherMetadataMessageFilePath + EvtPublisherMetadataHelpLink + EvtPublisherMetadataPublisherMessageID + EvtPublisherMetadataChannelReferences + EvtPublisherMetadataChannelReferencePath + EvtPublisherMetadataChannelReferenceIndex + EvtPublisherMetadataChannelReferenceID + EvtPublisherMetadataChannelReferenceFlags + EvtPublisherMetadataChannelReferenceMessageID + EvtPublisherMetadataLevels + EvtPublisherMetadataLevelName + EvtPublisherMetadataLevelValue + EvtPublisherMetadataLevelMessageID + EvtPublisherMetadataTasks + EvtPublisherMetadataTaskName + EvtPublisherMetadataTaskEventGuid + EvtPublisherMetadataTaskValue + EvtPublisherMetadataTaskMessageID + EvtPublisherMetadataOpcodes + EvtPublisherMetadataOpcodeName + EvtPublisherMetadataOpcodeValue + EvtPublisherMetadataOpcodeMessageID + EvtPublisherMetadataKeywords + EvtPublisherMetadataKeywordName + EvtPublisherMetadataKeywordValue + EvtPublisherMetadataKeywordMessageID +) + +func EvtGetPublisherMetadataProperty(publisherMetadataHandle EvtHandle, propertyID EvtPublisherMetadataPropertyID) (interface{}, error) { + var bufferUsed uint32 + err := _EvtGetPublisherMetadataProperty(publisherMetadataHandle, propertyID, 0, 0, nil, &bufferUsed) + if err != windows.ERROR_INSUFFICIENT_BUFFER { + return "", errors.Errorf("expected ERROR_INSUFFICIENT_BUFFER but got %v", err) + } + + buf := make([]byte, bufferUsed) + pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) + err = _EvtGetPublisherMetadataProperty(publisherMetadataHandle, propertyID, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtGetPublisherMetadataProperty") + } + + v, err := pEvtVariant.Data(buf) + if err != nil { + return nil, err + } + + switch t := v.(type) { + case EvtHandle: + return EvtObjectArrayPropertyHandle(t), nil + default: + return v, nil + } +} + +func EvtGetObjectArrayProperty(arrayHandle EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, index uint32) (interface{}, error) { + var bufferUsed uint32 + err := _EvtGetObjectArrayProperty(arrayHandle, propertyID, index, 0, 0, nil, &bufferUsed) + if err != windows.ERROR_INSUFFICIENT_BUFFER { + return nil, errors.Wrap(err, "failed in EvtGetObjectArrayProperty, expected ERROR_INSUFFICIENT_BUFFER") + } + + buf := make([]byte, bufferUsed) + pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) + err = _EvtGetObjectArrayProperty(arrayHandle, propertyID, index, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtGetObjectArrayProperty") + } + + value, err := pEvtVariant.Data(buf) + if err != nil { + return nil, errors.Wrap(err, "failed to read EVT_VARIANT value") + } + return value, nil +} + +type EvtObjectArrayPropertyHandle uint32 + +func (h EvtObjectArrayPropertyHandle) Close() error { + return _EvtClose(EvtHandle(h)) +} + +func EvtGetObjectArraySize(handle EvtObjectArrayPropertyHandle) (uint32, error) { + var arrayLen uint32 + if err := _EvtGetObjectArraySize(handle, &arrayLen); err != nil { + return 0, err + } + return arrayLen, nil +} + +func GetEventMetadataProperty(metadataHandle EvtHandle, propertyID EvtEventMetadataPropertyID) (interface{}, error) { + var bufferUsed uint32 + err := _EvtGetEventMetadataProperty(metadataHandle, 8, 0, 0, nil, &bufferUsed) + if err != windows.ERROR_INSUFFICIENT_BUFFER { + return nil, errors.Errorf("expected ERROR_INSUFFICIENT_BUFFER but got %v", err) + } + + buf := make([]byte, bufferUsed) + pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) + err = _EvtGetEventMetadataProperty(metadataHandle, propertyID, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) + if err != nil { + return nil, errors.Wrap(err, "_EvtGetEventMetadataProperty") + } + + return pEvtVariant.Data(buf) +} + +// EvtClearLog removes all events from the specified channel and writes them to +// the target log file. +func EvtClearLog(session EvtHandle, channelPath string, targetFilePath string) error { + channel, err := windows.UTF16PtrFromString(channelPath) + if err != nil { + return err + } + + var target *uint16 + if targetFilePath != "" { + target, err = windows.UTF16PtrFromString(targetFilePath) + if err != nil { + return err + } + } + + return _EvtClearLog(session, channel, target, 0) +} // Windows API calls //sys _EvtOpenLog(session EvtHandle, path *uint16, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenLog +//sys _EvtClearLog(session EvtHandle, channelPath *uint16, targetFilePath *uint16, flags uint32) (err error) = wevtapi.EvtClearLog //sys _EvtQuery(session EvtHandle, path *uint16, query *uint16, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtQuery //sys _EvtSubscribe(session EvtHandle, signalEvent uintptr, channelPath *uint16, query *uint16, bookmark EvtHandle, context uintptr, callback syscall.Handle, flags EvtSubscribeFlag) (handle EvtHandle, err error) = wevtapi.EvtSubscribe //sys _EvtCreateBookmark(bookmarkXML *uint16) (handle EvtHandle, err error) = wevtapi.EvtCreateBookmark @@ -317,5 +621,9 @@ const ( //sys _EvtNextChannelPath(channelEnum EvtHandle, channelPathBufferSize uint32, channelPathBuffer *uint16, channelPathBufferUsed *uint32) (err error) = wevtapi.EvtNextChannelPath //sys _EvtFormatMessage(publisherMetadata EvtHandle, event EvtHandle, messageID uint32, valueCount uint32, values uintptr, flags EvtFormatMessageFlag, bufferSize uint32, buffer *byte, bufferUsed *uint32) (err error) = wevtapi.EvtFormatMessage //sys _EvtOpenPublisherMetadata(session EvtHandle, publisherIdentity *uint16, logFilePath *uint16, locale uint32, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenPublisherMetadata - -//sys _StringFromGUID2(rguid *syscall.GUID, pStr *uint16, strSize uint32) (err error) = ole32.StringFromGUID2 +//sys _EvtGetPublisherMetadataProperty(publisherMetadata EvtHandle, propertyID EvtPublisherMetadataPropertyID, flags uint32, bufferSize uint32, variant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetPublisherMetadataProperty +//sys _EvtGetEventMetadataProperty(eventMetadata EvtHandle, propertyID EvtEventMetadataPropertyID, flags uint32, bufferSize uint32, variant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetEventMetadataProperty +//sys _EvtOpenEventMetadataEnum(publisherMetadata EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenEventMetadataEnum +//sys _EvtNextEventMetadata(enumerator EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtNextEventMetadata +//sys _EvtGetObjectArrayProperty(objectArray EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, arrayIndex uint32, flags uint32, bufferSize uint32, evtVariant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetObjectArrayProperty +//sys _EvtGetObjectArraySize(objectArray EvtObjectArrayPropertyHandle, arraySize *uint32) (err error) = wevtapi.EvtGetObjectArraySize diff --git a/winlogbeat/sys/wineventlog/template.go b/winlogbeat/sys/wineventlog/template.go new file mode 100644 index 00000000000..e5b5c9b99ae --- /dev/null +++ b/winlogbeat/sys/wineventlog/template.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package wineventlog + +import ( + "encoding/xml" +) + +type eventTemplate struct { + Data []eventData `xml:"data"` +} + +type eventData struct { + Name string `xml:"name,attr"` + Type string `xml:"outType,attr"` +} + +func (t *eventTemplate) Unmarshal(xmlData []byte) error { + return xml.Unmarshal(xmlData, t) +} diff --git a/winlogbeat/sys/wineventlog/template_test.go b/winlogbeat/sys/wineventlog/template_test.go new file mode 100644 index 00000000000..94b23fb9d1d --- /dev/null +++ b/winlogbeat/sys/wineventlog/template_test.go @@ -0,0 +1,43 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package wineventlog + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventTemplateUnmarshal(t *testing.T) { + const xmlTemplate = ` + +` + + et := &eventTemplate{} + assert.NoError(t, et.Unmarshal([]byte(xmlTemplate))) + assert.Len(t, et.Data, 8) +} diff --git a/winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.evtx b/winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.evtx new file mode 100644 index 00000000000..c37324d05be Binary files /dev/null and b/winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.evtx differ diff --git a/winlogbeat/sys/wineventlog/util_test.go b/winlogbeat/sys/wineventlog/util_test.go new file mode 100644 index 00000000000..5fd90f08003 --- /dev/null +++ b/winlogbeat/sys/wineventlog/util_test.go @@ -0,0 +1,153 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/andrewkroh/sys/windows/svc/eventlog" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" +) + +const ( + winlogbeatTestLogName = "WinEventLogTestGo" + + security4752File = "../../../x-pack/winlogbeat/module/security/test/testdata/4752.evtx" + sysmon9File = "../../../x-pack/winlogbeat/module/sysmon/test/testdata/sysmon-9.01.evtx" + winErrorReportingFile = "testdata/application-windows-error-reporting.evtx" +) + +// createLog creates a new event log and returns a handle for writing events +// to the log. +func createLog(t testing.TB) (log *eventlog.Log, tearDown func()) { + const name = winlogbeatTestLogName + const source = "wineventlog_test" + + existed, err := eventlog.InstallAsEventCreate(name, source, eventlog.Error|eventlog.Warning|eventlog.Info) + if err != nil { + t.Fatal(err) + } + + if existed { + EvtClearLog(NilHandle, name, "") + } + + log, err = eventlog.Open(source) + if err != nil { + eventlog.RemoveSource(name, source) + eventlog.RemoveProvider(name) + t.Fatal(err) + } + + setLogSize(t, winlogbeatTestLogName, 1024*1024*1024) // 1 GiB + + tearDown = func() { + log.Close() + EvtClearLog(NilHandle, name, "") + eventlog.RemoveSource(name, source) + eventlog.RemoveProvider(name) + } + + return log, tearDown +} + +// openLog opens an event log or .evtx file for reading. +func openLog(t testing.TB, log string, eventIDFilters ...string) EvtHandle { + var ( + err error + path = log + flags EvtQueryFlag = EvtQueryReverseDirection + ) + + if info, err := os.Stat(log); err == nil && info.Mode().IsRegular() { + flags |= EvtQueryFilePath + } else { + flags |= EvtQueryChannelPath + } + + var query string + if len(eventIDFilters) > 0 { + // Convert to URI. + abs, err := filepath.Abs(log) + if err != nil { + t.Fatal(err) + } + path = "file://" + filepath.ToSlash(abs) + + query, err = Query{Log: path, EventID: strings.Join(eventIDFilters, ",")}.Build() + if err != nil { + t.Fatal(err) + } + path = "" + } + + h, err := EvtQuery(NilHandle, path, query, flags) + if err != nil { + t.Fatal("Failed to open log", log, err) + } + return h +} + +// nextHandle reads one handle from the log. It returns done=true when there +// are no more items to read. +func nextHandle(t *testing.T, log EvtHandle) (handle EvtHandle, done bool) { + var numReturned uint32 + var handles [1]EvtHandle + + err := _EvtNext(log, 1, &handles[0], 0, 0, &numReturned) + if err != nil { + if err == windows.ERROR_NO_MORE_ITEMS { + return NilHandle, true + } + t.Fatal(err) + } + + return handles[0], false +} + +// mustNextHandle reads one handle from the log. +func mustNextHandle(t *testing.T, log EvtHandle) EvtHandle { + h, done := nextHandle(t, log) + if done { + t.Fatal("No more items to read.") + } + return h +} + +func logAsJSON(t testing.TB, object interface{}) { + data, err := json.MarshalIndent(object, "", " ") + if err != nil { + t.Fatal(err) + } + t.Log(string(data)) +} + +func assertEqualIgnoreCase(t *testing.T, expected, actual string) { + t.Helper() + assert.Equal(t, + strings.ToLower(expected), + strings.ToLower(actual), + ) +} diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows_test.go b/winlogbeat/sys/wineventlog/wineventlog_windows_test.go index f5411f446b9..9701cfd3679 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows_test.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows_test.go @@ -108,7 +108,3 @@ func TestChannels(t *testing.T) { } } } - -func TestExtension(t *testing.T) { - filepath.Ext("sysmon") -} diff --git a/winlogbeat/sys/wineventlog/winmeta.go b/winlogbeat/sys/wineventlog/winmeta.go new file mode 100644 index 00000000000..140b1c00066 --- /dev/null +++ b/winlogbeat/sys/wineventlog/winmeta.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +// winMeta contains the static values that are a common across Windows. These +// values are from winmeta.xml inside the Windows SDK. +var winMeta = &publisherMetadataStore{ + Keywords: map[int64]string{ + 0: "AnyKeyword", + 0x1000000000000: "Response Time", + 0x4000000000000: "WDI Diag", + 0x8000000000000: "SQM", + 0x10000000000000: "Audit Failure", + 0x20000000000000: "Audit Success", + 0x40000000000000: "Correlation Hint", + 0x80000000000000: "Classic", + }, + Opcodes: map[uint8]string{ + 0: "Info", + 1: "Start", + 2: "Stop", + 3: "DCStart", + 4: "DCStop", + 5: "Extension", + 6: "Reply", + 7: "Resume", + 8: "Suspend", + 9: "Send", + }, + Levels: map[uint8]string{ + 0: "Information", // "Log Always", but Event Viewer shows Information. + 1: "Critical", + 2: "Error", + 3: "Warning", + 4: "Information", + 5: "Verbose", + }, + Tasks: map[uint16]string{ + 0: "None", + }, +} diff --git a/winlogbeat/sys/wineventlog/zsyscall_windows.go b/winlogbeat/sys/wineventlog/zsyscall_windows.go index 0e15ef6a3d6..2dbe865c1a3 100644 --- a/winlogbeat/sys/wineventlog/zsyscall_windows.go +++ b/winlogbeat/sys/wineventlog/zsyscall_windows.go @@ -55,23 +55,28 @@ func errnoErr(e syscall.Errno) error { var ( modwevtapi = windows.NewLazySystemDLL("wevtapi.dll") - modole32 = windows.NewLazySystemDLL("ole32.dll") - procEvtOpenLog = modwevtapi.NewProc("EvtOpenLog") - procEvtQuery = modwevtapi.NewProc("EvtQuery") - procEvtSubscribe = modwevtapi.NewProc("EvtSubscribe") - procEvtCreateBookmark = modwevtapi.NewProc("EvtCreateBookmark") - procEvtUpdateBookmark = modwevtapi.NewProc("EvtUpdateBookmark") - procEvtCreateRenderContext = modwevtapi.NewProc("EvtCreateRenderContext") - procEvtRender = modwevtapi.NewProc("EvtRender") - procEvtClose = modwevtapi.NewProc("EvtClose") - procEvtSeek = modwevtapi.NewProc("EvtSeek") - procEvtNext = modwevtapi.NewProc("EvtNext") - procEvtOpenChannelEnum = modwevtapi.NewProc("EvtOpenChannelEnum") - procEvtNextChannelPath = modwevtapi.NewProc("EvtNextChannelPath") - procEvtFormatMessage = modwevtapi.NewProc("EvtFormatMessage") - procEvtOpenPublisherMetadata = modwevtapi.NewProc("EvtOpenPublisherMetadata") - procStringFromGUID2 = modole32.NewProc("StringFromGUID2") + procEvtOpenLog = modwevtapi.NewProc("EvtOpenLog") + procEvtClearLog = modwevtapi.NewProc("EvtClearLog") + procEvtQuery = modwevtapi.NewProc("EvtQuery") + procEvtSubscribe = modwevtapi.NewProc("EvtSubscribe") + procEvtCreateBookmark = modwevtapi.NewProc("EvtCreateBookmark") + procEvtUpdateBookmark = modwevtapi.NewProc("EvtUpdateBookmark") + procEvtCreateRenderContext = modwevtapi.NewProc("EvtCreateRenderContext") + procEvtRender = modwevtapi.NewProc("EvtRender") + procEvtClose = modwevtapi.NewProc("EvtClose") + procEvtSeek = modwevtapi.NewProc("EvtSeek") + procEvtNext = modwevtapi.NewProc("EvtNext") + procEvtOpenChannelEnum = modwevtapi.NewProc("EvtOpenChannelEnum") + procEvtNextChannelPath = modwevtapi.NewProc("EvtNextChannelPath") + procEvtFormatMessage = modwevtapi.NewProc("EvtFormatMessage") + procEvtOpenPublisherMetadata = modwevtapi.NewProc("EvtOpenPublisherMetadata") + procEvtGetPublisherMetadataProperty = modwevtapi.NewProc("EvtGetPublisherMetadataProperty") + procEvtGetEventMetadataProperty = modwevtapi.NewProc("EvtGetEventMetadataProperty") + procEvtOpenEventMetadataEnum = modwevtapi.NewProc("EvtOpenEventMetadataEnum") + procEvtNextEventMetadata = modwevtapi.NewProc("EvtNextEventMetadata") + procEvtGetObjectArrayProperty = modwevtapi.NewProc("EvtGetObjectArrayProperty") + procEvtGetObjectArraySize = modwevtapi.NewProc("EvtGetObjectArraySize") ) func _EvtOpenLog(session EvtHandle, path *uint16, flags uint32) (handle EvtHandle, err error) { @@ -87,6 +92,18 @@ func _EvtOpenLog(session EvtHandle, path *uint16, flags uint32) (handle EvtHandl return } +func _EvtClearLog(session EvtHandle, channelPath *uint16, targetFilePath *uint16, flags uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEvtClearLog.Addr(), 4, uintptr(session), uintptr(unsafe.Pointer(channelPath)), uintptr(unsafe.Pointer(targetFilePath)), uintptr(flags), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + func _EvtQuery(session EvtHandle, path *uint16, query *uint16, flags uint32) (handle EvtHandle, err error) { r0, _, e1 := syscall.Syscall6(procEvtQuery.Addr(), 4, uintptr(session), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(query)), uintptr(flags), 0, 0) handle = EvtHandle(r0) @@ -250,8 +267,70 @@ func _EvtOpenPublisherMetadata(session EvtHandle, publisherIdentity *uint16, log return } -func _StringFromGUID2(rguid *syscall.GUID, pStr *uint16, strSize uint32) (err error) { - r1, _, e1 := syscall.Syscall(procStringFromGUID2.Addr(), 3, uintptr(unsafe.Pointer(rguid)), uintptr(unsafe.Pointer(pStr)), uintptr(strSize)) +func _EvtGetPublisherMetadataProperty(publisherMetadata EvtHandle, propertyID EvtPublisherMetadataPropertyID, flags uint32, bufferSize uint32, variant *EvtVariant, bufferUsed *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEvtGetPublisherMetadataProperty.Addr(), 6, uintptr(publisherMetadata), uintptr(propertyID), uintptr(flags), uintptr(bufferSize), uintptr(unsafe.Pointer(variant)), uintptr(unsafe.Pointer(bufferUsed))) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtGetEventMetadataProperty(eventMetadata EvtHandle, propertyID EvtEventMetadataPropertyID, flags uint32, bufferSize uint32, variant *EvtVariant, bufferUsed *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEvtGetEventMetadataProperty.Addr(), 6, uintptr(eventMetadata), uintptr(propertyID), uintptr(flags), uintptr(bufferSize), uintptr(unsafe.Pointer(variant)), uintptr(unsafe.Pointer(bufferUsed))) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtOpenEventMetadataEnum(publisherMetadata EvtHandle, flags uint32) (handle EvtHandle, err error) { + r0, _, e1 := syscall.Syscall(procEvtOpenEventMetadataEnum.Addr(), 2, uintptr(publisherMetadata), uintptr(flags), 0) + handle = EvtHandle(r0) + if handle == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtNextEventMetadata(enumerator EvtHandle, flags uint32) (handle EvtHandle, err error) { + r0, _, e1 := syscall.Syscall(procEvtNextEventMetadata.Addr(), 2, uintptr(enumerator), uintptr(flags), 0) + handle = EvtHandle(r0) + if handle == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtGetObjectArrayProperty(objectArray EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, arrayIndex uint32, flags uint32, bufferSize uint32, evtVariant *EvtVariant, bufferUsed *uint32) (err error) { + r1, _, e1 := syscall.Syscall9(procEvtGetObjectArrayProperty.Addr(), 7, uintptr(objectArray), uintptr(propertyID), uintptr(arrayIndex), uintptr(flags), uintptr(bufferSize), uintptr(unsafe.Pointer(evtVariant)), uintptr(unsafe.Pointer(bufferUsed)), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtGetObjectArraySize(objectArray EvtObjectArrayPropertyHandle, arraySize *uint32) (err error) { + r1, _, e1 := syscall.Syscall(procEvtGetObjectArraySize.Addr(), 2, uintptr(objectArray), uintptr(unsafe.Pointer(arraySize)), 0) if r1 == 0 { if e1 != 0 { err = errnoErr(e1)