Cherry-pick elastic#11357 to 5.6: Fix Winlogbeat escaping CRLF sequen…

…ces (elastic#11372)

Previous fix (elastic#11006) made Winlogbeat escape CRLF control characters
which are expected in Windows event logs.

Fixes elastic#11328

(cherry picked from commit 6865403)

CHANGELOG.asciidoc

@@ -39,6 +39,7 @@ https://github.com/elastic/beats/compare/v5.6.16...5.6[Check the HEAD diff]
 *Winlogbeat*
 
 - Prevent Winlogbeat from dropping events with invalid XML. {pull}11006{11006}
+- Fix Winlogbeat escaping CR, LF and TAB characters. {issue}11328[11328] {pull}11357[11357]
 
 ==== Added
 

winlogbeat/sys/event_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -151,21 +152,13 @@ func TestXML(t *testing.T) {
 	}
 }
 
+// Tests that control characters other than CR and LF are escaped
+// when the event is decoded.
 func TestInvalidXML(t *testing.T) {
-	eventXML := fmt.Sprintf(`
-<Event>
-  <UserData>
-    <Operation_ClientFailure xmlns='http://manifests.microsoft.com/win/2006/windows/WMI'>
-      <Id>{00000000-0000-0000-0000-000000000000}</Id>
-      <Message>じゃあ宇宙カウボーイ。。。%s</Message>
-    </Operation_ClientFailure>
-  </UserData>
-</Event>
-`, "\x1b")
-	_, err := UnmarshalEventXML([]byte(eventXML))
-	if !assert.NoError(t, err) {
-		assert.Equal(t, err.Error(), "XML syntax error on line 6: illegal character code U+001B")
-	}
+	evXML := strings.Replace(allXML, "%1", "\t&#xD;\n\x1b", -1)
+	ev, err := UnmarshalEventXML([]byte(evXML))
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "Creating WSMan shell on server with ResourceUri: \t\r\n\\u001b", ev.Message)
 }
 
 func BenchmarkXMLUnmarshal(b *testing.B) {

winlogbeat/sys/xmlreader.go

@@ -58,7 +58,7 @@ func (r *xmlSafeReader) Read(d []byte) (n int, err error) {
 	}
 	for i := 0; i < len(r.buf); {
 		code, size := utf8.DecodeRune(r.buf[i:])
-		if unicode.IsControl(code) {
+		if !unicode.IsSpace(code) && unicode.IsControl(code) {
 			n = copy(d, r.buf[:i])
 			r.buf = r.buf[n+1:]
 			r.code = []byte(fmt.Sprintf("\\u%04x", code))

winlogbeat/tests/system/test_eventlogging.py

@@ -169,3 +169,27 @@ def test_utf16_characters(self):
         })
         self.assertTrue(len(evts), 1)
         self.assertEqual(evts[0]["message"], msg)
+
+    def test_multiline_events(self):
+        """
+        eventlogging - Event with newlines and control characters
+        """
+        msg = """
+A trusted logon process has been registered with the Local Security Authority.
+This logon process will be trusted to submit logon requests.
+
+Subject:
+
+Security ID:  SYSTEM
+Account Name:  MS4\x1e$
+Account Domain:  WORKGROUP
+Logon ID:  0x3e7
+Logon Process Name:  IKE"""
+        self.write_event_log(msg)
+        evts = self.read_events()
+        self.assertTrue(len(evts), 1)
+        self.assertEqual(unicode(self.api), evts[0]["type"], evts[0])
+        self.assertNotIn("event.original", evts[0], msg=evts[0])
+        self.assertIn("message", evts[0], msg=evts[0])
+        self.assertNotIn("\\u000a", evts[0]["message"], msg=evts[0])
+        self.assertEqual(unicode(msg), evts[0]["message"].decode('unicode-escape'), msg=evts[0])

winlogbeat/tests/system/test_wineventlog.py

@@ -307,3 +307,27 @@ def test_utf16_characters(self):
         })
         self.assertTrue(len(evts), 1)
         self.assertEqual(evts[0]["message"], msg)
+
+    def test_multiline_events(self):
+        """
+        wineventlog - Event with newlines and control characters
+        """
+        msg = """
+A trusted logon process has been registered with the Local Security Authority.
+This logon process will be trusted to submit logon requests.
+
+Subject:
+
+Security ID:  SYSTEM
+Account Name:  MS4\x1e$
+Account Domain:  WORKGROUP
+Logon ID:  0x3e7
+Logon Process Name:  IKE"""
+        self.write_event_log(msg)
+        evts = self.read_events()
+        self.assertTrue(len(evts), 1)
+        self.assertEqual(unicode(self.api), evts[0]["type"], msg=evts[0])
+        self.assertNotIn("event.original", evts[0], msg=evts[0])
+        self.assertIn("message", evts[0], msg=evts[0])
+        self.assertNotIn("\\u000a", evts[0]["message"], msg=evts[0])
+        self.assertEqual(unicode(msg), evts[0]["message"].decode('unicode-escape'), msg=evts[0])