Do not escape HTML by default anymore (#9914)

- export json encoder config object for initialization - set escape_html default to false json formatter - set escape_html default to false in ES, Console, LS outputs (other outputs require explicit formatter) - Adapt reference configs - Update docs
elastic · Jan 7, 2019 · f5c4544 · f5c4544
1 parent d59ae8c
commit f5c4544
Show file tree

Hide file tree

Showing 23 changed files with 114 additions and 100 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -15,6 +15,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename beat.timezone to event.timezone. {pull}9458[9458]
 - Use _doc as document type. {pull}9056[9056]{pull}9573[9573]
 - Update to Golang 1.11.3. {pull}9560[9560]
+- Embedded html is not escaped anymore by default. {pull}9914[9914]
 
 *Auditbeat*
 

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -367,7 +367,7 @@ output.elasticsearch:
   #compression_level: 0
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional protocol and basic auth credentials.
   #protocol: "https"
@@ -474,7 +474,7 @@ output.elasticsearch:
   #compression_level: 3
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional maximum time to live for a connection to Logstash, after which the
   # connection will be re-established.  A value of `0s` (the default) will
@@ -611,7 +611,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Metadata update configuration. Metadata contains leader information
   # used to decide which broker to use when publishing.
@@ -725,7 +725,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # The list of Redis servers to connect to. If load-balancing is enabled, the
   # events are distributed to the servers in the list. If one server becomes
@@ -845,7 +845,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Path to the directory where to save the generated files. The option is
   # mandatory.
@@ -880,7 +880,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
 #================================= Paths ======================================
 

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -1062,7 +1062,7 @@ output.elasticsearch:
   #compression_level: 0
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional protocol and basic auth credentials.
   #protocol: "https"
@@ -1169,7 +1169,7 @@ output.elasticsearch:
   #compression_level: 3
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional maximum time to live for a connection to Logstash, after which the
   # connection will be re-established.  A value of `0s` (the default) will
@@ -1306,7 +1306,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Metadata update configuration. Metadata contains leader information
   # used to decide which broker to use when publishing.
@@ -1420,7 +1420,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # The list of Redis servers to connect to. If load-balancing is enabled, the
   # events are distributed to the servers in the list. If one server becomes
@@ -1540,7 +1540,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Path to the directory where to save the generated files. The option is
   # mandatory.
@@ -1575,7 +1575,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
 #================================= Paths ======================================
 

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -511,7 +511,7 @@ output.elasticsearch:
   #compression_level: 0
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional protocol and basic auth credentials.
   #protocol: "https"
@@ -618,7 +618,7 @@ output.elasticsearch:
   #compression_level: 3
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional maximum time to live for a connection to Logstash, after which the
   # connection will be re-established.  A value of `0s` (the default) will
@@ -755,7 +755,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Metadata update configuration. Metadata contains leader information
   # used to decide which broker to use when publishing.
@@ -869,7 +869,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # The list of Redis servers to connect to. If load-balancing is enabled, the
   # events are distributed to the servers in the list. If one server becomes
@@ -989,7 +989,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Path to the directory where to save the generated files. The option is
   # mandatory.
@@ -1024,7 +1024,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
 #================================= Paths ======================================
 

diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
@@ -301,7 +301,7 @@ output.elasticsearch:
   #compression_level: 0
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional protocol and basic auth credentials.
   #protocol: "https"
@@ -408,7 +408,7 @@ output.elasticsearch:
   #compression_level: 3
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional maximum time to live for a connection to Logstash, after which the
   # connection will be re-established.  A value of `0s` (the default) will
@@ -545,7 +545,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Metadata update configuration. Metadata contains leader information
   # used to decide which broker to use when publishing.
@@ -659,7 +659,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # The list of Redis servers to connect to. If load-balancing is enabled, the
   # events are distributed to the servers in the list. If one server becomes
@@ -779,7 +779,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Path to the directory where to save the generated files. The option is
   # mandatory.
@@ -814,7 +814,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
 #================================= Paths ======================================
 

diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml
@@ -255,7 +255,7 @@ output.elasticsearch:
   #compression_level: 0
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional protocol and basic auth credentials.
   #protocol: "https"
@@ -362,7 +362,7 @@ output.elasticsearch:
   #compression_level: 3
 
   # Configure escaping HTML symbols in strings.
-  #escape_html: true
+  #escape_html: false
 
   # Optional maximum time to live for a connection to Logstash, after which the
   # connection will be re-established.  A value of `0s` (the default) will
@@ -499,7 +499,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Metadata update configuration. Metadata contains leader information
   # used to decide which broker to use when publishing.
@@ -613,7 +613,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # The list of Redis servers to connect to. If load-balancing is enabled, the
   # events are distributed to the servers in the list. If one server becomes
@@ -733,7 +733,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
   # Path to the directory where to save the generated files. The option is
   # mandatory.
@@ -768,7 +768,7 @@ output.elasticsearch:
     #pretty: false
 
     # Configure escaping HTML symbols in strings.
-    #escape_html: true
+    #escape_html: false
 
 #================================= Paths ======================================
 

diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc
@@ -145,9 +145,9 @@ The default value is 0.
 
 ===== `escape_html`
 
-Configure escaping of HTML in strings. Set to `false` to disable escaping.
+Configure escaping of HTML in strings. Set to `true` to enable escaping.
 
-The default value is `true`.
+The default value is `false`.
 
 
 ===== `worker`
@@ -605,9 +605,9 @@ The default value is 3.
 
 ===== `escape_html`
 
-Configure escaping of HTML in strings. Set to `false` to disable escaping.
+Configure escaping of HTML in strings. Set to `true` to enable escaping.
 
-The default value is `true`.
+The default value is `false`.
 
 ===== `worker`
 
@@ -1429,7 +1429,7 @@ codec. By default the `json` codec is used.
 
 *`json.pretty`*: If `pretty` is set to true, events will be nicely formatted. The default is false.
 
-*`json.escape_html`*: If `escape_html` is set to false, html symbols will not be escaped in strings. The default is true.
+*`json.escape_html`*: If `escape_html` is set to true, html symbols will be escaped in strings. The default is false.
 
 Example configuration that uses the `json` codec with pretty printing enabled to write events to the console:
 

diff --git a/libbeat/outputs/codec/json/json.go b/libbeat/outputs/codec/json/json.go
@@ -34,17 +34,18 @@ type Encoder struct {
 	folder *gotype.Iterator
 
 	version string
-	config  config
+	config  Config
 }
 
-type config struct {
+// Config is used to pass encoding parameters to New.
+type Config struct {
 	Pretty     bool
 	EscapeHTML bool
 }
 
-var defaultConfig = config{
+var defaultConfig = Config{
 	Pretty:     false,
-	EscapeHTML: true,
+	EscapeHTML: false,
 }
 
 func init() {
@@ -56,16 +57,13 @@ func init() {
 			}
 		}
 
-		return New(config.Pretty, config.EscapeHTML, info.Version), nil
+		return New(info.Version, config), nil
 	})
 }
 
 // New creates a new json Encoder.
-func New(pretty, escapeHTML bool, version string) *Encoder {
-	e := &Encoder{version: version, config: config{
-		Pretty:     pretty,
-		EscapeHTML: escapeHTML,
-	}}
+func New(version string, config Config) *Encoder {
+	e := &Encoder{version: version, config: config}
 	e.reset()
 	return e
 }

diff --git a/libbeat/outputs/codec/json/json_test.go b/libbeat/outputs/codec/json/json_test.go
@@ -26,7 +26,7 @@ import (
 
 func TestJsonCodec(t *testing.T) {
 	type testCase struct {
-		config   config
+		config   Config
 		in       common.MapStr
 		expected string
 	}
@@ -38,7 +38,7 @@ func TestJsonCodec(t *testing.T) {
 			expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
 		},
 		"pretty enabled": testCase{
-			config: config{Pretty: true},
+			config: Config{Pretty: true},
 			in:     common.MapStr{"msg": "message"},
 			expected: `{
   "@timestamp": "0001-01-01T00:00:00.000Z",
@@ -51,12 +51,12 @@ func TestJsonCodec(t *testing.T) {
 }`,
 		},
 		"html escaping enabled": testCase{
-			config:   config{EscapeHTML: true},
+			config:   Config{EscapeHTML: true},
 			in:       common.MapStr{"msg": "<hello>world</hello>"},
 			expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"\u003chello\u003eworld\u003c/hello\u003e"}`,
 		},
 		"html escaping disabled": testCase{
-			config:   config{EscapeHTML: false},
+			config:   Config{EscapeHTML: false},
 			in:       common.MapStr{"msg": "<hello>world</hello>"},
 			expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"<hello>world</hello>"}`,
 		},
@@ -66,7 +66,7 @@ func TestJsonCodec(t *testing.T) {
 		cfg, fields, expected := test.config, test.in, test.expected
 
 		t.Run(name, func(t *testing.T) {
-			codec := New(cfg.Pretty, cfg.EscapeHTML, "1.2.3")
+			codec := New("1.2.3", cfg)
 			actual, err := codec.Encode("test", &beat.Event{Fields: fields})
 
 			if err != nil {