Remove storing context in ES. (#1863)

fixes #411

model/error/event.go

@@ -128,7 +128,6 @@ func DecodeEvent(input interface{}, err error) (transform.Transformable, error)
 	}
 
 	if labels, ok := e.Context["tags"].(map[string]interface{}); ok {
-		delete(e.Context, "tags")
 		e.Labels = labels
 	}
 
@@ -203,21 +202,14 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event {
 		"error":     e.fields(tctx),
 		"processor": processorEntry,
 	}
-	delete(e.Context, "page")
-	delete(e.Context, "custom")
-	delete(e.Context, "user")
 	utility.Add(fields, "user", e.User.Fields())
 	utility.Add(fields, "client", e.User.ClientFields())
 	utility.Add(fields, "user_agent", e.User.UserAgentFields())
 	utility.Add(fields, "labels", e.Labels)
-	tctx.Metadata.Merge(fields)
-
 	utility.Add(fields, "http", m.HttpFields(e.Context))
 	utility.Add(fields, "url", m.UrlFields(e.Context))
-	delete(e.Context, "request")
-	delete(e.Context, "response")
 
-	utility.Add(fields, "context", e.Context)
+	tctx.Metadata.Merge(fields)
 
 	// sampled and type is nil if an error happens outside a transaction or an (old) agent is not sending sampled info
 	// agents must send semantically correct data

model/error/event_test.go

@@ -204,6 +204,7 @@ func TestErrorEventDecode(t *testing.T) {
 				Page:      &m.Page{Url: &url, Referer: &referer},
 				Context: map[string]interface{}{"a": "b", "user": map[string]interface{}{
 					"username": name, "email": email, "ip": userIp, "id": userId},
+					"tags": map[string]interface{}{"ab": "c"},
 					"page": map[string]interface{}{"url": url, "referer": referer}},
 				Exception: &Exception{
 					Message:    &exMsg,
@@ -274,7 +275,7 @@ func TestEventFields(t *testing.T) {
 		LoggerName:   &loggerName,
 	}
 
-	context := common.MapStr{"user": common.MapStr{"id": "888"}, "c1": "val"}
+	context := common.MapStr{"a": "b", "user": common.MapStr{"id": "888"}, "c1": "val"}
 
 	baseExceptionHash := md5.New()
 	io.WriteString(baseExceptionHash, *baseException().Message)

model/span/event.go

@@ -194,7 +194,6 @@ func DecodeEvent(input interface{}, err error) (transform.Transformable, error)
 	}
 
 	if labels, ok := event.Context["tags"].(map[string]interface{}); ok {
-		delete(event.Context, "tags")
 		event.Labels = labels
 	}
 
@@ -247,10 +246,7 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event {
 		"processor": processorEntry,
 		spanDocType: e.fields(tctx),
 	}
-	delete(e.Context, "http")
-	delete(e.Context, "db")
 	utility.Add(fields, "labels", e.Labels)
-	utility.Add(fields, "context", e.Context)
 	utility.AddId(fields, "parent", &e.ParentId)
 	utility.AddId(fields, "trace", &e.TraceId)
 	utility.AddId(fields, "transaction", &e.TransactionId)

model/span/event_test.go

@@ -245,7 +245,6 @@ func TestSpanTransform(t *testing.T) {
 						"method":   "get",
 					},
 				},
-				"context":   common.MapStr{"key": "val"},
 				"labels":    common.MapStr{"label.a": 12},
 				"processor": common.MapStr{"event": "span", "name": "transaction"},
 				"service":   common.MapStr{"name": "myService"},

model/transaction/event.go

@@ -114,7 +114,6 @@ func DecodeEvent(input interface{}, err error) (transform.Transformable, error)
 	}
 
 	if labels, ok := e.Context["tags"].(map[string]interface{}); ok {
-		delete(e.Context, "tags")
 		e.Labels = labels
 	}
 
@@ -182,24 +181,17 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event {
 		"processor":        processorEntry,
 		transactionDocType: e.fields(tctx),
 	}
-	delete(e.Context, "custom")
-	delete(e.Context, "user")
 	utility.Add(fields, "user", e.User.Fields())
-	delete(e.Context, "page")
 	utility.AddId(fields, "parent", e.ParentId)
 	utility.AddId(fields, "trace", &e.TraceId)
 	utility.Add(fields, "timestamp", utility.TimeAsMicros(e.Timestamp))
 	utility.Add(fields, "client", e.User.ClientFields())
 	utility.Add(fields, "user_agent", e.User.UserAgentFields())
 	utility.Add(fields, "labels", e.Labels)
-	tctx.Metadata.Merge(fields)
-
 	utility.Add(fields, "http", m.HttpFields(e.Context))
 	utility.Add(fields, "url", m.UrlFields(e.Context))
-	delete(e.Context, "request")
-	delete(e.Context, "response")
 
-	utility.Add(fields, "context", e.Context)
+	tctx.Metadata.Merge(fields)
 
 	events = append(events, beat.Event{Fields: fields, Timestamp: e.Timestamp})
 

model/transaction/event_test.go

@@ -80,7 +80,8 @@ func TestTransactionEventDecode(t *testing.T) {
 				"id": id, "type": trType, "name": name, "result": result,
 				"duration": duration, "timestamp": timestampEpoch,
 				"context": map[string]interface{}{"a": "b", "user": map[string]interface{}{
-					"username": name, "email": email, "ip": userIp, "id": userId}, "tags": map[string]interface{}{"foo": "bar"},
+					"username": name, "email": email, "ip": userIp, "id": userId},
+					"tags": map[string]interface{}{"foo": "bar"},
 					"page": map[string]interface{}{"url": url, "referer": referer}},
 				"marks": marks, "sampled": sampled,
 				"parent_id": parentId, "trace_id": traceId,
@@ -106,6 +107,7 @@ func TestTransactionEventDecode(t *testing.T) {
 				Page:      &model.Page{Url: &url, Referer: &referer},
 				Context: map[string]interface{}{"a": "b", "user": map[string]interface{}{
 					"username": name, "email": email, "ip": userIp, "id": userId},
+					"tags": map[string]interface{}{"foo": "bar"},
 					"page": map[string]interface{}{"url": url, "referer": referer}},
 			},
 		},
@@ -241,8 +243,6 @@ func TestEventMoveContext(t *testing.T) {
 		"response": common.MapStr{"status_code": 200},
 	}, result[0].Fields["http"])
 	assert.Equal(t, common.MapStr{"original": "http://www.elastic.co"}, result[0].Fields["url"])
-	assert.Nil(t, event.Context["request"])
-	assert.Nil(t, event.Context["response"])
 	assert.NotNil(t, event.Context["db"])
 
 	portMapping := []struct {
@@ -326,7 +326,7 @@ func TestEventsTransformWithMetadata(t *testing.T) {
 	}
 	txWithContext := Event{
 		Timestamp: timestamp,
-		Context:   common.MapStr{"custom": common.MapStr{"foo": "bar"}},
+		Context:   common.MapStr{"foo": "bar", "custom": common.MapStr{"foo": "bar"}},
 		User:      &user,
 		Labels:    common.MapStr{"a": "b"},
 		Page:      &model.Page{Url: &url, Referer: &referer},

processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json

@@ -9,17 +9,6 @@
             "client": {
                 "ip": "192.0.0.1"
             },
-            "context": {
-                "environment": {
-                    "browserHeight": 726,
-                    "browserWidth": 150,
-                    "language": "en-US",
-                    "platform": "MacIntel",
-                    "screenHeight": 800,
-                    "screenWidth": 1280,
-                    "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36"
-                }
-            },
             "error": {
                 "culprit": "test/e2e/general-usecase/bundle.js.map",
                 "exception": [

testdata/intake-v2/errors_2.ndjson

@@ -1,5 +1,5 @@
 {"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"framework":{"name":"Express","version":"1.2.3"},"agent":{"name":"elastic-node","version":"3.14.0"}},"process":{"pid":1234,"ppid":7788,"title":"node","argv":["node","server.js"]},"system":{"hostname":"prod1.example.com","architecture":"x64","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","pod":{"uid":"pod-uid","name":"pod-name"}}}}}
-{"error":{"id":"5f0e9d64c1854d21a6f44673ed561ec8","timestamp":1494342245999999,"culprit":"my.module.function_name","log":{"message":"My service could not talk to the database named foobar","param_message":"My service could not talk to the database named %s","logger_name":"my.logger.name","level":"warning","stacktrace":[{"abs_path":"/real/file/name.py","filename":"/webpack/file/name.py","function":"foo","vars":{"key":"value"},"pre_context":["line1","line2"],"context_line":"line3","library_frame":false,"lineno":3,"module":"App::MyModule","colno":4,"post_context":["line4","line5"]},{"filename":"lib/instrumentation/index.js","lineno":102,"function":"instrumented","abs_path":"/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js","vars":{"key":"value"},"pre_context":["  var trans = this.currentTransaction","","  return instrumented","","  function instrumented () {","    var prev = ins.currentTransaction","    ins.currentTransaction = trans"],"context_line":"    var result = original.apply(this, arguments)","post_context":["    ins.currentTransaction = prev","    return result","}","}","","Instrumentation.prototype._recoverTransaction = function (trans) {","  if (this.currentTransaction === trans) return"]}]},"exception":{"message":"The username root is unknown","type":"DbError","module":"__builtins__","code":42,"handled":false,"attributes":{"foo":"bar"},"stacktrace":[{"abs_path":"/real/file/name.py","filename":"file/name.py","function":"foo","vars":{"key":"value"},"pre_context":["line1","line2"],"context_line":"line3","library_frame":true,"lineno":3,"module":"App::MyModule","colno":4,"post_context":["line4","line5"]},{"filename":"lib/instrumentation/index.js","lineno":102,"function":"instrumented","abs_path":"/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js","vars":{"key":"value"},"pre_context":["  var trans = this.currentTransaction","","  return instrumented","","  function instrumented () {","    var prev = ins.currentTransaction","    ins.currentTransaction = trans"],"context_line":"    var result = original.apply(this, arguments)","post_context":["    ins.currentTransaction = prev","    return result","}","}","","Instrumentation.prototype._recoverTransaction = function (trans) {","  if (this.currentTransaction === trans) return"]}]},"context":{"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"},"request":{"socket":{"remote_address":"12.53.12.1","encrypted":true},"http_version":"1.1","method":"POST","url":{"protocol":"https:","full":"https://www.example.com/p/a/t/h?query=string#hash","hostname":"www.example.com","port":"8080","pathname":"/p/a/t/h","search":"?query=string","hash":"#hash","raw":"/p/a/t/h?query=string#hash"},"headers":{"user-agent":"Mozilla Chrome Edge","content-type":"text/html","cookie":"c1=v1; c2=v2","some-other-header":"foo","array":["foo","bar","baz"]},"cookies":{"c1":"v1","c2":"v2"},"env":{"SERVER_SOFTWARE":"nginx","GATEWAY_INTERFACE":"CGI/1.1"},"body":"Hello World"},"response":{"status_code":200,"headers":{"content-type":"application/json"},"headers_sent":true,"finished":true},"user":{"id":99,"username":"foo","email":"foo@example.com"},"tags":{"organization_uuid":"9f0e9d64-c185-4d21-a6f4-4673ed561ec8"},"custom":{"my_key":1,"some_other_value":"foo bar","and_objects":{"foo":["bar","baz"]}}},"transaction":{"id":"945254c5-67a5-417e-8a4e-aa29efcbfb79"}}}
+{"error":{"id":"5f0e9d64c1854d21a6f44673ed561ec8","timestamp":1494342245999999,"culprit":"my.module.function_name","log":{"message":"My service could not talk to the database named foobar","param_message":"My service could not talk to the database named %s","logger_name":"my.logger.name","level":"warning","stacktrace":[{"abs_path":"/real/file/name.py","filename":"/webpack/file/name.py","function":"foo","vars":{"key":"value"},"pre_context":["line1","line2"],"context_line":"line3","library_frame":false,"lineno":3,"module":"App::MyModule","colno":4,"post_context":["line4","line5"]},{"filename":"lib/instrumentation/index.js","lineno":102,"function":"instrumented","abs_path":"/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js","vars":{"key":"value"},"pre_context":["  var trans = this.currentTransaction","","  return instrumented","","  function instrumented () {","    var prev = ins.currentTransaction","    ins.currentTransaction = trans"],"context_line":"    var result = original.apply(this, arguments)","post_context":["    ins.currentTransaction = prev","    return result","}","}","","Instrumentation.prototype._recoverTransaction = function (trans) {","  if (this.currentTransaction === trans) return"]}]},"exception":{"message":"The username root is unknown","type":"DbError","module":"__builtins__","code":42,"handled":false,"attributes":{"foo":"bar"},"stacktrace":[{"abs_path":"/real/file/name.py","filename":"file/name.py","function":"foo","vars":{"key":"value"},"pre_context":["line1","line2"],"context_line":"line3","library_frame":true,"lineno":3,"module":"App::MyModule","colno":4,"post_context":["line4","line5"]},{"filename":"lib/instrumentation/index.js","lineno":102,"function":"instrumented","abs_path":"/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js","vars":{"key":"value"},"pre_context":["  var trans = this.currentTransaction","","  return instrumented","","  function instrumented () {","    var prev = ins.currentTransaction","    ins.currentTransaction = trans"],"context_line":"    var result = original.apply(this, arguments)","post_context":["    ins.currentTransaction = prev","    return result","}","}","","Instrumentation.prototype._recoverTransaction = function (trans) {","  if (this.currentTransaction === trans) return"]}]},"context":{"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"},"request":{"socket":{"remote_address":"12.53.12.1","encrypted":true},"http_version":"1.1","method":"POST","url":{"protocol":"https:","full":"https://www.example.com/p/a/t/h?query=string#hash","hostname":"www.example.com","port":"8080","pathname":"/p/a/t/h","search":"?query=string","hash":"#hash","raw":"/p/a/t/h?query=string#hash"},"headers":{"user-agent":"Mozilla Chrome Edge","content-type":"text/html","cookie":"c1=v1; c2=v2","some-other-header":"foo","array":["foo","bar","baz"]},"cookies":{"c1":"v1","c2":"v2"},"env":{"SERVER_SOFTWARE":"nginx","GATEWAY_INTERFACE":"CGI/1.1"},"body":"Hello World"},"response":{"status_code":200,"headers":{"content-type":"application/json"},"headers_sent":true,"finished":true},"user":{"id":99,"username":"foo","email":"foo@example.com"},"tags":{"organization_uuid":"9f0e9d64-c185-4d21-a6f4-4673ed561ec8"},"foo": "bar","custom":{"my_key":1,"some_other_value":"foo bar","and_objects":{"foo":["bar","baz"]}}},"transaction":{"id":"945254c5-67a5-417e-8a4e-aa29efcbfb79"}}}
 {"error":{"id":"8f0e9d68c1854d21a6f44673ed561ec8","timestamp":1494342245000000,"exception":{"message":"foo is not defined","code":"35"}}}
 {"error":{"id":"7f0e9d68c1854d21a6f44673ed561ec8","timestamp":1494342245000000,"exception":{"type":"connection error"}}}
 {"error":{"id":"0f0e9d67c1854d21a6f44673ed561ec8","timestamp":1494342245999000,"log":{"level":"custom log level","message":"Cannot read property 'baz' of undefined"}}}