From e0f4fd509235fbcc5e2c2bb60f127693d34288fb Mon Sep 17 00:00:00 2001 From: Jose Riguera Date: Wed, 29 May 2024 23:28:07 +0200 Subject: [PATCH] [receiver/cloudfoundryreceiver] Improve tests for log conversion --- .../cloudfoundryreceiver/converter_test.go | 190 ++++++++++++++---- 1 file changed, 153 insertions(+), 37 deletions(-) diff --git a/receiver/cloudfoundryreceiver/converter_test.go b/receiver/cloudfoundryreceiver/converter_test.go index c40d513613b0..2357085c1dfd 100644 --- a/receiver/cloudfoundryreceiver/converter_test.go +++ b/receiver/cloudfoundryreceiver/converter_test.go @@ -210,7 +210,7 @@ func TestSetTraceIDs(t *testing.T) { expected map[string]any }{ { - id: "w3c-zipkin", + id: "w3c-and-zipkin", logLine: `www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com" x_b3_traceid:"766afb1917794bb965d4f01306f9f912" x_b3_spanid:"65d4f01306f9f912" x_b3_parentspanid:"-" b3:"766afb1917794bb965d4f01306f9f912-65d4f01306f9f912" traceparent:"00-766afb1917794bb965d4f01306f9f912-65d4f01306f9f912-01" tracestate:"gorouter=65d4f01306f9f912"`, expected: map[string]any{ "err": nil, @@ -237,7 +237,7 @@ func TestSetTraceIDs(t *testing.T) { }, }, { - id: "not-found", + id: "no-tracing", logLine: `www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com"`, expected: map[string]any{ "err": nil, @@ -246,7 +246,7 @@ func TestSetTraceIDs(t *testing.T) { }, }, { - id: "error-w3c", + id: "w3c-error", logLine: `www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com" traceparent:"00-766afb1917794bb965d4f01306f9f912-65d4f01306f9f9122-01" tracestate:"gorouter=65d4f01306f9f912"`, expected: map[string]any{ "err": fmt.Errorf("encoding/hex: odd length hex string"), "traceID": "00000000000000000000000000000000", @@ -254,7 +254,7 @@ func TestSetTraceIDs(t *testing.T) { }, }, { - id: "error-w3c-format", + id: "w3c-format-error", logLine: `www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com" traceparent:"00-766afb1917794bb965d4f01306f9f912-65d4f01306f9f912" tracestate:"gorouter=65d4f01306f9f912"`, expected: map[string]any{ "err": fmt.Errorf("traceId W3C key traceparent with format 00 not valid in log"), "traceID": "00000000000000000000000000000000", @@ -262,7 +262,7 @@ func TestSetTraceIDs(t *testing.T) { }, }, { - id: "error-zipkin-format", + id: "zipkin-format-error", logLine: `www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com" x_b3_traceid:"766afb1917794bb965d4f01306f9f912" x_b3_spanid:"65d4f01306f9f912" x_b3_parentspanid:"-" b3:"766afb1917794bb965d4f01306f9f912-65d4f01306f9f912-01"`, expected: map[string]any{ "err": fmt.Errorf("traceId Zipkin key b3 not valid in log"), @@ -286,42 +286,158 @@ func TestSetTraceIDs(t *testing.T) { func TestConvertLogsEnvelope(t *testing.T) { now := time.Now() before := time.Now().Add(-time.Second) - - envelope := loggregator_v2.Envelope{ - Timestamp: before.UnixNano(), - SourceId: "uaa", - Tags: map[string]string{ - "origin": "gorouter", - "deployment": "cf", - "job": "router", - "index": "bc276108-8282-48a5-bae7-c009c4392246", - "ip": "10.244.0.34", + t.Parallel() + tests := []struct { + id string + envelope loggregator_v2.Envelope + expected map[string]any + }{ + { + id: "normal-without-sourcetype-tag", + envelope: loggregator_v2.Envelope{ + Timestamp: before.UnixNano(), + SourceId: "744e75bb-69d1-4cf4-b037-76875368097b", + Tags: map[string]string{}, + Message: &loggregator_v2.Envelope_Log{ + Log: &loggregator_v2.Log{ + Payload: []byte(`test-app. Says Hello. on index: 0`), + Type: loggregator_v2.Log_OUT, + }, + }, + }, + expected: map[string]any{ + "Timestamp": before, + "Attributes": map[string]string{ + "org.cloudfoundry.source_id": "744e75bb-69d1-4cf4-b037-76875368097b", + }, + "Body": `test-app. Says Hello. on index: 0`, + "SeverityNumber": plog.SeverityNumberInfo, + "SeverityText": plog.SeverityNumberInfo.String(), + "TraceID": "", + "SpanID": "", + }, }, - Message: &loggregator_v2.Envelope_Log{ - Log: &loggregator_v2.Log{ - Payload: []byte("log message payload"), - Type: loggregator_v2.Log_OUT, + { + id: "json-log-with-sourcetype", + envelope: loggregator_v2.Envelope{ + Timestamp: before.UnixNano(), + SourceId: "df75aec8-b937-4dc8-9b4d-c336e36e3895", + Tags: map[string]string{ + "source_type": "APP/PROC/WEB", + "origin": "rep", + "deployment": "cf", + "job": "diego-cell", + "index": "bc276108-8282-48a5-bae7-c009c4392246", + "ip": "10.80.0.2", + }, + Message: &loggregator_v2.Envelope_Log{ + Log: &loggregator_v2.Log{ + Payload: []byte(`{"timestamp":"2024-05-29T16:16:28.063062903Z","level":"info","source":"guardian","message":"guardian.api.garden-server.get-properties.got-properties","data":{"handle":"e885e8be-c6a7-43b1-5066-a821","session":"2.1.209666"}}`), + Type: loggregator_v2.Log_OUT, + }, + }, + }, + expected: map[string]any{ + "Timestamp": before, + "Attributes": map[string]string{ + "org.cloudfoundry.source_id": "df75aec8-b937-4dc8-9b4d-c336e36e3895", + "org.cloudfoundry.source_type": "APP/PROC/WEB", + "org.cloudfoundry.origin": "rep", + "org.cloudfoundry.deployment": "cf", + "org.cloudfoundry.job": "diego-cell", + "org.cloudfoundry.index": "bc276108-8282-48a5-bae7-c009c4392246", + "org.cloudfoundry.ip": "10.80.0.2", + }, + "Body": `{"timestamp":"2024-05-29T16:16:28.063062903Z","level":"info","source":"guardian","message":"guardian.api.garden-server.get-properties.got-properties","data":{"handle":"e885e8be-c6a7-43b1-5066-a821","session":"2.1.209666"}}`, + "SeverityNumber": plog.SeverityNumberInfo, + "SeverityText": plog.SeverityNumberInfo.String(), + "TraceID": "", + "SpanID": "", + }, + }, + { + id: "rtr-log-with-tracing", + envelope: loggregator_v2.Envelope{ + Timestamp: before.UnixNano(), + SourceId: "df75aec8-b937-4dc8-9b4d-c336e36e3899", + Tags: map[string]string{ + "source_type": "RTR", + "origin": "gorouter", + }, + Message: &loggregator_v2.Envelope_Log{ + Log: &loggregator_v2.Log{ + Payload: []byte(`www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com" x_b3_traceid:"766afb1917794bb965d4f01306f9f912" x_b3_spanid:"65d4f01306f9f912" x_b3_parentspanid:"-" b3:"766afb1917794bb965d4f01306f9f912-65d4f01306f9f912" traceparent:"00-766afb1917794bb965d4f01306f9f912-65d4f01306f9f912-01" tracestate:"gorouter=65d4f01306f9f912"`), + Type: loggregator_v2.Log_OUT, + }, + }, + }, + expected: map[string]any{ + "Timestamp": before, + "Attributes": map[string]string{ + "org.cloudfoundry.source_id": "df75aec8-b937-4dc8-9b4d-c336e36e3899", + "org.cloudfoundry.source_type": "RTR", + "org.cloudfoundry.origin": "gorouter", + }, + "Body": `www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com" x_b3_traceid:"766afb1917794bb965d4f01306f9f912" x_b3_spanid:"65d4f01306f9f912" x_b3_parentspanid:"-" b3:"766afb1917794bb965d4f01306f9f912-65d4f01306f9f912" traceparent:"00-766afb1917794bb965d4f01306f9f912-65d4f01306f9f912-01" tracestate:"gorouter=65d4f01306f9f912"`, + "SeverityNumber": plog.SeverityNumberInfo, + "SeverityText": plog.SeverityNumberInfo.String(), + "TraceID": "766afb1917794bb965d4f01306f9f912", + "SpanID": "65d4f01306f9f912", + }, + }, + { + id: "rtr-log-no-tracing", + envelope: loggregator_v2.Envelope{ + Timestamp: before.UnixNano(), + SourceId: "df75aec8-b937-4dc8-9b4d-c336e36e3845", + Tags: map[string]string{ + "source_type": "RTR", + }, + Message: &loggregator_v2.Envelope_Log{ + Log: &loggregator_v2.Log{ + Payload: []byte(`www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com"`), + Type: loggregator_v2.Log_OUT, + }, + }, + }, + expected: map[string]any{ + "Timestamp": before, + "Attributes": map[string]string{ + "org.cloudfoundry.source_id": "df75aec8-b937-4dc8-9b4d-c336e36e3845", + "org.cloudfoundry.source_type": "RTR", + }, + "Body": `www.example.com - [2024-05-21T15:40:13.892179798Z] "GET /articles/ssdfws HTTP/1.1" 200 0 110563 "-" "python-requests/2.26.0" "20.191.2.244:52238" "10.88.195.81:61222" x_forwarded_for:"18.21.57.150, 10.28.45.29, 35.16.25.46, 20.191.2.244" x_forwarded_proto:"https" vcap_request_id:"766afb19-1779-4bb9-65d4-f01306f9f912" response_time:0.191835 gorouter_time:0.000139 app_id:"e3267823-0938-43ce-85ff-003e3e3a5a23" app_index:"4" instance_id:"918dd283-a0ed-48be-7f0c-253b" x_cf_routererror:"-" x_forwarded_host:"www.example.com"`, + "SeverityNumber": plog.SeverityNumberInfo, + "SeverityText": plog.SeverityNumberInfo.String(), + "TraceID": "", + "SpanID": "", }, }, } - - logSlice := plog.NewLogRecordSlice() - e := convertEnvelopeToLogs(&envelope, logSlice, now) - require.Equal(t, nil, e) - require.Equal(t, 1, logSlice.Len()) - log := logSlice.At(0) - assert.Equal(t, "log message payload", log.Body().AsString()) - assert.Equal(t, plog.SeverityNumberInfo.String(), log.SeverityText()) - assert.Equal(t, pcommon.NewTimestampFromTime(before), log.Timestamp()) - assert.Equal(t, pcommon.NewTimestampFromTime(now), log.ObservedTimestamp()) - assertAttributes(t, log.Attributes(), map[string]string{ - "org.cloudfoundry.source_id": "uaa", - "org.cloudfoundry.origin": "gorouter", - "org.cloudfoundry.deployment": "cf", - "org.cloudfoundry.job": "router", - "org.cloudfoundry.index": "bc276108-8282-48a5-bae7-c009c4392246", - "org.cloudfoundry.ip": "10.244.0.34", - }) + for _, tt := range tests { + t.Run(tt.id, func(t *testing.T) { + logSlice := plog.NewLogRecordSlice() + e := convertEnvelopeToLogs(&tt.envelope, logSlice, now) + require.Equal(t, nil, e) + require.Equal(t, 1, logSlice.Len()) + log := logSlice.At(0) + assert.Equal(t, tt.expected["Body"], log.Body().AsString()) + assert.Equal(t, tt.expected["SeverityText"], log.SeverityText()) + assert.Equal(t, pcommon.NewTimestampFromTime(tt.expected["Timestamp"].(time.Time)), log.Timestamp()) + assert.Equal(t, pcommon.NewTimestampFromTime(now), log.ObservedTimestamp()) + assertAttributes(t, tt.expected["Attributes"].(map[string]string), log.Attributes()) + if tt.expected["TraceID"] == "" { + assert.True(t, log.TraceID().IsEmpty()) + } else { + assert.Equal(t, tt.expected["TraceID"], log.TraceID().String()) + } + if tt.expected["SpanID"] == "" { + assert.True(t, log.SpanID().IsEmpty()) + } else { + assert.Equal(t, tt.expected["SpanID"], log.SpanID().String()) + } + }) + } } func assertAttributes(t *testing.T, expected map[string]string, attributes pcommon.Map) {