parsing requestId from message and stamping it as attribute

src/cloudwatch/cloudwatch.go

@@ -3,6 +3,7 @@ package cloudwatch
 
 import (
 	"os"
+	"regexp"
 	"strconv"
 
 	"github.com/aws/aws-lambda-go/events"
@@ -52,14 +53,25 @@ func batchLogEntries(cloudwatchLogsData events.CloudwatchLogsData, channel chan
 
 	var currentBatch common.LogData
 
+	// Regular expression to match the pattern "RequestId: <UUID> <message>"
+	regularExpression := regexp.MustCompile(common.RequestIDRegex)
+
+	// Variable to keep track of the last requestId found
+	var lastRequestID string
+
 	for _, record := range cloudwatchLogsData.LogEvents {
 		messages := util.SplitLargeMessages(record.Message)
-
 		for _, message := range messages {
+
+			// logAttribute is a map of attributes for each individual log message.
+			logAttribute := common.LogAttributes{}
+
 			entry := common.Log{
-				Timestamp: strconv.FormatInt(record.Timestamp, 10),
-				Log:       message,
+				Timestamp:  strconv.FormatInt(record.Timestamp, 10),
+				Log:        message,
+				Attributes: logAttribute,
 			}
+			lastRequestID = util.AddRequestID(cloudwatchLogsData.LogGroup, message, logAttribute, lastRequestID, regularExpression)
 
 			if batchSize+len(message) > common.MaxPayloadSize || messageCount >= common.MaxPayloadMessages {
 				util.ProduceMessageToChannel(channel, currentBatch, attributes)

src/common/constants.go

@@ -46,3 +46,9 @@ const CloudTrailDigestRegex = ".*_CloudTrail-Digest_.*\\.json\\.gz$"
 
 // CloudTrailRegex is the regex pattern for CloudTrail files.
 const CloudTrailRegex = ".*_CloudTrail_.*\\.json\\.gz$"
+
+// RequestIDRegex is the regex pattern for RequestId.
+const RequestIDRegex = "RequestId:\\s([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})"
+
+// LambdaLogGroup is prefix for identifing log group belonging to lambda
+const LambdaLogGroup = "/aws/lambda"

src/util/message_util.go

@@ -3,8 +3,11 @@ package util
 
 import (
 	"encoding/json"
-	"github.com/newrelic/aws-unified-lambda-logging/common"
+	"regexp"
+	"strings"
 	"time"
+
+	"github.com/newrelic/aws-unified-lambda-logging/common"
 )
 
 // SplitLargeMessages splits a large message into smaller messages if its length exceeds the maximum message size.
@@ -93,3 +96,19 @@ func ParseCloudTrailEvents(message string) ([]string, error) {
 	}
 	return records, nil
 }
+
+// AddRequestID extracts the requestId from the message and updates the attributes map.
+// It returns the last requestId found to keep track of the requestId across log messages.
+func AddRequestID(logGroup, message string, logAttribute common.LogAttributes, lastRequestID string, regularExpression *regexp.Regexp) string {
+
+	if strings.HasPrefix(logGroup, common.LambdaLogGroup) {
+		matches := regularExpression.FindStringSubmatch(message)
+		if len(matches) == 2 {
+			lastRequestID = matches[1]
+			logAttribute["requestId"] = lastRequestID
+		} else if lastRequestID != "" {
+			logAttribute["requestId"] = lastRequestID
+		}
+	}
+	return lastRequestID
+}

src/util/message_util_test.go

@@ -2,10 +2,12 @@ package util
 
 import (
 	"encoding/json"
-	"github.com/newrelic/aws-unified-lambda-logging/common"
-	"github.com/stretchr/testify/assert"
 	"reflect"
+	"regexp"
 	"testing"
+
+	"github.com/newrelic/aws-unified-lambda-logging/common"
+	"github.com/stretchr/testify/assert"
 )
 
 // TestAddCustomMetaData tests adding custom meta data utility function.
@@ -209,3 +211,68 @@ func TestParseCTEvents(t *testing.T) {
 		})
 	}
 }
+
+// TestAddRequestId tests the AddRequestId function.
+func TestAddRequestId(t *testing.T) {
+	tests := []struct {
+		name           string
+		logGroup       string
+		message        string
+		logAttribute   common.LogAttributes
+		lastRequestID  string
+		expectedReqID  string
+		expectedAttrib common.LogAttributes
+	}{
+		{
+			name:          "Valid RequestId",
+			logGroup:      "/aws/lambda/test",
+			message:       "RequestId: d653fb2c-0234-46ff-ae6b-9a418b888420 hello world",
+			logAttribute:  common.LogAttributes{},
+			lastRequestID: "",
+			expectedReqID: "d653fb2c-0234-46ff-ae6b-9a418b888420",
+			expectedAttrib: common.LogAttributes{
+				"requestId": "d653fb2c-0234-46ff-ae6b-9a418b888420",
+			},
+		},
+		{
+			name:          "No RequestId in message",
+			logGroup:      "/aws/lambda/test",
+			message:       "hello world",
+			logAttribute:  common.LogAttributes{},
+			lastRequestID: "d653fb2c-0234-46ff-ae6b-9a418b888420",
+			expectedReqID: "d653fb2c-0234-46ff-ae6b-9a418b888420",
+			expectedAttrib: common.LogAttributes{
+				"requestId": "d653fb2c-0234-46ff-ae6b-9a418b888420",
+			},
+		},
+		{
+			name:           "No RequestId and no previous RequestId",
+			logGroup:       "/aws/lambda/test",
+			message:        "hello world",
+			logAttribute:   common.LogAttributes{},
+			lastRequestID:  "",
+			expectedReqID:  "",
+			expectedAttrib: common.LogAttributes{},
+		},
+		{
+			name:           "Non-matching log group",
+			logGroup:       "/aws/other/test",
+			message:        "RequestId: d653fb2c-0234-46ff-ae6b-9a418b888420 hello world",
+			logAttribute:   common.LogAttributes{},
+			lastRequestID:  "",
+			expectedReqID:  "",
+			expectedAttrib: common.LogAttributes{},
+		},
+	}
+
+	// Regular expression to match the pattern "RequestId: <UUID>"
+	re := regexp.MustCompile(`RequestId:\s([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})`)
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotReqID := AddRequestID(tt.logGroup, tt.message, tt.logAttribute, tt.lastRequestID, re)
+			assert.Equal(t, tt.expectedReqID, gotReqID)
+			assert.Equal(t, tt.expectedAttrib, tt.logAttribute)
+		})
+	}
+}