docs(kafka): refactor kafka documentation (#6854)

* Refactoring documentation

* Making Sonar Happy

Author: leandrodamascena

docs/utilities/kafka.md

@@ -0,0 +1,21 @@
+{
+   "eventSource":"aws:kafka",
+   "eventSourceArn":"arn:aws:kafka:eu-west-3:123456789012:cluster/powertools-kafka-esm/f138df86-9253-4d2a-b682-19e132396d4f-s3",
+   "bootstrapServers":"boot-z3majaui.c3.kafka-serverless.eu-west-3.amazonaws.com:9098",
+   "records":{
+      "python-with-avro-doc-3":[
+         {
+            "topic":"python-with-avro-doc",
+            "partition":3,
+            "offset":0,
+            "timestamp":1750547105187,
+            "timestampType":"CREATE_TIME",
+            "key":"MTIz",
+            "value":"AwBXT2qalUhN6oaj2CwEeaEWFFBvd2VydG9vbHMK",
+            "headers":[
+
+            ]
+         }
+      ]
+   }
+}

examples/kafka/consumer/events/kafka_event_avro.json

@@ -0,0 +1,21 @@
+{
+   "eventSource":"aws:kafka",
+   "eventSourceArn":"arn:aws:kafka:eu-west-3:123456789012:cluster/powertools-kafka-esm/f138df86-9253-4d2a-b682-19e132396d4f-s3",
+   "bootstrapServers":"boot-z3majaui.c3.kafka-serverless.eu-west-3.amazonaws.com:9098",
+   "records":{
+      "python-with-avro-doc-5":[
+         {
+            "topic":"python-with-avro-doc",
+            "partition":5,
+            "offset":0,
+            "timestamp":1750547462087,
+            "timestampType":"CREATE_TIME",
+            "key":"MTIz",
+            "value":"eyJuYW1lIjogIlBvd2VydG9vbHMiLCAiYWdlIjogNX0=",
+            "headers":[
+
+            ]
+         }
+      ]
+   }
+}

examples/kafka/consumer/events/kafka_event_json.json

@@ -0,0 +1,21 @@
+{
+   "eventSource":"aws:kafka",
+   "eventSourceArn":"arn:aws:kafka:eu-west-3:992382490249:cluster/powertools-kafka-esm/f138df86-9253-4d2a-b682-19e132396d4f-s3",
+   "bootstrapServers":"boot-z3majaui.c3.kafka-serverless.eu-west-3.amazonaws.com:9098",
+   "records":{
+      "python-with-avro-doc-5":[
+         {
+            "topic":"python-with-avro-doc",
+            "partition":5,
+            "offset":1,
+            "timestamp":1750624373324,
+            "timestampType":"CREATE_TIME",
+            "key":"MTIz",
+            "value":"Cgpwb3dlcnRvb2xzEAU=",
+            "headers":[
+
+            ]
+         }
+      ]
+   }
+}

examples/kafka/consumer/events/kafka_event_protobuf.json

@@ -0,0 +1,22 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Resources:
+  KafkaConsumerFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: app.lambda_handler
+      Runtime: python3.13
+      Timeout: 30
+      Events:
+        MSKEvent:
+          Type: MSK
+          Properties:
+            StartingPosition: LATEST
+            Stream: "arn:aws:lambda:eu-west-3:123456789012:event-source-mapping:11a2c814-dda3-4df8-b46f-4eeafac869ac"
+            Topics:
+              - my-topic-1
+            BatchSize: 100
+            MaximumBatchingWindowInSeconds: 5
+      Policies:
+        - AWSLambdaMSKExecutionRole
+

examples/kafka/consumer/sam/adjust_batch_size_configuration.yaml

@@ -0,0 +1,19 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Resources:
+  KafkaConsumerFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: app.lambda_handler
+      Runtime: python3.13
+      Timeout: 30
+      Events:
+        MSKEvent:
+          Type: MSK
+          Properties:
+            StartingPosition: LATEST
+            Stream: "arn:aws:lambda:eu-west-3:123456789012:event-source-mapping:11a2c814-dda3-4df8-b46f-4eeafac869ac"
+            Topics:
+              - my-topic-1
+      Policies:
+        - AWSLambdaMSKExecutionRole

examples/kafka/consumer/sam/getting_started_with_msk.yaml

@@ -0,0 +1,9 @@
+{
+    "type": "record",
+    "name": "User",
+    "namespace": "com.example",
+    "fields": [
+        {"name": "name", "type": "string"},
+        {"name": "age", "type": "int"}
+    ]
+}

examples/kafka/consumer/schemas/user.avsc

@@ -0,0 +1,4 @@
+{
+    "name": "...",
+    "age": "..."
+}

examples/kafka/consumer/schemas/user.json

@@ -0,0 +1,8 @@
+syntax = "proto3";
+
+package com.example;
+
+message User {
+  string name = 1;
+  int32 age = 2;
+}

examples/kafka/consumer/schemas/user.proto

@@ -0,0 +1,44 @@
+from aws_lambda_powertools import Logger
+from aws_lambda_powertools.utilities.kafka import ConsumerRecords, SchemaConfig, kafka_consumer
+from aws_lambda_powertools.utilities.typing import LambdaContext
+
+logger = Logger()
+
+# Define Avro schema
+avro_schema = """
+{
+    "type": "record",
+    "name": "User",
+    "namespace": "com.example",
+    "fields": [
+        {"name": "name", "type": "string"},
+        {"name": "age", "type": "int"}
+    ]
+}
+"""
+
+schema_config = SchemaConfig(
+    value_schema_type="AVRO",
+    value_schema=avro_schema,
+)
+
+
+@kafka_consumer(schema_config=schema_config)
+def lambda_handler(event: ConsumerRecords, context: LambdaContext):
+    for record in event.records:
+        # Log record coordinates for tracing
+        logger.info(f"Processing message from topic '{record.topic}'")
+        logger.info(f"Partition: {record.partition}, Offset: {record.offset}")
+        logger.info(f"Produced at: {record.timestamp}")
+
+        # Process message headers
+        logger.info(f"Headers: {record.headers}")
+
+        # Access the Avro deserialized message content
+        value = record.value
+        logger.info(f"Deserialized value: {value['name']}")
+
+        # For debugging, you can access the original raw data
+        logger.info(f"Raw message: {record.original_value}")
+
+    return {"statusCode": 200}