Azure Monitor Exporter: LogExporter (Azure#17117)

* saving wip

* tests

* wip

* log details

* mark classes internal while WIP

* tests

* disable Task.Delay

* cleanup

* cleanup

* cleanup

* fix datetime

* include condition; if not default

* todo

sdk/monitor/Microsoft.OpenTelemetry.Exporter.AzureMonitor/src/AzureMonitorConverter.cs

@@ -3,9 +3,12 @@
 
 using System.Collections.Generic;
 using System.Diagnostics;
-using OpenTelemetry;
+
 using Microsoft.OpenTelemetry.Exporter.AzureMonitor.Models;
 
+using OpenTelemetry;
+using OpenTelemetry.Logs;
+
 namespace Microsoft.OpenTelemetry.Exporter.AzureMonitor
 {
     /// <summary>
@@ -50,5 +53,24 @@ internal static List<TelemetryItem> Convert(Batch<Activity> batchActivity, strin
 
             return telemetryItems;
         }
+
+        internal static List<TelemetryItem> Convert(Batch<LogRecord> batchLogRecord, string instrumentationKey)
+        {
+            List<TelemetryItem> telemetryItems = new List<TelemetryItem>();
+            TelemetryItem telemetryItem;
+
+            foreach (var logRecord in batchLogRecord)
+            {
+                telemetryItem = TelemetryPartA.GetTelemetryItem(logRecord, instrumentationKey);
+                telemetryItem.Data = new MonitorBase
+                {
+                    BaseType = Telemetry_Base_Type_Mapping[TelemetryType.Message],
+                    BaseData = TelemetryPartB.GetMessageData(logRecord),
+                };
+                telemetryItems.Add(telemetryItem);
+            }
+
+            return telemetryItems;
+        }
     }
 }

sdk/monitor/Microsoft.OpenTelemetry.Exporter.AzureMonitor/src/AzureMonitorExporterLoggingExtensions.cs

@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+
+using Microsoft.OpenTelemetry.Exporter.AzureMonitor;
+
+using OpenTelemetry;
+using OpenTelemetry.Logs;
+
+namespace Microsoft.Extensions.Logging
+{
+    internal static class AzureMonitorExporterLoggingExtensions
+    {
+        public static OpenTelemetryLoggerOptions AddAzureMonitorLogExporter(this OpenTelemetryLoggerOptions loggerOptions, Action<AzureMonitorExporterOptions> configure = null)
+        {
+            if (loggerOptions == null)
+            {
+                throw new ArgumentNullException(nameof(loggerOptions));
+            }
+
+            var options = new AzureMonitorExporterOptions();
+            configure?.Invoke(options);
+
+            return loggerOptions.AddProcessor(new BatchExportProcessor<LogRecord>(new AzureMonitorLogExporter(options)));
+        }
+    }
+}

sdk/monitor/Microsoft.OpenTelemetry.Exporter.AzureMonitor/src/AzureMonitorLogExporter.cs

@@ -0,0 +1,54 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Threading;
+
+using Azure.Core.Pipeline;
+
+using OpenTelemetry;
+using OpenTelemetry.Logs;
+
+namespace Microsoft.OpenTelemetry.Exporter.AzureMonitor
+{
+    internal class AzureMonitorLogExporter : BaseExporter<LogRecord>
+    {
+        private readonly ITransmitter Transmitter;
+        private readonly AzureMonitorExporterOptions options;
+        private readonly string instrumentationKey;
+
+        public AzureMonitorLogExporter(AzureMonitorExporterOptions options) : this(options, new AzureMonitorTransmitter(options))
+        {
+        }
+
+        internal AzureMonitorLogExporter(AzureMonitorExporterOptions options, ITransmitter transmitter)
+        {
+            this.options = options ?? throw new ArgumentNullException(nameof(options));
+            ConnectionString.ConnectionStringParser.GetValues(this.options.ConnectionString, out this.instrumentationKey, out _);
+
+            this.Transmitter = transmitter;
+        }
+
+        /// <inheritdoc/>
+        public override ExportResult Export(in Batch<LogRecord> batch)
+        {
+            // Prevent Azure Monitor's HTTP operations from being instrumented.
+            using var scope = SuppressInstrumentationScope.Begin();
+
+            try
+            {
+                var telemetryItems = AzureMonitorConverter.Convert(batch, this.instrumentationKey);
+
+                // TODO: Handle return value, it can be converted as metrics.
+                // TODO: Validate CancellationToken and async pattern here.
+                this.Transmitter.TrackAsync(telemetryItems, false, CancellationToken.None).EnsureCompleted();
+                return ExportResult.Success;
+            }
+            catch (Exception ex)
+            {
+                AzureMonitorExporterEventSource.Log.Write($"FailedToExport{EventLevelSuffix.Error}", ex.LogAsyncException());
+                return ExportResult.Failure;
+            }
+        }
+    }
+}

sdk/monitor/Microsoft.OpenTelemetry.Exporter.AzureMonitor/src/TelemetryPartA.cs

@@ -4,9 +4,12 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
+
+using Microsoft.OpenTelemetry.Exporter.AzureMonitor.Models;
+
+using OpenTelemetry.Logs;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
-using Microsoft.OpenTelemetry.Exporter.AzureMonitor.Models;
 
 namespace Microsoft.OpenTelemetry.Exporter.AzureMonitor
 {
@@ -49,6 +52,36 @@ internal static TelemetryItem GetTelemetryItem(Activity activity, string instrum
             return telemetryItem;
         }
 
+        internal static TelemetryItem GetTelemetryItem(LogRecord logRecord, string instrumentationKey)
+        {
+            var name = PartA_Name_Mapping[TelemetryType.Message];
+            var time = logRecord.Timestamp.ToString(CultureInfo.InvariantCulture);
+
+            TelemetryItem telemetryItem = new TelemetryItem(name, time)
+            {
+                InstrumentationKey = instrumentationKey
+            };
+
+            // TODO: I WAS TOLD THIS MIGHT BE CHANGING. IGNORING FOR NOW.
+            //InitRoleInfo(activity);
+            //telemetryItem.Tags[ContextTagKeys.AiCloudRole.ToString()] = RoleName;
+            //telemetryItem.Tags[ContextTagKeys.AiCloudRoleInstance.ToString()] = RoleInstance;
+
+            if (logRecord.TraceId != default)
+            {
+                telemetryItem.Tags[ContextTagKeys.AiOperationId.ToString()] = logRecord.TraceId.ToHexString();
+            }
+
+            if (logRecord.SpanId != default)
+            {
+                telemetryItem.Tags[ContextTagKeys.AiOperationParentId.ToString()] = logRecord.SpanId.ToHexString();
+            }
+
+            telemetryItem.Tags[ContextTagKeys.AiInternalSdkVersion.ToString()] = SdkVersionUtils.SdkVersion;
+
+            return telemetryItem;
+        }
+
         internal static void InitRoleInfo(Activity activity)
         {
             if (RoleName != null || RoleInstance != null)

sdk/monitor/Microsoft.OpenTelemetry.Exporter.AzureMonitor/src/TelemetryPartB.cs

@@ -5,9 +5,13 @@
 using System.Diagnostics;
 using System.Globalization;
 using System.Runtime.CompilerServices;
-using OpenTelemetry.Trace;
+
+using Microsoft.Extensions.Logging;
 using Microsoft.OpenTelemetry.Exporter.AzureMonitor.Models;
 
+using OpenTelemetry.Logs;
+using OpenTelemetry.Trace;
+
 namespace Microsoft.OpenTelemetry.Exporter.AzureMonitor
 {
     /// <summary>
@@ -89,6 +93,14 @@ internal static RemoteDependencyData GetRemoteDependencyData(Activity activity)
             return dependency;
         }
 
+        internal static MessageData GetMessageData(LogRecord logRecord)
+        {
+            return new MessageData(version: 2, message: logRecord.State.ToString())
+            {
+                SeverityLevel = GetSeverityLevel(logRecord.LogLevel),
+            };
+        }
+
         private static TagEnumerationState EnumerateActivityTags(Activity activity)
         {
             var monitorTags = new TagEnumerationState
@@ -110,5 +122,27 @@ private static void AddPropertiesToTelemetry(IDictionary<string, string> destina
                 destination.Add(PartCTags[i].Key, PartCTags[i].Value?.ToString());
             }
         }
+
+        /// <summary>
+        /// Converts the <see cref="LogRecord.LogLevel"/> into corresponding Azure Monitor <see cref="SeverityLevel"/>.
+        /// </summary>
+        private static SeverityLevel GetSeverityLevel(LogLevel logLevel)
+        {
+            switch (logLevel)
+            {
+                case LogLevel.Critical:
+                    return SeverityLevel.Critical;
+                case LogLevel.Error:
+                    return SeverityLevel.Error;
+                case LogLevel.Warning:
+                    return SeverityLevel.Warning;
+                case LogLevel.Information:
+                    return SeverityLevel.Information;
+                case LogLevel.Debug:
+                case LogLevel.Trace:
+                default:
+                    return SeverityLevel.Verbose;
+            }
+        }
     }
 }

sdk/monitor/Microsoft.OpenTelemetry.Exporter.AzureMonitor/tests/Microsoft.OpenTelemetry.Exporter.AzureMonitor.Integration.Tests/TelemetryItemTests.cs

@@ -7,10 +7,13 @@
 using System.Linq;
 using System.Threading.Tasks;
 
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using Microsoft.OpenTelemetry.Exporter.AzureMonitor.Integration.Tests.TestFramework;
 using Microsoft.OpenTelemetry.Exporter.AzureMonitor.Models;
 
 using OpenTelemetry;
+using OpenTelemetry.Logs;
 using OpenTelemetry.Trace;
 
 using Xunit;
@@ -63,6 +66,28 @@ public void VerifyActivity(ActivityKind activityKind)
                 });
         }
 
+        [Theory]
+        [InlineData(LogLevel.Information, "Information")]
+        [InlineData(LogLevel.Warning, "Warning")]
+        [InlineData(LogLevel.Error, "Error")]
+        [InlineData(LogLevel.Critical, "Critical")]
+        [InlineData(LogLevel.Debug, "Verbose")]
+        [InlineData(LogLevel.Trace, "Verbose")]
+        public void VerifyILogger(LogLevel logLevel, string expectedSeverityLevel)
+        {
+            var message = "Hello World!";
+
+            var telemetryItem = this.RunLoggerTest(x => x.Log(logLevel: logLevel, message: message));
+
+            VerifyTelemetryItem.VerifyEvent(
+                telemetryItem: telemetryItem,
+                expectedVars: new ExpectedTelemetryItemValues
+                {
+                    Message = message,
+                    SeverityLevel = expectedSeverityLevel,
+                });
+        }
+
         private TelemetryItem RunActivityTest(Action<ActivitySource> testScenario)
         {
             // SETUP
@@ -88,10 +113,43 @@ private TelemetryItem RunActivityTest(Action<ActivitySource> testScenario)
 
             // CLEANUP
             processor.ForceFlush();
-            Task.Delay(100).Wait(); //TODO: HOW TO REMOVE THIS WAIT?
+            //Task.Delay(100).Wait(); //TODO: HOW TO REMOVE THIS WAIT?
+
+            Assert.True(transmitter.TelemetryItems.Any(), "test project did not capture telemetry");
+            return transmitter.TelemetryItems.Single();
+        }
+
+        private TelemetryItem RunLoggerTest(Action<ILogger<TelemetryItemTests>> testScenario)
+        {
+            // SETUP
+            var transmitter = new MockTransmitter();
+            var processor = new BatchExportProcessor<LogRecord>(new AzureMonitorLogExporter(
+                options: new AzureMonitorExporterOptions
+                {
+                    ConnectionString = EmptyConnectionString,
+                },
+                transmitter: transmitter));
+
+            var serviceCollection = new ServiceCollection().AddLogging(builder =>
+            {
+                builder.SetMinimumLevel(LogLevel.Trace)
+                    .AddOpenTelemetry(options => options
+                    .AddProcessor(processor));
+            });
+
+            // ACT
+            using var serviceProvider = serviceCollection.BuildServiceProvider();
+            var logger = serviceProvider.GetRequiredService<ILogger<TelemetryItemTests>>();
+            testScenario(logger);
+
+            // CLEANUP
+            processor.ForceFlush();
+            //Task.Delay(100).Wait(); //TODO: HOW TO REMOVE THIS WAIT?
 
             Assert.True(transmitter.TelemetryItems.Any(), "test project did not capture telemetry");
             return transmitter.TelemetryItems.Single();
         }
+
+        // TODO: INCLUDE ADDITIONAL TESTS VALIDATING ILOGGER + ACTIVITY
     }
 }

sdk/monitor/Microsoft.OpenTelemetry.Exporter.AzureMonitor/tests/Microsoft.OpenTelemetry.Exporter.AzureMonitor.Integration.Tests/TestFramework/ExpectedTelemetryItemValues.cs

@@ -3,11 +3,15 @@
 
 using System.Collections.Generic;
 
+using Microsoft.OpenTelemetry.Exporter.AzureMonitor.Models;
+
 namespace Microsoft.OpenTelemetry.Exporter.AzureMonitor.Integration.Tests.TestFramework
 {
-    public struct ExpectedTelemetryItemValues
+    internal struct ExpectedTelemetryItemValues
     {
         public string Name;
+        public string Message;
         public Dictionary<string, string> CustomProperties;
+        public SeverityLevel SeverityLevel;
     }
 }

sdk/monitor/Microsoft.OpenTelemetry.Exporter.AzureMonitor/tests/Microsoft.OpenTelemetry.Exporter.AzureMonitor.Integration.Tests/TestFramework/VerifyTelemetryItem.cs

@@ -50,5 +50,15 @@ public static void VerifyDependency(TelemetryItem telemetryItem, ExpectedTelemet
             Assert.Equal(expectedVars.Name, data.Name);
             Assert.Equal(expectedVars.CustomProperties, data.Properties);
         }
+
+        public static void VerifyEvent(TelemetryItem telemetryItem, ExpectedTelemetryItemValues expectedVars)
+        {
+            Assert.Equal("Message", telemetryItem.Name);
+            Assert.Equal(nameof(MessageData), telemetryItem.Data.BaseType);
+
+            var data = (MessageData)telemetryItem.Data.BaseData;
+            Assert.Equal(expectedVars.Message, data.Message);
+            Assert.Equal(expectedVars.SeverityLevel, data.SeverityLevel);
+        }
     }
 }