Working example

otel-dotnet-lambda-extension.sln

@@ -7,6 +7,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sample", "sample", "{67C330
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "sample", "sample\src\sample.csproj", "{EFCDD6DB-1DD2-4B5A-A888-58FB1E1DBD88}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{9AAD1C69-7A79-4C26-83FB-567B753CD9BA}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PracticalOtel.LambdaFlusher", "src\PracticalOtel.LambdaFlusher\PracticalOtel.LambdaFlusher.csproj", "{7E6BC80E-6EA4-4394-BAC8-FAA12FE541FF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -20,8 +24,13 @@ Global
 		{EFCDD6DB-1DD2-4B5A-A888-58FB1E1DBD88}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EFCDD6DB-1DD2-4B5A-A888-58FB1E1DBD88}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{EFCDD6DB-1DD2-4B5A-A888-58FB1E1DBD88}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7E6BC80E-6EA4-4394-BAC8-FAA12FE541FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7E6BC80E-6EA4-4394-BAC8-FAA12FE541FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7E6BC80E-6EA4-4394-BAC8-FAA12FE541FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7E6BC80E-6EA4-4394-BAC8-FAA12FE541FF}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{EFCDD6DB-1DD2-4B5A-A888-58FB1E1DBD88} = {67C330B0-6434-42B3-8F2B-42235D3030AE}
+		{7E6BC80E-6EA4-4394-BAC8-FAA12FE541FF} = {9AAD1C69-7A79-4C26-83FB-567B753CD9BA}
 	EndGlobalSection
 EndGlobal

sample/src/Program.cs

@@ -1,7 +1,4 @@
-using System.Diagnostics;
-using System.Diagnostics.Tracing;
 using OpenTelemetry;
-using OpenTelemetry.Exporter;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
 
@@ -13,59 +10,29 @@
 // Add AWS Lambda support. When application is run in Lambda Kestrel is swapped out as the web server with Amazon.Lambda.AspNetCoreServer. This
 // package will act as the webserver translating request and responses between the Lambda event source and ASP.NET Core.
 builder.Services.AddAWSLambdaHosting(LambdaEventSource.HttpApi);
-builder.Services.AddSingleton<ConsoleOpenTelemetryListener>();
 
 builder.Services.AddOpenTelemetry()
     .ConfigureResource(resource => resource
         .AddService(builder.Environment.ApplicationName)
     )
-    .WithTracing(tracingOptions => 
+    .WithTracing(tracingOptions =>
         tracingOptions.AddAspNetCoreInstrumentation()
             .AddHttpClientInstrumentation()
     )
+    .AddLambdaExtension()
     .UseOtlpExporter();
 
 
 var app = builder.Build();
 
-var tracerProvider = app.Services.GetRequiredService<TracerProvider>();
-ActivityListener listener = new()
-{
-    ShouldListenTo = _ => true,
-    Sample = (ref ActivityCreationOptions<ActivityContext> options) => ActivitySamplingResult.AllData,
-    ActivityStopped = activity =>
-    {
-        Console.WriteLine("Activity stopped: " + activity.Source.Name + " " + activity.DisplayName + " " + activity.Duration);
-        tracerProvider.ForceFlush();
-    }
-};
-
-ActivitySource.AddActivityListener(listener);
-
-var openTelemetryDebugLogger = app.Services.GetRequiredService<ConsoleOpenTelemetryListener>();
-
 app.UseHttpsRedirection();
 app.UseAuthorization();
 app.MapControllers();
 
-app.MapGet("/", () => {
+app.MapGet("/", () =>
+{
     Thread.Sleep(1000);
     return "Welcome to running ASP.NET Core Minimal API on AWS Lambda";
 });
 
 app.Run();
-
-
-public class ConsoleOpenTelemetryListener : EventListener
-{
-    protected override void OnEventSourceCreated(EventSource eventSource)
-    {
-        if (eventSource.Name.StartsWith("OpenTelemetry"))
-            EnableEvents(eventSource, EventLevel.Error);
-    }
-
-    protected override void OnEventWritten(EventWrittenEventArgs eventData)
-    {
-        Console.WriteLine(string.Format(eventData.Message, eventData.Payload?.Select(p => p?.ToString())?.ToArray()));
-    }
-}

sample/src/sample.csproj

@@ -17,4 +17,7 @@
     <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
     <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\PracticalOtel.LambdaFlusher\PracticalOtel.LambdaFlusher.csproj" />
+  </ItemGroup>
 </Project>

src/PracticalOtel.LambdaFlusher/ExtensionClient.cs

@@ -0,0 +1,164 @@
+using System.Text;
+using Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Lambda Extension API client
+/// </summary>
+internal class ExtensionClient : IDisposable
+{
+    #region HTTP header key names
+
+    /// <summary>
+    /// HTTP header that is used to register a new extension name with Extension API
+    /// </summary>
+    private const string LambdaExtensionNameHeader = "Lambda-Extension-Name";
+
+    /// <summary>
+    /// HTTP header used to provide extension registration id
+    /// </summary>
+    /// <remarks>
+    /// Registration endpoint reply will have this header value with a new id, assigned to this extension by the API.
+    /// All other endpoints will expect HTTP calls to have id header attached to all requests.
+    /// </remarks>
+    private const string LambdaExtensionIdHeader = "Lambda-Extension-Identifier";
+
+    /// <summary>
+    /// HTTP header to report Lambda Extension error type string.
+    /// </summary>
+    /// <remarks>
+    /// This header is used to report additional error details for Init and Shutdown errors.
+    /// </remarks>
+    private const string LambdaExtensionFunctionErrorTypeHeader = "Lambda-Extension-Function-Error-Type";
+
+    #endregion
+
+    #region Environment variable names
+
+    /// <summary>
+    /// Environment variable that holds server name and port number for Extension API endpoints
+    /// </summary>
+    private const string LambdaRuntimeApiAddress = "AWS_LAMBDA_RUNTIME_API";
+
+    #endregion
+
+    #region Instance properties
+
+    /// <summary>
+    /// Extension id, which is assigned to this extension after the registration
+    /// </summary>
+    public string? Id { get; private set; }
+
+    #endregion
+
+    #region Constructor and readonly variables
+
+    /// <summary>
+    /// Http client instance
+    /// </summary>
+    /// <remarks>This is an IDisposable object that must be properly disposed of,
+    /// thus <see cref="ExtensionClient"/> implements <see cref="IDisposable"/> interface too.</remarks>
+    private readonly HttpClient httpClient = new HttpClient();
+
+    /// <summary>
+    /// Extension name, calculated from the current executing assembly name
+    /// </summary>
+    private readonly string _extensionName;
+    private readonly ILogger _logger;
+
+    /// <summary>
+    /// Extension registration URL
+    /// </summary>
+    private readonly Uri registerUrl;
+
+    /// <summary>
+    /// Next event long poll URL
+    /// </summary>
+    private readonly Uri nextUrl;
+
+    /// <summary>
+    /// Constructor
+    /// </summary>
+    public ExtensionClient(string extensionName, ILogger logger)
+    {
+        _extensionName = extensionName ?? throw new ArgumentNullException(nameof(extensionName), "Extension name cannot be null");
+        _logger = logger;
+        this.httpClient.Timeout = Timeout.InfiniteTimeSpan;
+        var apiUri = new UriBuilder(Environment.GetEnvironmentVariable(LambdaRuntimeApiAddress)!).Uri;
+        var basePath = "2020-01-01/extension";
+
+        // Calculate all Extension API endpoints' URLs
+        this.registerUrl = new Uri(apiUri, $"{basePath}/register");
+        this.nextUrl = new Uri(apiUri, $"{basePath}/event/next");
+    }
+
+    #endregion
+
+    #region Private methods
+
+    /// <summary>
+    /// Register extension with Extension API
+    /// </summary>
+    /// <param name="events">Event types to by notified with</param>
+    /// <returns>Awaitable void</returns>
+    /// <remarks>This method is expected to be called just once when extension is being registered with the Extension API.</remarks>
+    public async Task RegisterExtensionAsync()
+    {
+        using var scope = OpenTelemetry.SuppressInstrumentationScope.Begin();
+
+        const string payload = @"{ ""events"": [""INVOKE""] }";
+
+        using var content = new StringContent(payload, Encoding.UTF8, "application/json");
+        content.Headers.Add(LambdaExtensionNameHeader, _extensionName);
+
+        using var response = await this.httpClient.PostAsync(this.registerUrl, content);
+
+        // if POST call didn't succeed
+        if (!response.IsSuccessStatusCode)
+        {
+            _logger.LogError("Error response received for registration request: {response}", await response.Content.ReadAsStringAsync());
+            response.EnsureSuccessStatusCode();
+        }
+
+        this.Id = response.Headers.GetValues(LambdaExtensionIdHeader).FirstOrDefault();
+        if (string.IsNullOrEmpty(this.Id))
+        {
+            throw new ApplicationException("Extension API register call didn't return a valid identifier.");
+        }
+
+        this.httpClient.DefaultRequestHeaders.Add(LambdaExtensionIdHeader, this.Id);
+    }
+
+    /// <summary>
+    /// Long poll for the next event from Extension API
+    /// </summary>
+    /// <returns>Awaitable tuple having event type and event details fields</returns>
+    /// <remarks>It is important to have httpClient.Timeout set to some value, that is longer than any expected wait time,
+    /// otherwise HttpClient will throw an exception when getting the next event details from the server.</remarks>
+    public async Task GetNextAsync()
+    {
+        using var scope = OpenTelemetry.SuppressInstrumentationScope.Begin();
+        var response = await this.httpClient.GetAsync(this.nextUrl);
+
+        if (!response.IsSuccessStatusCode)
+        {
+            _logger.LogError("Error response received for {url}: {response}", this.nextUrl.PathAndQuery, await response.Content.ReadAsStringAsync());
+            response.EnsureSuccessStatusCode();
+        }
+        Console.WriteLine("Received event: " + await response.Content.ReadAsStringAsync());
+    }
+
+    #endregion
+
+    #region IDisposable implementation
+
+    /// <summary>
+    /// Dispose of instance Disposable variables
+    /// </summary>
+    public void Dispose()
+    {
+        // Quick and dirty implementation to propagate Dispose call to HttpClient instance
+        ((IDisposable)httpClient).Dispose();
+    }
+
+    #endregion
+}

src/PracticalOtel.LambdaFlusher/LambdaExtensionSetup.cs

@@ -0,0 +1,16 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using PracticalOtel.LambdaFlusher;
+
+namespace OpenTelemetry;
+
+public static class LambdaExtensionSetup
+{
+    public static OpenTelemetryBuilder AddLambdaExtension(this OpenTelemetryBuilder builder)
+    {
+        builder.Services.AddHostedService<OtelLambdaExtensionService>();
+        builder.Services.AddSingleton(sp => 
+            new ExtensionClient("OtelLambdaExtensionService", sp.GetRequiredService<ILogger<ExtensionClient>>()));
+        return builder;
+    }
+}

src/PracticalOtel.LambdaFlusher/OtelLambdaExtensionService.cs

@@ -0,0 +1,50 @@
+using System.Diagnostics;
+using System.Threading.Channels;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using OpenTelemetry.Trace;
+
+namespace PracticalOtel.LambdaFlusher;
+
+internal class OtelLambdaExtensionService : BackgroundService
+{
+    private readonly ExtensionClient _extensionClient;
+    private readonly TracerProvider _tracerProvider;
+    private readonly ILogger<OtelLambdaExtensionService> _logger;
+
+    private static readonly Channel<Activity> _channel = Channel.CreateUnbounded<Activity>();
+
+    private readonly ActivityListener _listener = new()
+    {
+        ShouldListenTo = source => source.Name == "Microsoft.AspNetCore",
+        //ShouldListenTo = _ => true,
+        Sample = (ref ActivityCreationOptions<ActivityContext> options) => ActivitySamplingResult.AllData,
+        ActivityStopped = activity =>
+        {
+            Console.WriteLine("Activity stopped: " + activity.Source.Name + " " + activity.DisplayName + " " + activity.Duration);
+           _channel.Writer.WriteAsync(activity);
+        }
+    };
+
+    public OtelLambdaExtensionService(ExtensionClient extensionClient, TracerProvider tracerProvider, ILogger<OtelLambdaExtensionService> logger)
+    {
+        _extensionClient = extensionClient;
+        _tracerProvider = tracerProvider;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        ActivitySource.AddActivityListener(_listener);
+        await _extensionClient.RegisterExtensionAsync();
+
+
+        while(true)
+        {
+            await _extensionClient.GetNextAsync();
+            await _channel.Reader.WaitToReadAsync();
+            _tracerProvider.ForceFlush();
+        }
+    }
+}
+

src/PracticalOtel.LambdaFlusher/PracticalOtel.LambdaFlusher.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="OpenTelemetry" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
+  </ItemGroup>
+
+</Project>