Make DirectLogSubmissionSettings properly immutable and remove `Imm…

…utableDirectLogSubmissionSettings` (#6400)

## Summary of changes

Merge `DirectLogSubmissionSettings` with
`ImmutableDirectLogSubmissionSettings`

## Reason for change

There was never really a good reason for having these as separate types.
It was primarily to make testing a little easier and to mirror the
`TracerSettings`/`ImmutableTracerSettings` dichotomy, but as that's
going away, this is just unnecessary complexity

## Implementation details

- Moved the additional logic that was previously inside
`ImmutableDirectLogSubmissionSettings` into
`DirectLogSubmissionSettings`
- Renamed the `DirectLogSubmissionSettings` properties to match the
"Immutable" version (and remove the unnecessary prefix)
- Replace all usages of `Immutable*` with `DirectLogSubmissionSettings`
- Move `Immutable*Tests` into appropriate file and tweak
- Replace mutations with initialization (using `IConfigurationSource`)

## Test coverage

Essentially the same. I removed/tweaked some tests that are no longer
relevant

## Other details

Part of stack

- #6370
- #6376
- #6385
- #6386
- #6397 
- #6399
- #6400  👈 This PR
- #6405
- #6408
- #6415

tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.DirectLogSubmission.cs

@@ -17,21 +17,21 @@ internal static class DirectLogSubmission
             /// Default is empty (direct log submission disabled).
             /// Supports multiple values separated with semi-colons.
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionEnabledIntegrations"/>
+            /// <seealso cref="DirectLogSubmissionSettings.EnabledIntegrationNames"/>
             public const string EnabledIntegrations = "DD_LOGS_DIRECT_SUBMISSION_INTEGRATIONS";
 
             /// <summary>
             /// Set the name of the originating host for direct logs submission.
             /// Required for direct logs submission (default is machine name).
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionHost"/>
+            /// <seealso cref="DirectLogSubmissionSettings.Host"/>
             public const string Host = "DD_LOGS_DIRECT_SUBMISSION_HOST";
 
             /// <summary>
             /// Set the originating source for direct logs submission.
             /// Default is 'csharp'
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionSource"/>
+            /// <seealso cref="DirectLogSubmissionSettings.Source"/>
             public const string Source = "DD_LOGS_DIRECT_SUBMISSION_SOURCE";
 
             /// <summary>
@@ -40,44 +40,44 @@ internal static class DirectLogSubmission
             /// value are colon-separated. For example Key1:Value1, Key2:Value2. If not provided,
             /// <see cref="ConfigurationKeys.GlobalTags"/> are used instead
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionGlobalTags"/>
+            /// <seealso cref="DirectLogSubmissionSettings.GlobalTags"/>
             public const string GlobalTags = "DD_LOGS_DIRECT_SUBMISSION_TAGS";
 
             /// <summary>
             /// Configuration key for the url to send logs to.
             /// Default value uses the domain set in <see cref="ConfigurationKeys.Site"/>, so defaults to
             /// <c>https://http-intake.logs.datadoghq.com:443</c>.
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionUrl"/>
+            /// <seealso cref="DirectLogSubmissionSettings.IntakeUrl"/>
             public const string Url = "DD_LOGS_DIRECT_SUBMISSION_URL";
 
             /// <summary>
             /// Configuration key for the minimum level logs should have to be sent to the intake.
             /// Default value is <c>Information</c>.
             /// Should be one of <c>Verbose</c>,<c>Debug</c>,<c>Information</c>,<c>Warning</c>,<c>Error</c>,<c>Fatal</c>
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionMinimumLevel"/>
+            /// <seealso cref="DirectLogSubmissionSettings.MinimumLevel"/>
             public const string MinimumLevel = "DD_LOGS_DIRECT_SUBMISSION_MINIMUM_LEVEL";
 
             /// <summary>
             /// Configuration key for the maximum number of logs to send at one time
             /// Default value is <c>1,000</c>, the maximum accepted by the Datadog log API
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionBatchSizeLimit"/>
+            /// <seealso cref="DirectLogSubmissionSettings.BatchSizeLimit"/>
             public const string BatchSizeLimit = "DD_LOGS_DIRECT_SUBMISSION_MAX_BATCH_SIZE";
 
             /// <summary>
             /// Configuration key for the maximum number of logs to hold in internal queue at any one time
             /// Default value is <c>100,000</c>.
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionQueueSizeLimit"/>
+            /// <seealso cref="DirectLogSubmissionSettings.QueueSizeLimit"/>
             public const string QueueSizeLimit = "DD_LOGS_DIRECT_SUBMISSION_MAX_QUEUE_SIZE";
 
             /// <summary>
             /// Configuration key for the time to wait between checking for batches
             /// Default value is <c>2</c>s.
             /// </summary>
-            /// <seealso cref="DirectLogSubmissionSettings.DirectLogSubmissionBatchPeriod"/>
+            /// <seealso cref="DirectLogSubmissionSettings.BatchPeriod"/>
             public const string BatchPeriodSeconds = "DD_LOGS_DIRECT_SUBMISSION_BATCH_PERIOD_SECONDS";
         }
     }

tracer/src/Datadog.Trace/Configuration/ImmutableTracerSettings.cs

@@ -141,7 +141,7 @@ internal ImmutableTracerSettings(TracerSettings settings, bool unusedParamNotToU
             IsDataStreamsLegacyHeadersEnabled = settings.IsDataStreamsLegacyHeadersEnabled;
             IsRareSamplerEnabled = settings.IsRareSamplerEnabled;
 
-            LogSubmissionSettings = ImmutableDirectLogSubmissionSettings.Create(settings.LogSubmissionSettings);
+            LogSubmissionSettings = settings.LogSubmissionSettings;
             _logsInjectionEnabled = settings.LogSubmissionSettings.LogsInjectionEnabled;
 
             // we cached the static instance here, because is being used in the hotpath
@@ -461,7 +461,7 @@ internal ImmutableTracerSettings(TracerSettings settings, bool unusedParamNotToU
         /// </summary>
         internal int QueryStringReportingSize { get; }
 
-        internal ImmutableDirectLogSubmissionSettings LogSubmissionSettings { get; }
+        internal DirectLogSubmissionSettings LogSubmissionSettings { get; }
 
         /// <summary>
         /// Gets a value indicating whether to enable the updated WCF instrumentation that delays execution

tracer/src/Datadog.Trace/Logging/DirectSubmission/DirectLogSubmissionManager.cs

@@ -16,14 +16,14 @@ internal class DirectLogSubmissionManager
     {
         private static readonly IDatadogLogger Logger = DatadogLogging.GetLoggerFor<DirectLogSubmissionManager>();
 
-        private DirectLogSubmissionManager(ImmutableDirectLogSubmissionSettings settings, IDirectSubmissionLogSink sink, LogFormatter formatter)
+        private DirectLogSubmissionManager(DirectLogSubmissionSettings settings, IDirectSubmissionLogSink sink, LogFormatter formatter)
         {
             Settings = settings;
             Sink = sink;
             Formatter = formatter;
         }
 
-        public ImmutableDirectLogSubmissionSettings Settings { get; }
+        public DirectLogSubmissionSettings Settings { get; }
 
         public IDirectSubmissionLogSink Sink { get; }
 
@@ -32,7 +32,7 @@ private DirectLogSubmissionManager(ImmutableDirectLogSubmissionSettings settings
         public static DirectLogSubmissionManager Create(
             DirectLogSubmissionManager? previous,
             ImmutableTracerSettings settings,
-            ImmutableDirectLogSubmissionSettings directLogSettings,
+            DirectLogSubmissionSettings directLogSettings,
             ImmutableAzureAppServiceSettings? azureAppServiceSettings,
             string serviceName,
             string env,
@@ -56,7 +56,7 @@ public static DirectLogSubmissionManager Create(
             var apiFactory = LogsTransportStrategy.Get(directLogSettings);
             var logsApi = new LogsApi(directLogSettings.ApiKey, apiFactory);
 
-            return new DirectLogSubmissionManager(directLogSettings, new DirectSubmissionLogSink(logsApi, formatter, directLogSettings.BatchingOptions), formatter);
+            return new DirectLogSubmissionManager(directLogSettings, new DirectSubmissionLogSink(logsApi, formatter, directLogSettings.CreateBatchingSinkOptions()), formatter);
         }
 
         public async Task DisposeAsync()

tracer/src/Datadog.Trace/Logging/DirectSubmission/DirectLogSubmissionSettings.cs

@@ -7,10 +7,12 @@
 
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Linq;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
 using Datadog.Trace.Configuration.Telemetry;
+using Datadog.Trace.Logging.DirectSubmission.Sink.PeriodicBatching;
 using Datadog.Trace.PlatformHelpers;
 
 namespace Datadog.Trace.Logging.DirectSubmission
@@ -28,17 +30,25 @@ internal class DirectLogSubmissionSettings
         private const string IntakePrefix = "https://http-intake.logs.";
         private const string DefaultSite = "datadoghq.com";
         private const string IntakeSuffix = ":443";
+        private readonly bool[]? _enabledIntegrations;
+
+        internal static readonly IntegrationId[] SupportedIntegrations =
+        {
+            IntegrationId.Serilog,
+            IntegrationId.ILogger,
+            IntegrationId.Log4Net,
+            IntegrationId.NLog,
+            IntegrationId.XUnit,
+        };
 
         public DirectLogSubmissionSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry)
         {
-            // TODO: Combine DirectLogSubmissionSettings and ImmutableDirectLogSubmissionSettings
             source ??= NullConfigurationSource.Instance;
             var config = new ConfigurationBuilder(source, telemetry);
-
-            DirectLogSubmissionHost = config
+            Host = config
                                      .WithKeys(ConfigurationKeys.DirectLogSubmission.Host)
                                      .AsString(HostMetadata.Instance.Hostname ?? string.Empty);
-            DirectLogSubmissionSource = config
+            Source = config
                                        .WithKeys(ConfigurationKeys.DirectLogSubmission.Source)
                                        .AsString(DefaultSource);
 
@@ -56,7 +66,7 @@ public DirectLogSubmissionSettings(IConfigurationSource? source, IConfigurationT
                                          },
                                          validator: x => !string.IsNullOrEmpty(x));
 
-            DirectLogSubmissionMinimumLevel = config
+            MinimumLevel = config
                                              .WithKeys(ConfigurationKeys.DirectLogSubmission.MinimumLevel)
                                              .GetAs(
                                                   () => new DefaultResult<DirectSubmissionLogLevel>(DefaultMinimumLevel, nameof(DirectSubmissionLogLevel.Information)),
@@ -65,25 +75,18 @@ public DirectLogSubmissionSettings(IConfigurationSource? source, IConfigurationT
 
             var globalTags = config
                             .WithKeys(ConfigurationKeys.DirectLogSubmission.GlobalTags)
-                            .AsDictionary();
+                            .AsDictionary()
+                           ?.Where(kvp => !string.IsNullOrWhiteSpace(kvp.Key) && !string.IsNullOrWhiteSpace(kvp.Value))
+                            .ToDictionary(kvp => kvp.Key.Trim(), kvp => kvp.Value.Trim());
 
-            DirectLogSubmissionGlobalTags = globalTags?.Where(kvp => !string.IsNullOrWhiteSpace(kvp.Key) && !string.IsNullOrWhiteSpace(kvp.Value))
-                                                       .ToDictionary(kvp => kvp.Key.Trim(), kvp => kvp.Value.Trim())
-                                         ?? new Dictionary<string, string>();
+            GlobalTags = new ReadOnlyDictionary<string, string>(globalTags ?? []);
 
-            var logSubmissionIntegrations = config
-                                           .WithKeys(ConfigurationKeys.DirectLogSubmission.EnabledIntegrations)
-                                           .AsString()
-                                          ?.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries) ??
-                                            Enumerable.Empty<string>();
-            DirectLogSubmissionEnabledIntegrations = new HashSet<string>(logSubmissionIntegrations, StringComparer.OrdinalIgnoreCase);
-
-            DirectLogSubmissionBatchSizeLimit = config
+            BatchSizeLimit = config
                                                .WithKeys(ConfigurationKeys.DirectLogSubmission.BatchSizeLimit)
                                                .AsInt32(DefaultBatchSizeLimit, x => x > 0)
                                                .Value;
 
-            DirectLogSubmissionQueueSizeLimit = config
+            QueueSizeLimit = config
                                                .WithKeys(ConfigurationKeys.DirectLogSubmission.QueueSizeLimit)
                                                .AsInt32(DefaultQueueSizeLimit, x => x > 0)
                                                .Value;
@@ -93,20 +96,56 @@ public DirectLogSubmissionSettings(IConfigurationSource? source, IConfigurationT
                      .AsInt32(DefaultBatchPeriodSeconds, x => x > 0)
                      .Value;
 
-            DirectLogSubmissionBatchPeriod = TimeSpan.FromSeconds(seconds);
+            BatchPeriod = TimeSpan.FromSeconds(seconds);
+
+            ApiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString() ?? string.Empty;
+            bool[]? enabledIntegrations = null;
+
+            List<string> validationErrors = [];
+            var logSubmissionIntegrations = config
+                                           .WithKeys(ConfigurationKeys.DirectLogSubmission.EnabledIntegrations)
+                                           .AsString()
+                                          ?.Split([';'], StringSplitOptions.RemoveEmptyEntries);
 
-            ApiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString();
+            if (logSubmissionIntegrations is { Length: > 0 })
+            {
+                foreach (var integrationName in logSubmissionIntegrations)
+                {
+                    if (!IntegrationRegistry.TryGetIntegrationId(integrationName, out var integrationId))
+                    {
+                        validationErrors.Add(
+                            "Unknown integration: " + integrationName +
+                            ". Use a valid logs integration name: " +
+                            string.Join(", ", SupportedIntegrations.Select(x => IntegrationRegistry.GetName(x))));
+                        continue;
+                    }
+
+                    if (!SupportedIntegrations.Contains(integrationId))
+                    {
+                        validationErrors.Add(
+                            "Integration: " + integrationName + " is not a supported direct log submission integration. " +
+                            "Use one of " + string.Join(", ", SupportedIntegrations.Select(x => IntegrationRegistry.GetName(x))));
+                        continue;
+                    }
+
+                    enabledIntegrations ??= new bool[IntegrationRegistry.Ids.Count];
+                    if (!enabledIntegrations[(int)integrationId])
+                    {
+                        enabledIntegrations[(int)integrationId] = true;
+                    }
+                }
+            }
 
-            var isEnabled = DirectLogSubmissionEnabledIntegrations.Count > 0;
-            var validationErrors = new List<string>();
+            var isEnabled = enabledIntegrations is not null;
+            _enabledIntegrations = enabledIntegrations;
 
-            if (string.IsNullOrWhiteSpace(DirectLogSubmissionHost))
+            if (string.IsNullOrWhiteSpace(Host))
             {
                 isEnabled = false;
                 validationErrors.Add($"Missing required setting '{ConfigurationKeys.DirectLogSubmission.Host}'.");
             }
 
-            if (string.IsNullOrWhiteSpace(DirectLogSubmissionSource))
+            if (string.IsNullOrWhiteSpace(Source))
             {
                 isEnabled = false;
                 validationErrors.Add($"Missing required setting '{ConfigurationKeys.DirectLogSubmission.Source}'.");
@@ -119,7 +158,7 @@ public DirectLogSubmissionSettings(IConfigurationSource? source, IConfigurationT
             }
             else
             {
-                DirectLogSubmissionUrl = uri;
+                IntakeUrl = uri;
             }
 
             if (string.IsNullOrWhiteSpace(ApiKey))
@@ -145,69 +184,77 @@ public DirectLogSubmissionSettings(IConfigurationSource? source, IConfigurationT
         /// </summary>
         internal IReadOnlyList<string> ValidationErrors { get; }
 
-        /// <summary>
-        /// Gets the integrations enabled for direct log submission
-        /// </summary>
-        /// <seealso cref="ConfigurationKeys.DirectLogSubmission.EnabledIntegrations" />
-        internal HashSet<string> DirectLogSubmissionEnabledIntegrations { get; }
-
         /// <summary>
         /// Gets the originating host name for direct logs submission
         /// </summary>
         /// <seealso cref="ConfigurationKeys.DirectLogSubmission.Host" />
-        internal string DirectLogSubmissionHost { get; }
+        internal string Host { get; }
 
         /// <summary>
         /// Gets the originating source for direct logs submission
         /// </summary>
         /// <seealso cref="ConfigurationKeys.DirectLogSubmission.Source" />
-        internal string DirectLogSubmissionSource { get; }
+        internal string Source { get; }
 
         /// <summary>
         /// Gets the global tags, which are applied to all directly submitted logs. If not provided,
         /// <see cref="TracerSettings.GlobalTags"/> are used instead
         /// </summary>
         /// <seealso cref="ConfigurationKeys.DirectLogSubmission.GlobalTags" />
-        internal IDictionary<string, string> DirectLogSubmissionGlobalTags { get; }
+        internal IReadOnlyDictionary<string, string> GlobalTags { get; }
 
         /// <summary>
         /// Gets the url to send logs to
         /// </summary>
         /// <seealso cref="ConfigurationKeys.DirectLogSubmission.Url" />
-        internal Uri? DirectLogSubmissionUrl { get; }
+        internal Uri? IntakeUrl { get; }
 
         /// <summary>
         /// Gets the minimum level logs should have to be sent to the intake.
         /// </summary>
         /// <seealso cref="ConfigurationKeys.DirectLogSubmission.Url" />
-        internal DirectSubmissionLogLevel DirectLogSubmissionMinimumLevel { get; }
+        internal DirectSubmissionLogLevel MinimumLevel { get; }
 
         /// <summary>
         /// Gets the maximum number of logs to send at one time
         /// </summary>
         /// <seealso cref="ConfigurationKeys.DirectLogSubmission.BatchSizeLimit"/>
-        internal int DirectLogSubmissionBatchSizeLimit { get; }
+        internal int BatchSizeLimit { get; }
 
         /// <summary>
         /// Gets the maximum number of logs to hold in internal queue at any one time
         /// </summary>
         /// <seealso cref="ConfigurationKeys.DirectLogSubmission.QueueSizeLimit"/>
-        internal int DirectLogSubmissionQueueSizeLimit { get; }
+        internal int QueueSizeLimit { get; }
 
         /// <summary>
         /// Gets or sets the time to wait between checking for batches
         /// </summary>
         /// <seealso cref="ConfigurationKeys.DirectLogSubmission.BatchPeriodSeconds"/>
-        internal TimeSpan DirectLogSubmissionBatchPeriod { get; set; }
+        internal TimeSpan BatchPeriod { get; set; }
 
         /// <summary>
         /// Gets the Datadog API key
         /// </summary>
-        internal string? ApiKey { get; }
+        internal string ApiKey { get; }
 
         /// <summary>
         /// Gets or sets a value indicating whether logs injection is enabled or disabled
         /// </summary>
         internal bool LogsInjectionEnabled { get; set; }
+
+        public IEnumerable<string> EnabledIntegrationNames
+            => SupportedIntegrations.Where(x => _enabledIntegrations?[(int)x] == true).Select(x => IntegrationRegistry.GetName(x));
+
+        public BatchingSinkOptions CreateBatchingSinkOptions()
+            => new(
+                batchSizeLimit: BatchSizeLimit,
+                queueLimit: QueueSizeLimit,
+                period: BatchPeriod);
+
+        public bool IsIntegrationEnabled(IntegrationId integrationId)
+        {
+            return IsEnabled && _enabledIntegrations is not null && _enabledIntegrations[(int)integrationId];
+        }
     }
 }

tracer/src/Datadog.Trace/Logging/DirectSubmission/Formatting/LogFormatter.cs

@@ -40,7 +40,7 @@ internal class LogFormatter
 
         public LogFormatter(
             ImmutableTracerSettings settings,
-            ImmutableDirectLogSubmissionSettings directLogSettings,
+            DirectLogSubmissionSettings directLogSettings,
             ImmutableAzureAppServiceSettings? aasSettings,
             string serviceName,
             string env,