Add hosting config support to control 3.x log disablement

release_notes.md

@@ -4,4 +4,5 @@
 -->
 *Please note we have reached end-of-life (EOL) support for v3.x.* For more information on supported runtime versions, please see [here.](https://learn.microsoft.com/en-us/azure/azure-functions/functions-versions?tabs=v4&pivots=programming-language-csharp)
 
-- Disable all non host startup logs for v3.x (#10399)
+- Disable all non host startup logs for v3.x (#10399)
+- Add hosting config (`DisableHostLogs`) to enable/disable v3.x logs (#10552)

src/WebJobs.Script.WebHost/Diagnostics/ILoggingBuilderExtensions.cs

@@ -8,12 +8,12 @@ namespace Microsoft.Extensions.Logging
 {
     public static class ILoggingBuilderExtensions
     {
-        public static void AddWebJobsSystem<T>(this ILoggingBuilder builder) where T : SystemLoggerProvider
+        public static void AddWebJobsSystem<T>(this ILoggingBuilder builder, bool restrictHostLogs = false) where T : SystemLoggerProvider
         {
             builder.Services.AddSingleton<ILoggerProvider, T>();
 
             // Log all logs to SystemLogger
-            builder.AddDefaultWebJobsFilters<T>(LogLevel.Trace);
+            builder.AddDefaultWebJobsFilters<T>(LogLevel.Trace, restrictHostLogs);
         }
     }
 }

src/WebJobs.Script.WebHost/WebScriptHostBuilderExtension.cs

@@ -75,9 +75,12 @@ public static IHostBuilder AddWebScriptHost(this IHostBuilder builder, IServiceP
                 })
                 .ConfigureLogging(loggingBuilder =>
                 {
+                    var hostingConfigOptions = rootServiceProvider.GetService<IOptions<FunctionsHostingConfigOptions>>();
+                    var restrictHostLogs = RestrictHostLogs(hostingConfigOptions.Value);
+
                     loggingBuilder.Services.AddSingleton<ILoggerFactory, ScriptLoggerFactory>();
+                    loggingBuilder.AddWebJobsSystem<SystemLoggerProvider>(restrictHostLogs);
 
-                    loggingBuilder.AddWebJobsSystem<SystemLoggerProvider>();
                     if (environment.IsAzureMonitorEnabled())
                     {
                         loggingBuilder.Services.AddSingleton<ILoggerProvider, AzureMonitorDiagnosticLoggerProvider>();
@@ -170,6 +173,12 @@ private static void ConfigureRegisteredBuilders<TBuilder>(TBuilder builder, ISer
             }
         }
 
+        private static bool RestrictHostLogs(FunctionsHostingConfigOptions options)
+        {
+            // Feature flag should take precedence over the host configuration
+            return !FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableHostLogs) && options.RestrictHostLogs;
+        }
+
         /// <summary>
         /// Used internally to register Newtonsoft formatters with our ScriptHost.
         /// </summary>

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -85,6 +85,22 @@ public bool SwtIssuerEnabled
             }
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether non-critical logs should be disabled in the host.
+        /// </summary>
+        public bool RestrictHostLogs
+        {
+            get
+            {
+                return GetFeatureOrDefault(ScriptConstants.HostingConfigRestrictHostLogs, "0") == "1";
+            }
+
+            set
+            {
+                _features[ScriptConstants.HostingConfigRestrictHostLogs] = value ? "1" : "0";
+            }
+        }
+
         /// <summary>
         /// Gets feature by name.
         /// </summary>

src/WebJobs.Script/Extensions/ScriptLoggingBuilderExtensions.cs

@@ -6,7 +6,6 @@
 using System.Linq;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Azure.WebJobs.Script;
-using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
@@ -15,17 +14,25 @@ namespace Microsoft.Extensions.Logging
 {
     public static class ScriptLoggingBuilderExtensions
     {
-        private static ConcurrentDictionary<string, bool> _filteredCategoryCache = new ConcurrentDictionary<string, bool>();
+        private static ConcurrentDictionary<string, bool> _filteredCategoryCache = new ();
+        private static ImmutableArray<string> _allowedLogCategoryPrefixes = ImmutableArray<string>.Empty;
 
-        public static ILoggingBuilder AddDefaultWebJobsFilters(this ILoggingBuilder builder)
+        // For testing only
+        internal static ImmutableArray<string> AllowedSystemLogPrefixes => _allowedLogCategoryPrefixes;
+
+        public static ILoggingBuilder AddDefaultWebJobsFilters(this ILoggingBuilder builder, bool restrictHostLogs = false)
         {
+            SetSystemLogCategoryPrefixes(restrictHostLogs);
+
             builder.SetMinimumLevel(LogLevel.None);
             builder.AddFilter((c, l) => Filter(c, l, LogLevel.Information));
             return builder;
         }
 
-        public static ILoggingBuilder AddDefaultWebJobsFilters<T>(this ILoggingBuilder builder, LogLevel level) where T : ILoggerProvider
+        public static ILoggingBuilder AddDefaultWebJobsFilters<T>(this ILoggingBuilder builder, LogLevel level, bool restrictHostLogs = false) where T : ILoggerProvider
         {
+            SetSystemLogCategoryPrefixes(restrictHostLogs);
+
             builder.AddFilter<T>(null, LogLevel.None);
             builder.AddFilter<T>((c, l) => Filter(c, l, level));
             return builder;
@@ -38,11 +45,18 @@ internal static bool Filter(string category, LogLevel actualLevel, LogLevel minL
 
         private static bool IsFiltered(string category)
         {
-            ImmutableArray<string> systemLogCategoryPrefixes = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableHostLogs)
-                                                                ? ScriptConstants.SystemLogCategoryPrefixes
-                                                                : ScriptConstants.RestrictedSystemLogCategoryPrefixes;
+            return _filteredCategoryCache.GetOrAdd(category, c => _allowedLogCategoryPrefixes.Any(p => c.StartsWith(p)));
+        }
+
+        private static void SetSystemLogCategoryPrefixes(bool restrictHostLogs)
+        {
+            var previous = _allowedLogCategoryPrefixes;
+            _allowedLogCategoryPrefixes = restrictHostLogs ? ScriptConstants.RestrictedSystemLogCategoryPrefixes : ScriptConstants.SystemLogCategoryPrefixes;
 
-            return _filteredCategoryCache.GetOrAdd(category, c => systemLogCategoryPrefixes.Any(p => category.StartsWith(p)));
+            if (!previous.IsDefault && !previous.SequenceEqual(_allowedLogCategoryPrefixes))
+            {
+                _filteredCategoryCache.Clear();
+            }
         }
 
         public static void AddConsoleIfEnabled(this ILoggingBuilder builder, HostBuilderContext context)

src/WebJobs.Script/ScriptConstants.cs

@@ -127,6 +127,7 @@ public static class ScriptConstants
         public const string FeatureFlagEnableHostLogs = "EnableHostLogs";
         public const string HostingConfigSwtAuthenticationEnabled = "SwtAuthenticationEnabled";
         public const string HostingConfigSwtIssuerEnabled = "SwtIssuerEnabled";
+        public const string HostingConfigRestrictHostLogs = "RestrictHostLogs";
 
         public const string SiteAzureFunctionsUriFormat = "https://{0}.azurewebsites.net/azurefunctions";
         public const string ScmSiteUriFormat = "https://{0}.scm.azurewebsites.net";

test/TestFunctions/TestFunctions.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
     <StyleCopTreatErrorsAsWarnings>false</StyleCopTreatErrorsAsWarnings>
-    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
     <WarningsAsErrors />
   </PropertyGroup>
 

test/WebJobs.Script.Tests.Integration/EventHubs/EventHubsEndToEndTestsBase.cs

@@ -21,7 +21,7 @@ public EventHubsEndToEndTestsBase(TestFixture fixture)
             _fixture = fixture;
         }
 
-        [Fact]
+        [Fact(Skip = "SAS authentication has been disabled for the namespace.")]
         public async Task EventHub()
         {
             // Event Hub needs the following environment vars:

test/WebJobs.Script.Tests.Integration/Management/InstanceManagerTests.cs

@@ -356,7 +356,7 @@ public async Task ValidateContext_InvalidZipUrl_WebsiteUseZip_ReturnsError()
         {
             var environmentSettings = new Dictionary<string, string>()
             {
-                { EnvironmentSettingNames.AzureWebsiteZipDeployment, "http://invalid.com/invalid/dne" }
+                { EnvironmentSettingNames.AzureWebsiteZipDeployment, "http://invalid.test/invalid/dne" }
             };
 
             var environment = new TestEnvironment();
@@ -439,7 +439,7 @@ public async Task ValidateContext_Succeeds_For_WebsiteUseZip_Only()
         {
             var environment = new Dictionary<string, string>()
             {
-                { EnvironmentSettingNames.AzureWebsiteZipDeployment, "http://microsoft.com" }
+                { EnvironmentSettingNames.AzureWebsiteZipDeployment, "https://valid-zip.test" }
             };
             var assignmentContext = new HostAssignmentContext
             {
@@ -449,7 +449,19 @@ public async Task ValidateContext_Succeeds_For_WebsiteUseZip_Only()
                 IsWarmupRequest = false
             };
 
-            string error = await _instanceManager.ValidateContext(assignmentContext);
+            var handlerMock = new Mock<HttpMessageHandler>();
+            handlerMock.Protected().Setup<Task<HttpResponseMessage>>("SendAsync",
+                ItExpr.IsAny<HttpRequestMessage>(),
+                ItExpr.IsAny<CancellationToken>()).ReturnsAsync(new HttpResponseMessage
+                {
+                    StatusCode = HttpStatusCode.OK,
+                });
+
+            var instanceManager = new InstanceManager(_optionsFactory, TestHelpers.CreateHttpClientFactory(handlerMock.Object).CreateClient(),
+                _scriptWebEnvironment, _environment, _loggerFactory.CreateLogger<InstanceManager>(),
+                new TestMetricsLogger(), null, _runFromPackageHandler, _packageDownloadHandler.Object);
+
+            string error = await instanceManager.ValidateContext(assignmentContext);
             Assert.Null(error);
 
             string[] expectedOutputLines =
@@ -539,8 +551,8 @@ public async Task ValidateContext_Succeeds_For_WebsiteUseZip_With_ScmPackageDefi
         {
             var environment = new Dictionary<string, string>()
             {
-                { EnvironmentSettingNames.AzureWebsiteZipDeployment, "http://microsoft.com" },
-                { EnvironmentSettingNames.ScmRunFromPackage, "http://microsoft.com" }
+                { EnvironmentSettingNames.AzureWebsiteZipDeployment, "https://valid.test" },
+                { EnvironmentSettingNames.ScmRunFromPackage, "https://valid.tests" }
             };
             var assignmentContext = new HostAssignmentContext
             {
@@ -550,7 +562,19 @@ public async Task ValidateContext_Succeeds_For_WebsiteUseZip_With_ScmPackageDefi
                 IsWarmupRequest = false
             };
 
-            string error = await _instanceManager.ValidateContext(assignmentContext);
+            var handlerMock = new Mock<HttpMessageHandler>();
+            handlerMock.Protected().Setup<Task<HttpResponseMessage>>("SendAsync",
+                ItExpr.IsAny<HttpRequestMessage>(),
+                ItExpr.IsAny<CancellationToken>()).ReturnsAsync(new HttpResponseMessage
+                {
+                    StatusCode = HttpStatusCode.OK,
+                });
+
+            var instanceManager = new InstanceManager(_optionsFactory, TestHelpers.CreateHttpClientFactory(handlerMock.Object).CreateClient(),
+                _scriptWebEnvironment, _environment, _loggerFactory.CreateLogger<InstanceManager>(),
+                new TestMetricsLogger(), null, _runFromPackageHandler, _packageDownloadHandler.Object);
+
+            string error = await instanceManager.ValidateContext(assignmentContext);
             Assert.Null(error);
 
             string[] expectedOutputLines =

test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_Node.cs

@@ -39,7 +39,7 @@ public SamplesEndToEndTests_Node(TestFixture fixture)
             _settingsManager = ScriptSettingsManager.Instance;
         }
 
-        [Fact]
+        [Fact(Skip = "SAS authentication has been disabled for the namespace.")]
         public async Task EventHubTrigger()
         {
             // write 3 events

test/WebJobs.Script.Tests.Shared/TestHelpers.cs

@@ -23,6 +23,7 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 using Microsoft.WebJobs.Script.Tests;
+using Moq;
 using Xunit.Abstractions;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
@@ -475,5 +476,19 @@ public static IAzureStorageProvider GetAzureStorageProvider(IConfiguration confi
             var azureStorageProvider = tempHost.Services.GetRequiredService<IAzureStorageProvider>();
             return azureStorageProvider;
         }
+
+        /// <summary>
+        /// Mock an HttpClientFactory and its CreateClient functionality.
+        /// </summary>
+        /// <param name="handler">Some tests pass a mock HttpHandler into their HttpClient.</param>
+        /// <returns>IHttpClientFactory.</returns>
+        public static IHttpClientFactory CreateHttpClientFactory(HttpMessageHandler handler = null)
+        {
+            var httpClient = handler == null ? new HttpClient() : new HttpClient(handler);
+            var mockFactory = new Mock<IHttpClientFactory>();
+            mockFactory.Setup(m => m.CreateClient(It.IsAny<string>()))
+                 .Returns(httpClient);
+            return mockFactory.Object;
+        }
     }
 }

test/WebJobs.Script.Tests.Shared/TestHostBuilderExtensions.cs

@@ -29,13 +29,13 @@ namespace Microsoft.WebJobs.Script.Tests
 {
     public static class TestHostBuilderExtensions
     {
-        public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder builder, Action<ScriptApplicationHostOptions> configure = null, bool runStartupHostedServices = false)
+        public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder builder, Action<ScriptApplicationHostOptions> configure = null, bool runStartupHostedServices = false, IEnvironment environment = null)
         {
-            return builder.ConfigureDefaultTestWebScriptHost(null, configure, runStartupHostedServices);
+            return builder.ConfigureDefaultTestWebScriptHost(null, configure, runStartupHostedServices, environment: environment);
         }
 
         public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder builder, Action<IWebJobsBuilder> configureWebJobs,
-            Action<ScriptApplicationHostOptions> configure = null, bool runStartupHostedServices = false, Action<IServiceCollection> configureRootServices = null)
+            Action<ScriptApplicationHostOptions> configure = null, bool runStartupHostedServices = false, Action<IServiceCollection> configureRootServices = null, IEnvironment environment = null)
         {
             var webHostOptions = new ScriptApplicationHostOptions()
             {
@@ -48,9 +48,18 @@ public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder b
 
             // Register root services
             var services = new ServiceCollection();
+
+            if (environment is not null)
+            {
+                services.AddSingleton<IEnvironment>(environment);
+            }
+            else
+            {
+                services.AddSingleton<IEnvironment>(SystemEnvironment.Instance);
+            }
+
             AddMockedSingleton<IDebugStateProvider>(services);
             AddMockedSingleton<IScriptHostManager>(services);
-            AddMockedSingleton<IEnvironment>(services);
             AddMockedSingleton<IScriptWebHostEnvironment>(services);
             AddMockedSingleton<IEventGenerator>(services);
             AddMockedSingleton<IFunctionInvocationDispatcherFactory>(services);

test/WebJobs.Script.Tests/Configuration/DefaultDependencyValidatorTests.cs

@@ -36,27 +36,28 @@ public async Task Validator_AllValid()
         [Fact]
         public async Task Validator_InvalidServices_LogsError()
         {
-            SystemEnvironment.Instance.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableHostLogs);
-
-            LogMessage invalidServicesMessage = await RunTest(configureJobHost: s =>
+            using (new TestScopedEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableHostLogs))
             {
-                s.AddSingleton<IHostedService, MyHostedService>();
-                s.AddSingleton<IScriptEventManager, MyScriptEventManager>();
-                s.AddSingleton<IMetricsLogger>(new MyMetricsLogger());
-
-                // Try removing system logger
-                var descriptor = s.Single(p => p.ImplementationType == typeof(SystemLoggerProvider));
-                s.Remove(descriptor);
-            });
-
-            Assert.NotNull(invalidServicesMessage);
-
-            IEnumerable<string> messageLines = invalidServicesMessage.Exception.Message.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim());
-            Assert.Equal(5, messageLines.Count());
-            Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyHostedService).AssemblyQualifiedName));
-            Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyScriptEventManager).AssemblyQualifiedName));
-            Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyMetricsLogger).AssemblyQualifiedName));
-            Assert.Contains(messageLines, p => p.StartsWith("[Missing]") && p.EndsWith(typeof(SystemLoggerProvider).AssemblyQualifiedName));
+                LogMessage invalidServicesMessage = await RunTest(configureJobHost: s =>
+                {
+                    s.AddSingleton<IHostedService, MyHostedService>();
+                    s.AddSingleton<IScriptEventManager, MyScriptEventManager>();
+                    s.AddSingleton<IMetricsLogger>(new MyMetricsLogger());
+
+                    // Try removing system logger
+                    var descriptor = s.Single(p => p.ImplementationType == typeof(SystemLoggerProvider));
+                    s.Remove(descriptor);
+                });
+
+                Assert.NotNull(invalidServicesMessage);
+
+                IEnumerable<string> messageLines = invalidServicesMessage.Exception.Message.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim());
+                Assert.Equal(5, messageLines.Count());
+                Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyHostedService).AssemblyQualifiedName));
+                Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyScriptEventManager).AssemblyQualifiedName));
+                Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyMetricsLogger).AssemblyQualifiedName));
+                Assert.Contains(messageLines, p => p.StartsWith("[Missing]") && p.EndsWith(typeof(SystemLoggerProvider).AssemblyQualifiedName));
+            }
         }
 
         [Fact]