Detect .NET SDK & VSLANG Custom Language Settings & Apply To MSBuild (#…

…8503)

Fixes #1596

Changes Made
SetConsoleUI now calls into a helper which sets the encoding to support non-en languages and checks if an environment variable exists to change the language to.

Testing
Setting DOTNET_CLI_UI_LANGUAGE=ja now changes msbuild correctly:
image

Doing a complicated build (aka building MSBuild) to use multiple threads shows other threads seem to use the same UI culture:

image

See that chcp remains the same after execution:
image

(Was set to 65001 temporarily but back to the original page before execution.)

Notes
Much of this code is a port of this code: dotnet/sdk#29755
There are some details about the code here.

[!] In addition, it will introduce a breaking change for msbuild just like the SDK.
The break is documented here for the sdk: dotnet/docs#34250

newc/Program.cs

@@ -0,0 +1,2 @@
+// See https://aka.ms/new-console-template for more information
+Console.WriteLine("Hello, World!");

newc/newc.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

src/Build/Logging/DistributedLoggers/DistributedFileLogger.cs

@@ -97,6 +97,7 @@ public void Initialize(IEventSource eventSource)
             ErrorUtilities.VerifyThrowArgumentNull(eventSource, nameof(eventSource));
             ParseFileLoggerParameters();
             string fileName = _logFile;
+
             try
             {
                 // Create a new file logger and pass it some parameters to make the build log very detailed

src/Build/Logging/FileLogger.cs

@@ -39,6 +39,11 @@ public FileLogger()
                 colorReset: BaseConsoleLogger.DontResetColor)
         {
             WriteHandler = Write;
+
+            if (EncodingUtilities.GetExternalOverriddenUILanguageIfSupportableWithEncoding() != null)
+            {
+                _encoding = Encoding.UTF8;
+            }
         }
 
         #endregion

src/Deprecated/Engine/Logging/FileLogger.cs

@@ -117,7 +117,6 @@ public override void Initialize(IEventSource eventSource, int nodeCount)
         /// <summary>
         /// The handler for the write delegate of the console logger we are deriving from.
         /// </summary>
-        /// <owner>KieranMo</owner>
         /// <param name="text">The text to write to the log</param>
         private void Write(string text)
         {
@@ -143,7 +142,6 @@ private void Write(string text)
         /// <summary>
         /// Shutdown method implementation of ILogger - we need to flush and close our logfile.
         /// </summary>
-        /// <owner>KieranMo</owner>
         public override void Shutdown()
         {
             fileWriter?.Close();
@@ -152,7 +150,6 @@ public override void Shutdown()
         /// <summary>
         /// Parses out the logger parameters from the Parameters string.
         /// </summary>
-        /// <owner>KieranMo</owner>
         private void ParseFileLoggerParameters()
         {
             if (this.Parameters != null)
@@ -180,7 +177,6 @@ private void ParseFileLoggerParameters()
         /// <summary>
         /// Apply a parameter parsed by the file logger.
         /// </summary>
-        /// <owner>KieranMo</owner>
         private void ApplyFileLoggerParameter(string parameterName, string parameterValue)
         {
             switch (parameterName.ToUpperInvariant())

src/Framework/EncodingUtilities.cs

@@ -3,11 +3,15 @@
 
 using System;
 using System.Diagnostics;
+using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
+using System.Security;
 using System.Text;
 
 using Microsoft.Build.Framework;
+using Microsoft.Win32;
 
 #nullable disable
 
@@ -247,5 +251,96 @@ internal static Encoding BatchFileEncoding(string contents, string encodingSpeci
                         : EncodingUtilities.Utf8WithoutBom;
             }
         }
+#nullable enable
+        /// <summary>
+        /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect the SDK variable.
+        /// To use the corresponding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases.
+        /// This code introduces a breaking change in .NET 8 due to the encoding of the console being changed.
+        /// If the environment variables are undefined, this function should be a no-op.
+        /// </summary>
+        /// <returns>
+        /// The custom language that was set by the user for an 'external' tool besides MSBuild.
+        /// Returns <see langword="null"/> if none are set.
+        /// </returns>
+        public static CultureInfo? GetExternalOverriddenUILanguageIfSupportableWithEncoding()
+        {
+            if (!ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_8))
+            {
+                return null;
+            }
+
+            CultureInfo? externalLanguageSetting = GetExternalOverriddenUILanguage();
+            if (externalLanguageSetting != null)
+            {
+                if (
+                    !externalLanguageSetting.TwoLetterISOLanguageName.Equals("en", StringComparison.InvariantCultureIgnoreCase) &&
+                    CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()
+                    )
+                {
+                    // Setting both encodings causes a change in the CHCP, making it so we don't need to P-Invoke CHCP ourselves.
+                    Console.OutputEncoding = Encoding.UTF8;
+                    // If the InputEncoding is not set, the encoding will work in CMD but not in PowerShell, as the raw CHCP page won't be changed.
+                    Console.InputEncoding = Encoding.UTF8;
+                    return externalLanguageSetting;
+                }
+                else if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                {
+                    return externalLanguageSetting;
+                }
+            }
+
+            return null;
+        }
+
+        public static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version.Major >= 10) // UTF-8 is only officially supported on 10+.
+            {
+                try
+                {
+                    using RegistryKey? windowsVersionRegistry = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion");
+                    string? buildNumber = windowsVersionRegistry?.GetValue("CurrentBuildNumber")?.ToString();
+                    const int buildNumberThatOfficiallySupportsUTF8 = 18363;
+                    return buildNumber != null && (int.Parse(buildNumber) >= buildNumberThatOfficiallySupportsUTF8 || ForceUniversalEncodingOptInEnabled());
+                }
+                catch (Exception ex) when (ex is SecurityException or ObjectDisposedException)
+                {
+                    // We don't want to break those in VS on older versions of Windows with a non-en language.
+                    // Allow those without registry permissions to force the encoding, however.
+                    return ForceUniversalEncodingOptInEnabled();
+                }
+            }
+
+            return false;
+        }
+
+        private static bool ForceUniversalEncodingOptInEnabled()
+        {
+            return string.Equals(Environment.GetEnvironmentVariable("DOTNET_CLI_FORCE_UTF8_ENCODING"), "true", StringComparison.OrdinalIgnoreCase);
+        }
+
+        /// <summary>
+        /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE.)
+        /// Does NOT check System Locale or OS Display Language.
+        /// Ported from the .NET SDK: https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs
+        /// </summary>
+        /// <returns>The custom language that was set by the user for an 'external' tool besides MSBuild.
+        /// Returns null if none are set.</returns>
+        private static CultureInfo? GetExternalOverriddenUILanguage()
+        {
+            // DOTNET_CLI_UI_LANGUAGE=<culture name> is the main way for users to customize the CLI's UI language via the .NET SDK.
+            string? dotnetCliLanguage = Environment.GetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE");
+            if (dotnetCliLanguage != null)
+            {
+                try
+                {
+                    return new CultureInfo(dotnetCliLanguage);
+                }
+                catch (CultureNotFoundException) { }
+            }
+
+            return null;
+        }
     }
 }
+

src/MSBuild.UnitTests/XMake_Tests.cs

@@ -9,6 +9,7 @@
 using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
+using System.Runtime.InteropServices;
 using System.Threading;
 using Microsoft.Build.CommandLine;
 using Microsoft.Build.Framework;
@@ -642,6 +643,60 @@ public void SetConsoleUICulture()
             thisThread.CurrentUICulture = originalUICulture;
         }
 
+
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void ConsoleUIRespectsSDKLanguage(bool enableFeature)
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !EncodingUtilities.CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding())
+            {
+                return; // The feature to detect .NET SDK Languages is not enabled on this machine, so don't test it.
+            }
+
+            const string DOTNET_CLI_UI_LANGUAGE = nameof(DOTNET_CLI_UI_LANGUAGE);
+            using TestEnvironment testEnvironment = TestEnvironment.Create();
+            // Save the current environment info so it can be restored.
+            var originalUILanguage = Environment.GetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE);
+
+            var originalOutputEncoding = Console.OutputEncoding;
+            var originalInputEncoding = Console.InputEncoding;
+            Thread thisThread = Thread.CurrentThread;
+            CultureInfo originalUICulture = thisThread.CurrentUICulture;
+
+            try
+            {
+                // Set the UI language based on the SDK environment var.
+                testEnvironment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, "ja"); // Japanese chose arbitrarily.
+                ChangeWaves.ResetStateForTests();
+                if (!enableFeature)
+                {
+                    testEnvironment.SetEnvironmentVariable("MSBUILDDISABLEFEATURESFROMVERSION", ChangeWaves.Wave17_8.ToString());
+                }
+                MSBuildApp.SetConsoleUI();
+
+                Assert.Equal(enableFeature ? new CultureInfo("ja") : CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(), thisThread.CurrentUICulture);
+                if (enableFeature)
+                {
+                    Assert.Equal(65001, Console.OutputEncoding.CodePage); // UTF-8 enabled for correct rendering.
+                }
+            }
+            finally
+            {
+                // Restore the current UI culture back to the way it was at the beginning of this unit test.
+                thisThread.CurrentUICulture = originalUICulture;
+                // Restore for full framework
+                CultureInfo.CurrentCulture = originalUICulture;
+                CultureInfo.DefaultThreadCurrentUICulture = originalUICulture;
+
+                // MSBuild should also restore the encoding upon exit, but we don't create that context here.
+                Console.OutputEncoding = originalOutputEncoding;
+                Console.InputEncoding = originalInputEncoding;
+
+                BuildEnvironmentHelper.ResetInstance_ForUnitTestsOnly();
+            }
+        }
+
         /// <summary>
         /// We shouldn't change the UI culture if the current UI culture is invariant.
         /// In other cases, we can get an exception on CultureInfo creation when System.Globalization.Invariant enabled.
@@ -822,6 +877,10 @@ public void TestEnvironmentTest()
         [Fact]
         public void MSBuildEngineLogger()
         {
+            using TestEnvironment testEnvironment = TestEnvironment.Create();
+            testEnvironment.SetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE", "en"); // build machines may have other values.
+            CultureInfo.CurrentUICulture = new CultureInfo("en"); // Validate that the thread will produce an english log regardless of the machine OS language
+
             string oldValueForMSBuildLoadMicrosoftTargetsReadOnly = Environment.GetEnvironmentVariable("MSBuildLoadMicrosoftTargetsReadOnly");
             string projectString =
                    "<?xml version=\"1.0\" encoding=\"utf-8\"?>" +
@@ -858,6 +917,8 @@ public void MSBuildEngineLogger()
 
                 var logFileContents = File.ReadAllText(logFile);
 
+                Assert.Equal(new CultureInfo("en"), Thread.CurrentThread.CurrentUICulture);
+
                 logFileContents.ShouldContain("Process = ");
                 logFileContents.ShouldContain("MSBuild executable path = ");
                 logFileContents.ShouldContain("Command line arguments = ");

src/MSBuild/AutomaticEncodingRestorer.cs

@@ -0,0 +1,68 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Security;
+using System.Text;
+
+namespace Microsoft.Build.CommandLine
+{
+    /// <summary>
+    /// Ported from https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/dotnet/AutomaticEncodingRestorer.cs.
+    /// A program can change the encoding of the console which would affect other programs.
+    /// We would prefer to have a pattern where the program does not affect encoding of other programs.
+    /// Create this class in a function akin to Main and let it manage the console encoding resources to return it to the state before execution upon destruction.
+    /// </summary>
+    public class AutomaticEncodingRestorer : IDisposable
+    {
+        private Encoding? _originalOutputEncoding = null;
+        private Encoding? _originalInputEncoding = null;
+
+        public AutomaticEncodingRestorer()
+        {
+            try
+            {
+#if NET7_0_OR_GREATER
+                if (OperatingSystem.IsIOS() || OperatingSystem.IsAndroid() || OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs, and they're only available past net 5.
+                {
+                    return;
+                }
+#endif
+                _originalOutputEncoding = Console.OutputEncoding;
+
+#if NET7_0_OR_GREATER
+                if (OperatingSystem.IsBrowser()) // Input Encoding is also unavailable in this platform. (No concern for net472 as browser is unavailable.)
+                {
+                    return;
+                }
+#endif
+                _originalInputEncoding = Console.InputEncoding;
+            }
+            catch (Exception ex) when (ex is IOException || ex is SecurityException)
+            {
+                // The encoding is unavailable. Do nothing.
+            }
+        }
+
+        public void Dispose()
+        {
+            try
+            {
+                if (_originalOutputEncoding != null)
+                {
+                    Console.OutputEncoding = _originalOutputEncoding;
+                }
+                if (_originalInputEncoding != null)
+                {
+                    Console.InputEncoding = _originalInputEncoding;
+                }
+            }
+            catch (Exception ex) when (ex is IOException || ex is SecurityException)
+            {
+                // The encoding is unavailable. Do nothing.
+            }
+        }
+    }
+}

src/MSBuild/MSBuild.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <Import Project="..\Shared\FileSystemSources.proj" />
   <Import Project="..\Shared\DebuggingSources.proj" />
@@ -48,7 +48,8 @@
     <NoPackageAnalysis>true</NoPackageAnalysis>
     <AddAppConfigToBuildOutputs>false</AddAppConfigToBuildOutputs>
 
-    <DebugType Condition="'$(Platform)' == 'x64'">full</DebugType><!-- Setting DebugType here goes hand in hand with eng\AfterSigning.targets. This is to prompt the x64 build to produce a 'full' .pdb that's `more compatible` then 'portable' and 'embedded' .pdbs. This doesn't get set on 32 bit architecture, which will default to 'embedded' and 'pdb2pdb' will convert those as needed. See https://github.com/dotnet/msbuild/pull/5070 for context. -->
+    <!-- Setting DebugType here goes hand in hand with eng\AfterSigning.targets. This is to prompt the x64 build to produce a 'full' .pdb that's `more compatible` then 'portable' and 'embedded' .pdbs. This doesn't get set on 32 bit architecture, which will default to 'embedded' and 'pdb2pdb' will convert those as needed. See https://github.com/dotnet/msbuild/pull/5070 for context. -->
+    <DebugType Condition="'$(Platform)' == 'x64'">full</DebugType>
     <DefineConstants>$(DefineConstants);MSBUILDENTRYPOINTEXE</DefineConstants>
   </PropertyGroup>
 
@@ -163,6 +164,7 @@
     <Compile Include="AssemblyResources.cs">
       <ExcludeFromStyleCop>true</ExcludeFromStyleCop>
     </Compile>
+    <Compile Include="AutomaticEncodingRestorer.cs" />
     <Compile Include="CommandLineSwitches.cs">
       <ExcludeFromStyleCop>true</ExcludeFromStyleCop>
     </Compile>
@@ -218,13 +220,13 @@
     <PackageReference Include="Microsoft.IO.Redist" Condition="'$(FeatureMSIORedist)' == 'true'" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
   </ItemGroup>
-    <!-- Manually download this library for RoslynCodeTaskFactory.
+  <!-- Manually download this library for RoslynCodeTaskFactory.
            See target AddRefAssemblies below. -->
-    <ItemGroup>
-      <PackageDownload Include="netstandard.library">
-        <Version>[2.0.3]</Version>
-      </PackageDownload>
-    </ItemGroup>
+  <ItemGroup>
+    <PackageDownload Include="netstandard.library">
+      <Version>[2.0.3]</Version>
+    </PackageDownload>
+  </ItemGroup>
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">
     <!-- File for Assemblies we depend on -->
     <Reference Include="System" />