Detect .NET SDK & VSLANG Custom Language Settings & Apply To MSBuild

src/MSBuild.UnitTests/XMake_Tests.cs

@@ -9,6 +9,7 @@
 using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
+using System.Runtime.InteropServices;
 using System.Threading;
 using Microsoft.Build.CommandLine;
 using Microsoft.Build.Framework;
@@ -642,6 +643,49 @@ public void SetConsoleUICulture()
             thisThread.CurrentUICulture = originalUICulture;
         }
 
+
+        [Fact]
+        public void ConsoleUIRespectsSDKLanguage()
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !MSBuildApp.CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding())
+            {
+                return; // The feature to detect .NET SDK Languages is not enabled on this machine, so don't test it.
+            }
+
+            const string DOTNET_CLI_UI_LANGUAGE = nameof(DOTNET_CLI_UI_LANGUAGE);
+            using TestEnvironment testEnvironment = TestEnvironment.Create();
+            // Save the current environment info so it can be restored.
+            var originalUILanguage = Environment.GetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE);
+
+            var originalOutputEncoding = Console.OutputEncoding;
+            var originalInputEncoding = Console.InputEncoding;
+            Thread thisThread = Thread.CurrentThread;
+            CultureInfo originalUICulture = thisThread.CurrentUICulture;
+
+            try
+            {
+                // Set the UI language based on the SDK environment var.
+                testEnvironment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, "ja"); // japanese chose arbitrarily.
+
+                MSBuildApp.SetConsoleUI();
+
+                Assert.Equal(new CultureInfo("ja"), thisThread.CurrentUICulture);
+                Assert.Equal(65001, Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering.
+            }
+            finally
+            {
+                // Restore the current UI culture back to the way it was at the beginning of this unit test.
+                thisThread.CurrentUICulture = originalUICulture;
+                // Restore for full framework
+                CultureInfo.CurrentCulture = originalUICulture;
+                CultureInfo.DefaultThreadCurrentUICulture = originalUICulture;
+
+                // MSbuild should also restore the encoding upon exit, but we don't create that context here.
+                Console.OutputEncoding = originalOutputEncoding;
+                Console.InputEncoding = originalInputEncoding;
+            }
+        }
+
 #if FEATURE_SYSTEM_CONFIGURATION
         /// <summary>
         /// Invalid configuration file should not dump stack.
@@ -798,6 +842,10 @@ public void TestEnvironmentTest()
         [Fact]
         public void MSBuildEngineLogger()
         {
+            using TestEnvironment testEnvironment = TestEnvironment.Create();
+            testEnvironment.SetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE", "en"); // build machines may have other values.
+            CultureInfo.CurrentUICulture = new CultureInfo("en"); // Validate that the thread will produce an english log regardless of the machine OS language
+
             string oldValueForMSBuildLoadMicrosoftTargetsReadOnly = Environment.GetEnvironmentVariable("MSBuildLoadMicrosoftTargetsReadOnly");
             string projectString =
                    "<?xml version=\"1.0\" encoding=\"utf-8\"?>" +
@@ -834,6 +882,8 @@ public void MSBuildEngineLogger()
 
                 var logFileContents = File.ReadAllText(logFile);
 
+                Assert.Equal(new CultureInfo("en"), Thread.CurrentThread.CurrentUICulture);
+
                 logFileContents.ShouldContain("Process = ");
                 logFileContents.ShouldContain("MSBuild executable path = ");
                 logFileContents.ShouldContain("Command line arguments = ");

src/MSBuild/AutomaticEncodingRestorer.cs

@@ -0,0 +1,68 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Security;
+using System.Text;
+
+namespace Microsoft.Build.CommandLine
+{
+    /// <summary>
+    /// Ported from https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/dotnet/AutomaticEncodingRestorer.cs.
+    /// A program can change the encoding of the console which would affect other programs.
+    /// We would prefer to have a pattern where the program does not affect encoding of other programs.
+    /// Create this class in a function akin to Main and let it manage the console encoding resources to return it to the state before execution upon destruction.
+    /// </summary>
+    public class AutomaticEncodingRestorer : IDisposable
+    {
+        private Encoding? _originalOutputEncoding = null;
+        private Encoding? _originalInputEncoding = null;
+
+        public AutomaticEncodingRestorer()
+        {
+            try
+            {
+#if NET7_0_OR_GREATER
+                if (OperatingSystem.IsIOS() || OperatingSystem.IsAndroid() || OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs, and they're only available past net 5.
+                {
+                    return;
+                }
+#endif
+                _originalOutputEncoding = Console.OutputEncoding;
+
+#if NET7_0_OR_GREATER
+                if (OperatingSystem.IsBrowser()) // Input Encoding is also unavailable in this platform. (No concern for net472 as browser is unavailable.)
+                {
+                    return;
+                }
+#endif
+                _originalInputEncoding = Console.InputEncoding;
+            }
+            catch (Exception ex) when (ex is IOException || ex is SecurityException)
+            {
+                // The encoding is unavailable. Do nothing.
+            }
+        }
+
+        public void Dispose()
+        {
+            try
+            {
+                if (_originalOutputEncoding != null)
+                {
+                    Console.OutputEncoding = _originalOutputEncoding;
+                }
+                if (_originalInputEncoding != null)
+                {
+                    Console.InputEncoding = _originalInputEncoding;
+                }
+            }
+            catch (Exception ex) when (ex is IOException || ex is SecurityException)
+            {
+                // The encoding is unavailable. Do nothing.
+            }
+        }
+    }
+}

src/MSBuild/MSBuild.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <Import Project="..\Shared\FileSystemSources.proj" />
   <Import Project="..\Shared\DebuggingSources.proj" />
@@ -48,7 +48,8 @@
     <NoPackageAnalysis>true</NoPackageAnalysis>
     <AddAppConfigToBuildOutputs>false</AddAppConfigToBuildOutputs>
 
-    <DebugType Condition="'$(Platform)' == 'x64'">full</DebugType><!-- Setting DebugType here goes hand in hand with eng\AfterSigning.targets. This is to prompt the x64 build to produce a 'full' .pdb that's `more compatible` then 'portable' and 'embedded' .pdbs. This doesn't get set on 32 bit architecture, which will default to 'embedded' and 'pdb2pdb' will convert those as needed. See https://github.com/dotnet/msbuild/pull/5070 for context. -->
+    <!-- Setting DebugType here goes hand in hand with eng\AfterSigning.targets. This is to prompt the x64 build to produce a 'full' .pdb that's `more compatible` then 'portable' and 'embedded' .pdbs. This doesn't get set on 32 bit architecture, which will default to 'embedded' and 'pdb2pdb' will convert those as needed. See https://github.com/dotnet/msbuild/pull/5070 for context. -->
+    <DebugType Condition="'$(Platform)' == 'x64'">full</DebugType>
     <DefineConstants>$(DefineConstants);MSBUILDENTRYPOINTEXE</DefineConstants>
   </PropertyGroup>
 
@@ -162,6 +163,7 @@
     <Compile Include="AssemblyResources.cs">
       <ExcludeFromStyleCop>true</ExcludeFromStyleCop>
     </Compile>
+    <Compile Include="AutomaticEncodingRestorer.cs" />
     <Compile Include="CommandLineSwitches.cs">
       <ExcludeFromStyleCop>true</ExcludeFromStyleCop>
     </Compile>
@@ -217,13 +219,13 @@
     <PackageReference Include="Microsoft.IO.Redist" Condition="'$(FeatureMSIORedist)' == 'true'" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
   </ItemGroup>
-    <!-- Manually download this library for RoslynCodeTaskFactory.
+  <!-- Manually download this library for RoslynCodeTaskFactory.
            See target AddRefAssemblies below. -->
-    <ItemGroup>
-      <PackageDownload Include="netstandard.library">
-        <Version>[2.0.3]</Version>
-      </PackageDownload>
-    </ItemGroup>
+  <ItemGroup>
+    <PackageDownload Include="netstandard.library">
+      <Version>[2.0.3]</Version>
+    </PackageDownload>
+  </ItemGroup>
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">
     <!-- File for Assemblies we depend on -->
     <Reference Include="System" />

src/MSBuild/XMake.cs

@@ -40,6 +40,7 @@
 using Microsoft.Build.Internal;
 using Microsoft.Build.Logging.LiveLogger;
 using System.Runtime.InteropServices;
+using Microsoft.Win32;
 
 #nullable disable
 
@@ -233,6 +234,10 @@ string[] args
                 DumpCounters(true /* initialize only */);
             }
 
+            // Setup the console UI.
+            using AutomaticEncodingRestorer _ = new();
+            SetConsoleUI();
+
             int exitCode;
             if (
                 ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_4) &&
@@ -664,9 +669,6 @@ public static ExitType Execute(
                 // check the operating system the code is running on
                 VerifyThrowSupportedOS();
 
-                // Setup the console UI.
-                SetConsoleUI();
-
                 // reset the application state for this new build
                 ResetBuildState();
 
@@ -1635,14 +1637,20 @@ internal static void SetConsoleUI()
             Thread thisThread = Thread.CurrentThread;
 
             // Eliminate the complex script cultures from the language selection.
-            thisThread.CurrentUICulture = CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture();
+            var desiredCulture = GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture();
+            thisThread.CurrentUICulture = desiredCulture;
+
+            // For full framework, both the above and below must be set. This is not true in core, but it is a no op in core.
+            // https://learn.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.defaultthreadcurrentculture?redirectedfrom=MSDN&view=net-7.0#remarks
+            CultureInfo.CurrentUICulture = desiredCulture;
+            CultureInfo.DefaultThreadCurrentUICulture = desiredCulture;
 
             // Determine if the language can be displayed in the current console codepage, otherwise set to US English
             int codepage;
 
             try
             {
-                codepage = System.Console.OutputEncoding.CodePage;
+                codepage = Console.OutputEncoding.CodePage;
             }
             catch (NotSupportedException)
             {
@@ -1669,6 +1677,89 @@ internal static void SetConsoleUI()
 #endif
         }
 
+        /// <summary>
+        /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect the SDK variable.
+        /// To use the correspoding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases.
+        /// This code introduces a breaking change due to the encoding of the console being changed.
+        /// If the environment variables are undefined, this function should be a no-op.
+        /// </summary>
+        /// <returns>
+        /// The custom language that was set by the user for an 'external' tool besides MSBuild.
+        /// Returns <see langword="null"/> if none are set.
+        /// </returns>
+        private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding()
+        {
+            CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage();
+            if (externalLanguageSetting != null)
+            {
+                if (
+                    !externalLanguageSetting.TwoLetterISOLanguageName.Equals("en", StringComparison.InvariantCultureIgnoreCase) &&
+                    CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()
+                    )
+                {
+                    // Setting both encodings causes a change in the CHCP, making it so we dont need to P-Invoke ourselves.
+                    Console.OutputEncoding = Encoding.UTF8;
+                    // If the InputEncoding is not set, the encoding will work in CMD but not in Powershell, as the raw CHCP page won't be changed.
+                    Console.InputEncoding = Encoding.UTF8;
+                    return externalLanguageSetting;
+                }
+                else if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                {
+                    return externalLanguageSetting;
+                }
+            }
+            return null;
+        }
+
+        public static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version.Major >= 10) // UTF-8 is only officially supported on 10+.
+            {
+                try
+                {
+                    using RegistryKey windowsVersionRegistry = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion");
+                    var buildNumber = windowsVersionRegistry.GetValue("CurrentBuildNumber").ToString();
+                    const int buildNumberThatOfficialySupportsUTF8 = 18363;
+                    return int.Parse(buildNumber) >= buildNumberThatOfficialySupportsUTF8 || ForceUniversalEncodingOptInEnabled();
+                }
+                catch (Exception ex) when (ex is SecurityException || ex is ObjectDisposedException)
+                {
+                    // We don't want to break those in VS on older versions of Windows with a non-en language.
+                    // Allow those without registry permissions to force the encoding, however.
+                    return ForceUniversalEncodingOptInEnabled();
+                }
+            }
+            return false;
+        }
+
+        private static bool ForceUniversalEncodingOptInEnabled()
+        {
+            return string.Equals(Environment.GetEnvironmentVariable("DOTNET_CLI_FORCE_UTF8_ENCODING"), "true", StringComparison.OrdinalIgnoreCase);
+        }
+
+        /// <summary>
+        /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE.)
+        /// Does NOT check System Locale or OS Display Language.
+        /// Ported from the .NET SDK: https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs
+        /// </summary>
+        /// <returns>The custom language that was set by the user for an 'external' tool besides MSBuild.
+        /// Returns null if none are set.</returns>
+        private static CultureInfo GetExternalOverriddenUILanguage()
+        {
+            // DOTNET_CLI_UI_LANGUAGE=<culture name> is the main way for users to customize the CLI's UI language via the .NET SDK.
+            string dotnetCliLanguage = Environment.GetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE");
+            if (dotnetCliLanguage != null)
+            {
+                try
+                {
+                    return new CultureInfo(dotnetCliLanguage);
+                }
+                catch (CultureNotFoundException) { }
+            }
+
+            return null;
+        }
+
         /// <summary>
         /// Gets all specified switches, from the command line, as well as all
         /// response files, including the auto-response file.