Fix InputLanguage.FromCulture() for languages without LANGID value

Many supported Windows languages do not have a unique pre-assigned LANGID to
identify them and instead use a value 0x1000 (LOCALE_CUSTOM_UNSPECIFIED) since
Windows 8.

Windows tries to allocate and provide a unique transient value (from a pool four
values: 0x2000, 0x2400, 0x2800, 0x2C00) in these cases in the HKL lowerword to
distinguish between different languages via the old Win32 APIs, but these
values can't be used to identify the keyboard layout directly, and in result we
can't get the InputLanguage correctly in the FromCulture method.

Aquire corresponging IETF BCP 47 language tag with a call to a proper API
and compare locale names instead of LANGID to fix these corner cases.

It turns out that the LCIDToLocaleName API, which is used inside CultureInfo, may
return incorrect language tags for transient language identifiers. For example, it
returns "nqo-GN" and "jv-Java-ID"  instead of the "nqo" and "jv-Java" (as seen in
the Get-WinUserLanguageList PowerShell cmdlet).

I had to use undocumented registry keys to to extract proper language tag from a
LANGID. This workaround was approved by a Windows team.
#8573 (comment)

src/System.Windows.Forms.Primitives/src/NativeMethods.txt

@@ -415,6 +415,10 @@ LoadRegTypeLib
 LoadTypeLib
 LOCALE_IMEASURE
 LOCALE_NAME_SYSTEM_DEFAULT
+LOCALE_TRANSIENT_KEYBOARD1
+LOCALE_TRANSIENT_KEYBOARD2
+LOCALE_TRANSIENT_KEYBOARD3
+LOCALE_TRANSIENT_KEYBOARD4
 LOGPALETTE
 LOGPEN
 LPtoDP

src/System.Windows.Forms/src/System/Windows/Forms/InputLanguage.cs

@@ -27,7 +27,7 @@ internal InputLanguage(IntPtr handle)
  /// <summary>
  /// Returns the culture of the current input language.
  /// </summary>
- public CultureInfo Culture => new CultureInfo(PARAM.LOWORD(_handle));
+ public CultureInfo Culture => new CultureInfo(LanguageTag);
 
  /// <summary>
  /// Gets or sets the input language for the current thread.
@@ -94,7 +94,7 @@ public static unsafe InputLanguageCollection InstalledInputLanguages
  }
  }
 
- private static string s_keyboardLayoutsRegistryPath => @"SYSTEM\CurrentControlSet\Control\Keyboard Layouts";
+ private const string KeyboardLayoutsRegistryPath = @"SYSTEM\CurrentControlSet\Control\Keyboard Layouts";
 
  /// <summary>
  /// Returns the name of the current keyboard layout as it appears in the Windows
@@ -105,7 +105,7 @@ public string LayoutName
  get
  {
  // https://learn.microsoft.com/windows/win32/intl/using-registry-string-redirection#create-resources-for-keyboard-layout-strings
- using RegistryKey? key = Registry.LocalMachine.OpenSubKey($@"{s_keyboardLayoutsRegistryPath}\{LayoutId}");
+ using RegistryKey? key = Registry.LocalMachine.OpenSubKey($@"{KeyboardLayoutsRegistryPath}\{LayoutId}");
  return key.GetMUIString("Layout Display Name", "Layout Text") ?? SR.UnknownInputLanguageLayout;
  }
  }
@@ -139,24 +139,16 @@ internal string LayoutId
  // Extract special layout id from the device handle
  int layoutId = device & 0x0FFF;
 
- using RegistryKey? key = Registry.LocalMachine.OpenSubKey(s_keyboardLayoutsRegistryPath);
+ using RegistryKey? key = Registry.LocalMachine.OpenSubKey(KeyboardLayoutsRegistryPath);
  if (key is not null)
  {
  // Match keyboard layout by layout id
  foreach (string subKeyName in key.GetSubKeyNames())
  {
  using RegistryKey? subKey = key.OpenSubKey(subKeyName);
- if (subKey is null)
- {
- continue;
- }
-
- if (subKey.GetValue("Layout Id") is not string subKeyLayoutId)
- {
- continue;
- }
-
- if (layoutId == Convert.ToInt32(subKeyLayoutId, 16))
+ if (subKey is not null
+ && subKey.GetValue("Layout Id") is string subKeyLayoutId
+ && Convert.ToInt32(subKeyLayoutId, 16) == layoutId)
  {
  Debug.Assert(subKeyName.Length == 8, $"unexpected key length in registry: {subKey.Name}");
  return subKeyName.ToUpperInvariant();
@@ -180,6 +172,57 @@ internal string LayoutId
  }
  }
 
+ private const string UserProfileRegistryPath = @"Control Panel\International\User Profile";
+
+ /// <summary>
+ /// Returns the <see href="https://learn.microsoft.com/globalization/locale/standard-locale-names">BCP 47 language
+ /// tag</see> of the current input language.
+ /// </summary>
+ private string LanguageTag
+ {
+ get
+ {
+ // According to the GetKeyboardLayout API function docs low word of HKL contains input language identifier.
+ int langId = PARAM.LOWORD(_handle);
+
+ // We need to convert the language identifier to a language tag, because they are deprecated and may have a
+ // transient value.
+ // https://learn.microsoft.com/globalization/locale/other-locale-names#lcid
+ // https://learn.microsoft.com/windows/win32/winmsg/wm-inputlangchange#remarks
+ //
+ // It turns out that the LCIDToLocaleName API, which is used inside CultureInfo, may return incorrect
+ // language tags for transient language identifiers. For example, it returns "nqo-GN" and "jv-Java-ID"
+ // instead of the "nqo" and "jv-Java" (as seen in the Get-WinUserLanguageList PowerShell cmdlet).
+ //
+ // Try to extract proper language tag from registry as a workaround approved by a Windows team.
+ // https://github.com/dotnet/winforms/pull/8573#issuecomment-1542600949
+ //
+ // NOTE: this logic may break in future versions of Windows since it is not documented.
+ if (langId == PInvoke.LOCALE_TRANSIENT_KEYBOARD1 ||
+ langId == PInvoke.LOCALE_TRANSIENT_KEYBOARD2 ||
+ langId == PInvoke.LOCALE_TRANSIENT_KEYBOARD3 ||
+ langId == PInvoke.LOCALE_TRANSIENT_KEYBOARD4)
+ {
+ using RegistryKey? key = Registry.CurrentUser.OpenSubKey(UserProfileRegistryPath);
+ if (key is not null && key.GetValue("Languages") is string[] languages)
+ {
+ foreach (string language in languages)
+ {
+ using RegistryKey? subKey = key.OpenSubKey(language);
+ if (subKey is not null
+ && subKey.GetValue("TransientLangId") is int transientLangId
+ && transientLangId == langId)
+ {
+ return language;
+ }
+ }
+ }
+ }
+
+ return CultureInfo.GetCultureInfo(langId).Name;
+ }
+ }
+
  /// <summary>
  /// Creates an InputLanguageChangedEventArgs given a windows message.
  /// </summary>
@@ -213,13 +256,9 @@ public override bool Equals(object? value)
  {
  ArgumentNullException.ThrowIfNull(culture);
 
- // KeyboardLayoutId is the LCID for built-in cultures, but it is the CU-preferred keyboard language for
- // custom cultures.
- int lcid = culture.KeyboardLayoutId;
-
  foreach (InputLanguage? lang in InstalledInputLanguages)
  {
- if ((unchecked((int)(long)lang!._handle) & 0xFFFF) == lcid)
+ if (culture.Equals(lang?.Culture))
  {
  return lang;
  }

src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/InputLanguageChangedEventArgsTests.cs

@@ -35,7 +35,6 @@ public static IEnumerable<object[]> Ctor_NoSuchCultureInfo_TestData()
  {
  yield return new object[] { CultureInfo.InvariantCulture };
  yield return new object[] { new CultureInfo("en") };
- yield return new object[] { new UnknownKeyboardCultureInfo() };
  }
 
  [Theory]
@@ -72,13 +71,4 @@ public void Ctor_NullInputLanguage_ThrowsNullReferenceException()
  {
  Assert.Throws<ArgumentNullException>("inputLanguage", () => new InputLanguageChangedEventArgs((InputLanguage)null, 0));
  }
-
- private class UnknownKeyboardCultureInfo : CultureInfo
- {
- public UnknownKeyboardCultureInfo() : base("en-US")
- {
- }
-
- public override int KeyboardLayoutId => int.MaxValue;
- }
 }

src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/InputLanguageChangingEventArgsTests.cs

@@ -36,7 +36,6 @@ public static IEnumerable<object[]> Ctor_NoSuchCultureInfo_TestData()
  {
  yield return new object[] { CultureInfo.InvariantCulture };
  yield return new object[] { new CultureInfo("en") };
- yield return new object[] { new UnknownKeyboardCultureInfo() };
  }
 
  [Theory]
@@ -73,13 +72,4 @@ public void Ctor_NullInputLanguage_ThrowsNullReferenceException()
  {
  Assert.Throws<ArgumentNullException>("inputLanguage", () => new InputLanguageChangingEventArgs((InputLanguage)null, true));
  }
-
- private class UnknownKeyboardCultureInfo : CultureInfo
- {
- public UnknownKeyboardCultureInfo() : base("en-US")
- {
- }
-
- public override int KeyboardLayoutId => int.MaxValue;
- }
 }

src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/InputLanguageTests.cs

@@ -95,8 +95,8 @@ public void InputLanguage_FromCulture_Roundtrip_Success()
  [Fact]
  public void InputLanguage_FromCulture_NoSuchCulture_ReturnsNull()
  {
- var unknownCulture = new UnknownKeyboardCultureInfo();
- Assert.Null(InputLanguage.FromCulture(unknownCulture));
+ var invariantCulture = CultureInfo.InvariantCulture;
+ Assert.Null(InputLanguage.FromCulture(invariantCulture));
  }
 
  [Fact]
@@ -126,19 +126,44 @@ public static IEnumerable<object[]> InputLanguageLayoutId_TestData()
  [MemberData(nameof(InputLanguageLayoutId_TestData))]
  public void InputLanguage_InputLanguageLayoutId_Expected(int langId, int device, string languageTag, string layoutId, string layoutName)
  {
- var language = new InputLanguage(PARAM.FromLowHigh(langId, device));
- Assert.Equal(languageTag, language.Culture.Name);
- Assert.Equal(layoutId, language.LayoutId);
- Assert.Equal(layoutName, language.LayoutName);
- VerifyInputLanguage(language);
+ InputLanguage language = new(PARAM.FromLowHigh(langId, device));
+ VerifyInputLanguage(language, languageTag, layoutId, layoutName);
+ }
+
+ public static IEnumerable<object[]> SupplementalInputLanguages_TestData()
+ {
+ yield return new object[] { "got-Goth", "000C0C00", "Gothic" };
+ yield return new object[] { "jv-Java", "00110C00", "Javanese" };
+ yield return new object[] { "nqo", "00090C00", "N’Ko" };
+ yield return new object[] { "zgh-Tfng", "0000105F", "Tifinagh (Basic)" };
+ }
+
+ [Theory]
+ [MemberData(nameof(SupplementalInputLanguages_TestData))]
+ public void InputLanguage_FromCulture_SupplementalInputLanguages_Expected(string languageTag, string layoutId, string layoutName)
+ {
+ // Also installs default keyboard layout for this language
+ // https://learn.microsoft.com/windows-hardware/manufacture/desktop/default-input-locales-for-windows-language-packs
+ InstallUserLanguage(languageTag);
+
+ try
+ {
+ CultureInfo culture = new(languageTag);
+ InputLanguage language = InputLanguage.FromCulture(culture);
+ VerifyInputLanguage(language, languageTag, layoutId, layoutName);
+ }
+ finally
+ {
+ UninstallUserLanguage(languageTag);
+ }
  }
 
  [Theory]
  [InlineData(0x0000, 0x0409)]
  [InlineData(0xffff, 0x0409)]
  public void InputLanguage_Culture_ThrowsArgumentException(int langId, int device)
  {
- var language = new InputLanguage(PARAM.FromLowHigh(langId, device));
+ InputLanguage language = new(PARAM.FromLowHigh(langId, device));
  Assert.ThrowsAny<ArgumentException>(() => language.Culture);
  }
 
@@ -147,10 +172,19 @@ public void InputLanguage_Culture_ThrowsArgumentException(int langId, int device
  [InlineData(0x0409, 0xffff)]
  public void InputLanguage_LayoutName_UnknownExpected(int langId, int device)
  {
- var language = new InputLanguage(PARAM.FromLowHigh(langId, device));
+ InputLanguage language = new(PARAM.FromLowHigh(langId, device));
  Assert.Equal(SR.UnknownInputLanguageLayout, language.LayoutName);
  }
 
+ private static void VerifyInputLanguage(InputLanguage language, string languageTag, string layoutId, string layoutName)
+ {
+ Assert.NotNull(language);
+ Assert.NotEqual(IntPtr.Zero, language.Handle);
+ Assert.Equal(languageTag, language.Culture.Name);
+ Assert.Equal(layoutId, language.LayoutId);
+ Assert.Equal(layoutName, language.LayoutName);
+ }
+
  private static void VerifyInputLanguage(InputLanguage language)
  {
  Assert.NotEqual(IntPtr.Zero, language.Handle);
@@ -161,12 +195,41 @@ private static void VerifyInputLanguage(InputLanguage language)
  Assert.DoesNotContain('\0', language.LayoutName);
  }
 
- private class UnknownKeyboardCultureInfo : CultureInfo
+ private static void RunPowerShellScript(string path)
  {
- public UnknownKeyboardCultureInfo() : base("en-US")
- {
- }
+ using Process process = new();
+
+ process.StartInfo.FileName = "powershell.exe";
+ process.StartInfo.Arguments = $"-NoProfile -ExecutionPolicy ByPass -File \"{path}\"";
+
+ process.Start();
+ process.WaitForExit();
+ }
+
+ private static void InstallUserLanguage(string languageTag)
+ {
+ string file = Path.Combine(Path.GetTempPath(), $"install-language-{languageTag}.ps1");
+ string script = $$"""
+ $list = Get-WinUserLanguageList
+ $list.Add("{{languageTag}}")
+ Set-WinUserLanguageList $list -force
+ """;
+
+ using TempFile tempFile = new(file, script);
+ RunPowerShellScript(tempFile.Path);
+ }
+
+ private static void UninstallUserLanguage(string languageTag)
+ {
+ string file = Path.Combine(Path.GetTempPath(), $"uninstall-language-{languageTag}.ps1");
+ string script = $$"""
+ $list = Get-WinUserLanguageList
+ $item = $list | Where-Object {$_.LanguageTag -like "{{languageTag}}"}
+ $list.Remove($item)
+ Set-WinUserLanguageList $list -force
+ """;
 
- public override int KeyboardLayoutId => int.MaxValue;
+ using TempFile tempFile = new(file, script);
+ RunPowerShellScript(tempFile.Path);
  }
 }