Stop accepting underscores as subtag separators

components/experimental/src/personnames/specifications/derive_locale.rs

@@ -112,12 +112,12 @@ mod tests {
         let mut locale = locale!("fr");
         lc.maximize(&mut locale.id);
         assert_eq!(
-            effective_locale(&locale!("de_Latn_ch"), &locale),
-            Ok(&locale!("de_Latn_ch"))
+            effective_locale(&locale!("de-Latn-ch"), &locale),
+            Ok(&locale!("de-Latn-ch"))
         );
         assert_eq!(
-            effective_locale(&locale, &locale!("de_Latn_ch")),
-            Ok(&locale!("fr_Latn_FR"))
+            effective_locale(&locale, &locale!("de-Latn-ch")),
+            Ok(&locale!("fr-Latn-FR"))
         );
     }
 
@@ -127,11 +127,11 @@ mod tests {
         let mut locale = locale!("ja");
         lc.maximize(&mut locale.id);
         assert_eq!(
-            effective_locale(&locale!("de_Latn_ch"), &locale),
+            effective_locale(&locale!("de-Latn-ch"), &locale),
             Ok(&locale!("ja-Jpan-JP"))
         );
         assert_eq!(
-            effective_locale(&locale, &locale!("de_Latn_ch")),
+            effective_locale(&locale, &locale!("de-Latn-ch")),
             Ok(&locale!("de-Latn-CH"))
         );
     }
@@ -142,27 +142,27 @@ mod tests {
         let mut locale = locale!("ja");
         lc.maximize(&mut locale.id);
         assert_eq!(
-            effective_locale(&locale!("ja_Hani_JP"), &locale),
-            Ok(&locale!("ja_Hani_JP"))
+            effective_locale(&locale!("ja-Hani-JP"), &locale),
+            Ok(&locale!("ja-Hani-JP"))
         );
         assert_eq!(
-            effective_locale(&locale!("ja_Kana_JP"), &locale),
+            effective_locale(&locale!("ja-Kana-JP"), &locale),
             Ok(&locale!("ja-Kana-JP"))
         );
         assert_eq!(
-            effective_locale(&locale!("ja_Hira_JP"), &locale),
+            effective_locale(&locale!("ja-Hira-JP"), &locale),
             Ok(&locale!("ja-Hira-JP"))
         );
         assert_eq!(
-            effective_locale(&locale, &locale!("ja_Hani_JP")),
+            effective_locale(&locale, &locale!("ja-Hani-JP")),
             Ok(&locale!("ja-Jpan-JP"))
         );
         assert_eq!(
-            effective_locale(&locale, &locale!("ja_Kana_JP")),
+            effective_locale(&locale, &locale!("ja-Kana-JP")),
             Ok(&locale!("ja-Jpan-JP"))
         );
         assert_eq!(
-            effective_locale(&locale, &locale!("ja_Hira_JP")),
+            effective_locale(&locale, &locale!("ja-Hira-JP")),
             Ok(&locale!("ja-Jpan-JP"))
         );
     }
@@ -173,23 +173,23 @@ mod tests {
         let scripts = icu_properties::PropertyNamesShort::<icu_properties::props::Script>::new();
         assert_eq!(
             likely_person_name_locale(&person_name("Miyazaki", "Hayao").unwrap(), swe, scripts),
-            Ok(locale!("und_Latn"))
+            Ok(locale!("und-Latn"))
         );
         assert_eq!(
             likely_person_name_locale(&person_name("駿", "宮崎").unwrap(), swe, scripts),
-            Ok(locale!("und_Hani"))
+            Ok(locale!("und-Hani"))
         );
         assert_eq!(
             likely_person_name_locale(&person_name("하야오", "미야자키").unwrap(), swe, scripts),
-            Ok(locale!("und_Hang"))
+            Ok(locale!("und-Hang"))
         );
         assert_eq!(
             likely_person_name_locale(
                 &person_name("アルベルト", "アインシュタイン").unwrap(),
                 swe,
                 scripts
             ),
-            Ok(locale!("und_Kana"))
+            Ok(locale!("und-Kana"))
         );
     }
 

components/experimental/src/personnames/specifications/derive_name_order.rs

@@ -69,7 +69,7 @@ mod tests {
         // Match "und"
         assert_eq!(
             name_order_derive(
-                &locale!("de_Latn_ch"),
+                &locale!("de-Latn-ch"),
                 &surname_first,
                 &given_first,
                 fallbacker
@@ -84,7 +84,7 @@ mod tests {
         // since "und" is a catch all set in given first, it is a perfect match.
         assert_eq!(
             name_order_derive(
-                &locale!("ja_Jpan_jp"),
+                &locale!("ja-Jpan-jp"),
                 &surname_first,
                 &given_first,
                 fallbacker
@@ -104,7 +104,7 @@ mod tests {
 
         assert_eq!(
             name_order_derive(
-                &locale!("en_Latn_SG"),
+                &locale!("en-Latn-SG"),
                 &surname_first,
                 &given_first,
                 fallbacker
@@ -116,7 +116,7 @@ mod tests {
         // This is not matching because of zh, but because of und-CN
         assert_eq!(
             name_order_derive(
-                &locale!("zh_Hans_CN"),
+                &locale!("zh-Hans-CN"),
                 &surname_first,
                 &given_first,
                 fallbacker
@@ -128,7 +128,7 @@ mod tests {
         // This is not matching because of zh, but because of und-CN
         assert_eq!(
             name_order_derive(
-                &locale!("zh_Hans"),
+                &locale!("zh-Hans"),
                 &surname_first,
                 &given_first,
                 fallbacker

components/experimental/tests/displaynames/tests.rs

@@ -22,7 +22,7 @@ fn test_concatenate() {
             should_borrow: true,
         },
         TestCase {
-            input_1: &locale!("zh_Hans"),
+            input_1: &locale!("zh-Hans"),
             expected: "Simplified Chinese",
             should_borrow: true,
         },

components/experimental/tests/personnames/tests.rs

@@ -233,14 +233,14 @@ fn test_space_replacement_spec_formatting_locale_ja() -> Result<(), PersonNamesF
 
     let person_name = DefaultPersonName::new(
         person_data,
-        Some(locale!("de_Latn_CH")),
+        Some(locale!("de-Latn-CH")),
         Some(PreferredOrder::GivenFirst),
     )?;
 
     let formatter = PersonNamesFormatter::try_new_unstable(
         &TestingProvider,
         PersonNamesFormatterOptions::new(
-            locale!("ja_JP"),
+            locale!("ja-JP"),
             FormattingOrder::GivenFirst,
             FormattingLength::Medium,
             FormattingUsage::Referring,
@@ -276,14 +276,14 @@ fn test_space_replacement_spec_formatting_locale_ja_jpan_script(
 
     let person_name = DefaultPersonName::new(
         person_data,
-        Some(locale!("de_Jpan_CH")),
+        Some(locale!("de-Jpan-CH")),
         Some(PreferredOrder::GivenFirst),
     )?;
 
     let formatter = PersonNamesFormatter::try_new_unstable(
         &TestingProvider,
         PersonNamesFormatterOptions::new(
-            locale!("ja_JP"),
+            locale!("ja-JP"),
             FormattingOrder::GivenFirst,
             FormattingLength::Medium,
             FormattingUsage::Referring,
@@ -319,14 +319,14 @@ fn test_space_replacement_spec_formatting_locale_ja_compatible(
 
     let person_name = DefaultPersonName::new(
         person_data,
-        Some(locale!("ja_Jpan_JP")),
+        Some(locale!("ja-Jpan-JP")),
         Some(PreferredOrder::SurnameFirst),
     )?;
 
     let formatter = PersonNamesFormatter::try_new_unstable(
         &TestingProvider,
         PersonNamesFormatterOptions::new(
-            locale!("ja_JP"),
+            locale!("ja-JP"),
             FormattingOrder::GivenFirst,
             FormattingLength::Medium,
             FormattingUsage::Referring,
@@ -362,14 +362,14 @@ fn test_space_replacement_spec_formatting_locale_de_compatible(
 
     let person_name = DefaultPersonName::new(
         person_data,
-        Some(locale!("de_Latn_CH")),
+        Some(locale!("de-Latn-CH")),
         Some(PreferredOrder::GivenFirst),
     )?;
 
     let formatter = PersonNamesFormatter::try_new_unstable(
         &TestingProvider,
         PersonNamesFormatterOptions::new(
-            locale!("de_CH"),
+            locale!("de-CH"),
             FormattingOrder::GivenFirst,
             FormattingLength::Medium,
             FormattingUsage::Referring,
@@ -405,14 +405,14 @@ fn test_space_replacement_spec_formatting_locale_de_jpan_script(
 
     let person_name = DefaultPersonName::new(
         person_data,
-        Some(locale!("de_Jpan_CH")),
+        Some(locale!("de-Jpan-CH")),
         Some(PreferredOrder::GivenFirst),
     )?;
 
     let formatter = PersonNamesFormatter::try_new_unstable(
         &TestingProvider,
         PersonNamesFormatterOptions::new(
-            locale!("de_CH"),
+            locale!("de-CH"),
             FormattingOrder::GivenFirst,
             FormattingLength::Medium,
             FormattingUsage::Referring,
@@ -450,14 +450,14 @@ fn test_space_replacement_spec_formatting_locale_und_latn_jp(
 
     let person_name = DefaultPersonName::new(
         person_data,
-        Some(locale!("und_Latn_JP")),
+        Some(locale!("und-Latn-JP")),
         Some(PreferredOrder::GivenFirst),
     )?;
 
     let formatter = PersonNamesFormatter::try_new_unstable(
         &TestingProvider,
         PersonNamesFormatterOptions::new(
-            locale!("de_CH"),
+            locale!("de-CH"),
             FormattingOrder::GivenFirst,
             FormattingLength::Medium,
             FormattingUsage::Referring,

components/locale/src/expander.rs

@@ -478,12 +478,12 @@ impl LocaleExpander {
     ///
     /// let lc = LocaleExpander::new();
     ///
-    /// let mut locale = locale!("zh_TW");
+    /// let mut locale = locale!("zh-TW");
     /// assert_eq!(
     ///     lc.minimize_favor_script(&mut locale.id),
     ///     TransformResult::Modified
     /// );
-    /// assert_eq!(locale, locale!("zh_Hant"));
+    /// assert_eq!(locale, locale!("zh-Hant"));
     /// ```
     pub fn minimize_favor_script(&self, langid: &mut LanguageIdentifier) -> TransformResult {
         self.minimize_impl(langid, false)

components/locale_core/benches/iai_langid.rs

@@ -63,7 +63,7 @@ fn bench_langid_strict_cmp() {
     // Tests the cost of comparing a langid against byte strings.
     use core::cmp::Ordering;
 
-    let lid = langid!("en_us");
+    let lid = langid!("en-us");
 
     let result = LIDS_STR
         .iter()
@@ -76,7 +76,7 @@ fn bench_langid_strict_cmp() {
 fn bench_langid_matching() {
     // Tests matching a LID against other LIDs.
 
-    let lid = langid!("en_us");
+    let lid = langid!("en-us");
 
     let count = LIDS.iter().filter(|l| lid == **l).count();
     assert_eq!(count, 1);
@@ -85,7 +85,7 @@ fn bench_langid_matching() {
 fn bench_langid_matching_str() {
     // Tests matching a LID against list of str.
 
-    let lid = langid!("en_us");
+    let lid = langid!("en-us");
 
     let count = LIDS_STR.iter().filter(|&l| lid.normalizing_eq(l)).count();
     assert_eq!(count, 1);

components/locale_core/src/langid.rs

@@ -66,7 +66,7 @@ use alloc::borrow::Cow;
 ///     subtags::{language, region, script, variant},
 /// };
 ///
-/// let li = langid!("eN_latn_Us-Valencia");
+/// let li = langid!("eN-latn-Us-Valencia");
 ///
 /// assert_eq!(li.language, language!("en"));
 /// assert_eq!(li.script, Some(script!("Latn")));
@@ -175,7 +175,7 @@ impl LanguageIdentifier {
     /// use icu::locale::LanguageIdentifier;
     ///
     /// assert_eq!(
-    ///     LanguageIdentifier::normalize("pL_latn_pl").as_deref(),
+    ///     LanguageIdentifier::normalize("pL-latn-pl").as_deref(),
     ///     Ok("pl-Latn-PL")
     /// );
     /// ```
@@ -194,7 +194,7 @@ impl LanguageIdentifier {
     /// use icu::locale::LanguageIdentifier;
     ///
     /// assert_eq!(
-    ///     LanguageIdentifier::normalize("pL_latn_pl").as_deref(),
+    ///     LanguageIdentifier::normalize("pL-latn-pl").as_deref(),
     ///     Ok("pl-Latn-PL")
     /// );
     /// ```

components/locale_core/src/locale.rs

@@ -39,14 +39,14 @@ use core::str::FromStr;
 ///  * *valid* - well-formed and only uses registered language subtags, extensions, keywords, types...
 ///  * *canonical* - valid and no deprecated codes or structure.
 ///
-/// At the moment parsing normalizes a well-formed locale identifier converting
-/// `_` separators to `-` and adjusting casing to conform to the Unicode standard.
-///
 /// Any syntactically invalid subtags will cause the parsing to fail with an error.
 ///
 /// This operation normalizes syntax to be well-formed. No legacy subtag replacements is performed.
 /// For validation and canonicalization, see `LocaleCanonicalizer`.
 ///
+/// ICU4X's Locale parsing does not allow for non-BCP-47-compatible locales [allowed by UTS 35 for backwards compatability][tr35-bcp].
+/// Furthermore, it currently does not allow for language tags to have more than three characters.
+///
 /// # Examples
 ///
 /// Simple example:
@@ -75,7 +75,7 @@ use core::str::FromStr;
 /// ```
 /// use icu::locale::{subtags::*, Locale};
 ///
-/// let loc: Locale = "eN_latn_Us-Valencia_u-hC-H12"
+/// let loc: Locale = "eN-latn-Us-Valencia-u-hC-H12"
 ///     .parse()
 ///     .expect("Failed to parse.");
 ///
@@ -89,6 +89,7 @@ use core::str::FromStr;
 /// ```
 ///
 /// [`Unicode Locale Identifier`]: https://unicode.org/reports/tr35/tr35.html#Unicode_locale_identifier
+/// [tr35-bcp]: https://unicode.org/reports/tr35/#BCP_47_Conformance
 #[derive(Default, PartialEq, Eq, Clone, Hash)] // no Ord or PartialOrd: see docs
 #[allow(clippy::exhaustive_structs)] // This struct is stable (and invoked by a macro)
 pub struct Locale {
@@ -159,7 +160,7 @@ impl Locale {
     /// use icu::locale::Locale;
     ///
     /// assert_eq!(
-    ///     Locale::normalize_utf8(b"pL_latn_pl-U-HC-H12").as_deref(),
+    ///     Locale::normalize_utf8(b"pL-latn-pl-U-HC-H12").as_deref(),
     ///     Ok("pl-Latn-PL-u-hc-h12")
     /// );
     /// ```
@@ -178,7 +179,7 @@ impl Locale {
     /// use icu::locale::Locale;
     ///
     /// assert_eq!(
-    ///     Locale::normalize("pL_latn_pl-U-HC-H12").as_deref(),
+    ///     Locale::normalize("pL-latn-pl-U-HC-H12").as_deref(),
     ///     Ok("pl-Latn-PL-u-hc-h12")
     /// );
     /// ```