refactor(js-macro): align whitespacing handling with js version

ref: lingui/js-lingui#1897
lingui · Mar 29, 2024 · b43ab5e · b43ab5e
1 parent 71dc5c2
commit b43ab5e
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 57 deletions.
diff --git a/src/builder.rs b/src/builder.rs
@@ -6,9 +6,6 @@ use swc_core::{
     },
 };
 
-use crate::{
-  normalize_witespaces_js::normalize_whitespaces_js,
-};
 use crate::tokens::{IcuChoice, CaseOrOffset, MsgToken};
 
 fn dedup_values(mut v: Vec<ValueWithPlaceholder>) -> Vec<ValueWithPlaceholder> {
@@ -54,7 +51,7 @@ pub struct MessageBuilder {
 }
 
 impl MessageBuilder {
-    pub fn parse(tokens: Vec<MsgToken>, jsx: bool) -> MessageBuilderResult {
+    pub fn parse(tokens: Vec<MsgToken>) -> MessageBuilderResult {
         let mut builder = MessageBuilder {
             message: String::new(),
             components_stack: Vec::new(),
@@ -64,15 +61,11 @@ impl MessageBuilder {
         };
 
         builder.from_tokens(tokens);
-        builder.to_args(jsx)
+        builder.to_args()
     }
 
-    pub fn to_args(mut self, jsx: bool) -> MessageBuilderResult {
-      let message_str = if jsx {
-        self.message
-      } else {
-        normalize_whitespaces_js(&self.message)
-      };
+    pub fn to_args(mut self) -> MessageBuilderResult {
+      let message_str = self.message;
 
       let message = Box::new(Expr::Lit(Lit::Str(Str {
         span: DUMMY_SP,

diff --git a/src/js_macro_folder.rs b/src/js_macro_folder.rs
@@ -24,7 +24,7 @@ impl<'a> JsMacroFolder<'a> {
     }
 
     fn create_message_descriptor_from_tokens(&mut self, tokens: Vec<MsgToken>) -> Expr {
-      let parsed = MessageBuilder::parse(tokens, false);
+      let parsed = MessageBuilder::parse(tokens);
 
       let mut props: Vec<PropOrSpread> = vec![
         create_key_value_prop("id", generate_message_id(&parsed.message_str, "").into()),
@@ -98,7 +98,7 @@ impl<'a> JsMacroFolder<'a> {
         if let Some(prop) = message_prop {
           let tokens = self.ctx.try_tokenize_expr(&prop.value).unwrap_or_else(|| Vec::new());
 
-          let parsed = MessageBuilder::parse(tokens, false);
+          let parsed = MessageBuilder::parse(tokens);
 
           if !id_prop.is_some() {
             new_props.push(

diff --git a/src/lib.rs b/src/lib.rs
@@ -13,7 +13,6 @@ use swc_core::{
 };
 
 mod tests;
-mod normalize_witespaces_js;
 mod builder;
 mod tokens;
 mod ast_utils;
@@ -57,7 +56,7 @@ impl LinguiMacroFolder {
             el.visit_children_with(&mut trans_visitor);
         }
 
-      let parsed = MessageBuilder::parse(trans_visitor.tokens, true);
+      let parsed = MessageBuilder::parse(trans_visitor.tokens);
       let id_attr = get_jsx_attr(&el.opening, "id");
 
       let context_attr_val = get_jsx_attr(&el.opening, "context")

diff --git a/src/macro_utils.rs b/src/macro_utils.rs
@@ -110,7 +110,7 @@ impl MacroCtx {
         let mut tokens: Vec<MsgToken> = Vec::with_capacity(tpl.quasis.len());
 
         for (i, tpl_element) in tpl.quasis.iter().enumerate() {
-            tokens.push(MsgToken::String(tpl_element.raw.to_string()));
+            tokens.push(MsgToken::String(tpl_element.cooked.as_ref().unwrap_or(&tpl_element.raw).to_string()));
 
             if let Some(exp) = tpl.exprs.get(i) {
                 if let Expr::Call(call) = exp.as_ref() {

diff --git a/src/normalize_witespaces_js.rs b/src/normalize_witespaces_js.rs
diff --git a/src/tests/js_t.rs b/src/tests/js_t.rs
@@ -156,12 +156,46 @@ to!(
      r#"
         import { i18n } from "@lingui/core";
         i18n._({
-            id: "EfogM+",
-            message: "Multiline\nstring"
+            id: "amQF7O",
+            message: "Multiline\n           string"
         });
      "#
 );
 
+to!(
+    js_continuation_character,
+    r#"
+       import { t } from '@lingui/macro';
+         t`Multiline\
+           string`;
+    "#,
+     r#"
+        import { i18n } from "@lingui/core";
+        i18n._({
+            id: "d1nA7b",
+            message: "Multiline           string"
+        });
+     "#
+);
+to!(
+    unicode_characters_interpreted,
+    r#"
+       import { t } from '@lingui/macro';
+       t`Message \u0020`;
+       t`Bienvenue\xA0!`
+    "#,
+     r#"
+        import { i18n } from "@lingui/core";
+        i18n._({
+            id: "dZXeyN",
+            message: "Message  "
+        });
+        i18n._({
+            id: "9K3RGd",
+            message: "Bienvenue !"
+        });
+     "#
+);
 to!(
     js_support_message_descriptor_in_t_fn,
     r#"