Implement Syntax 0.8, part3

fluent-syntax/src/ast.js

@@ -96,9 +96,10 @@ export class Placeable extends PatternElement {
 export class Expression extends SyntaxNode {}
 
 export class StringLiteral extends Expression {
-  constructor(value) {
+  constructor(raw, value) {
     super();
     this.type = "StringLiteral";
+    this.raw = raw;
     this.value = value;
   }
 }
@@ -135,6 +136,14 @@ export class VariableReference extends Expression {
   }
 }
 
+export class FunctionReference extends Expression {
+  constructor(id) {
+    super();
+    this.type = "FunctionReference";
+    this.id = id;
+  }
+}
+
 export class SelectExpression extends Expression {
   constructor(selector, variants) {
     super();
@@ -236,13 +245,6 @@ export class ResourceComment extends BaseComment {
   }
 }
 
-export class Function extends Identifier {
-  constructor(name) {
-    super(name);
-    this.type = "Function";
-  }
-}
-
 export class Junk extends SyntaxNode {
   constructor(content) {
     super();

fluent-syntax/src/parser.js

@@ -601,7 +601,7 @@ export default class FluentParser {
 
     if (next === "\\" || next === "\"") {
       ps.next();
-      return `\\${next}`;
+      return [`\\${next}`, next];
     }
 
     if (next === "u") {
@@ -618,7 +618,15 @@ export default class FluentParser {
         sequence += ch;
       }
 
-      return `\\u${sequence}`;
+      const codepoint = parseInt(sequence, 16);
+      const unescaped = codepoint <= 0xD7FF || 0xE000 <= codepoint
+        // It's a Unicode scalar value.
+        ? String.fromCodePoint(codepoint)
+        // Escape sequences reresenting surrogate code points are well-formed
+        // but invalid in Fluent. Replace them with U+FFFD REPLACEMENT
+        // CHARACTER.
+        : "�";
+      return [`\\u${sequence}`, unescaped];
     }
 
     throw new ParseError("E0025", next);
@@ -718,7 +726,7 @@ export default class FluentParser {
 
       ps.expectChar(")");
 
-      const func = new AST.Function(literal.id.name);
+      const func = new AST.FunctionReference(literal.id);
       if (this.withSpans) {
         func.addSpan(literal.span.start, literal.span.end);
       }
@@ -805,16 +813,20 @@ export default class FluentParser {
   }
 
   getString(ps) {
-    let val = "";
+    let raw = "";
+    let value = "";
 
     ps.expectChar("\"");
 
     let ch;
     while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) {
       if (ch === "\\") {
-        val += this.getEscapeSequence(ps);
+        const [sequence, unescaped] = this.getEscapeSequence(ps);
+        raw += sequence;
+        value += unescaped;
       } else {
-        val += ch;
+        raw += ch;
+        value += ch;
       }
     }
 
@@ -824,8 +836,7 @@ export default class FluentParser {
 
     ps.expectChar("\"");
 
-    return new AST.StringLiteral(val);
-
+    return new AST.StringLiteral(raw, value);
   }
 
   getLiteral(ps) {

fluent-syntax/src/serializer.js

@@ -96,7 +96,7 @@ function serializeMessage(message) {
     parts.push(serializeComment(message.comment));
   }
 
-  parts.push(`${serializeIdentifier(message.id)} =`);
+  parts.push(`${message.id.name} =`);
 
   if (message.value) {
     parts.push(serializeValue(message.value));
@@ -118,7 +118,7 @@ function serializeTerm(term) {
     parts.push(serializeComment(term.comment));
   }
 
-  parts.push(`-${serializeIdentifier(term.id)} =`);
+  parts.push(`-${term.id.name} =`);
   parts.push(serializeValue(term.value));
 
   for (const attribute of term.attributes) {
@@ -131,9 +131,8 @@ function serializeTerm(term) {
 
 
 function serializeAttribute(attribute) {
-  const id = serializeIdentifier(attribute.id);
   const value = indent(serializeValue(attribute.value));
-  return `\n    .${id} =${value}`;
+  return `\n    .${attribute.id.name} =${value}`;
 }
 
 
@@ -184,7 +183,7 @@ function serializeVariant(variant) {
 function serializeElement(element) {
   switch (element.type) {
     case "TextElement":
-      return serializeTextElement(element);
+      return element.value;
     case "Placeable":
       return serializePlaceable(element);
     default:
@@ -193,11 +192,6 @@ function serializeElement(element) {
 }
 
 
-function serializeTextElement(text) {
-  return text.value;
-}
-
-
 function serializePlaceable(placeable) {
   const expr = placeable.expression;
 
@@ -217,15 +211,16 @@ function serializePlaceable(placeable) {
 function serializeExpression(expr) {
   switch (expr.type) {
     case "StringLiteral":
-      return serializeStringLiteral(expr);
+      return `"${expr.raw}"`;
     case "NumberLiteral":
-      return serializeNumberLiteral(expr);
+      return expr.value;
     case "MessageReference":
-      return serializeMessageReference(expr);
+    case "FunctionReference":
+      return expr.id.name;
     case "TermReference":
-      return serializeTermReference(expr);
+      return `-${expr.id.name}`;
     case "VariableReference":
-      return serializeVariableReference(expr);
+      return `$${expr.id.name}`;
     case "AttributeExpression":
       return serializeAttributeExpression(expr);
     case "VariantExpression":
@@ -242,31 +237,6 @@ function serializeExpression(expr) {
 }
 
 
-function serializeStringLiteral(expr) {
-  return `"${expr.value}"`;
-}
-
-
-function serializeNumberLiteral(expr) {
-  return expr.value;
-}
-
-
-function serializeMessageReference(expr) {
-  return serializeIdentifier(expr.id);
-}
-
-
-function serializeTermReference(expr) {
-  return `-${serializeIdentifier(expr.id)}`;
-}
-
-
-function serializeVariableReference(expr) {
-  return `$${serializeIdentifier(expr.id)}`;
-}
-
-
 function serializeSelectExpression(expr) {
   const parts = [];
   const selector = `${serializeExpression(expr.selector)} ->`;
@@ -283,8 +253,7 @@ function serializeSelectExpression(expr) {
 
 function serializeAttributeExpression(expr) {
   const ref = serializeExpression(expr.ref);
-  const name = serializeIdentifier(expr.name);
-  return `${ref}.${name}`;
+  return `${ref}.${expr.name.name}`;
 }
 
 
@@ -296,7 +265,7 @@ function serializeVariantExpression(expr) {
 
 
 function serializeCallExpression(expr) {
-  const fun = serializeFunction(expr.callee);
+  const fun = serializeExpression(expr.callee);
   const positional = expr.positional.map(serializeExpression).join(", ");
   const named = expr.named.map(serializeNamedArgument).join(", ");
   if (expr.positional.length > 0 && expr.named.length > 0) {
@@ -307,40 +276,16 @@ function serializeCallExpression(expr) {
 
 
 function serializeNamedArgument(arg) {
-  const name = serializeIdentifier(arg.name);
-  const value = serializeArgumentValue(arg.value);
-  return `${name}: ${value}`;
+  const value = serializeExpression(arg.value);
+  return `${arg.name.name}: ${value}`;
 }
 
 
-function serializeArgumentValue(argval) {
-  switch (argval.type) {
-    case "StringLiteral":
-      return serializeStringLiteral(argval);
-    case "NumberLiteral":
-      return serializeNumberLiteral(argval);
-    default:
-      throw new Error(`Unknown argument type: ${argval.type}`);
-  }
-}
-
-
-function serializeIdentifier(identifier) {
-  return identifier.name;
-}
-
 function serializeVariantKey(key) {
   switch (key.type) {
     case "Identifier":
-      return serializeIdentifier(key);
-    case "NumberLiteral":
-      return serializeNumberLiteral(key);
+      return key.name;
     default:
-      throw new Error(`Unknown variant key type: ${key.type}`);
+      return serializeExpression(key);
   }
 }
-
-
-function serializeFunction(fun) {
-  return fun.name;
-}

fluent-syntax/test/fixtures_reference/astral.json

@@ -68,7 +68,8 @@
                         "type": "Placeable",
                         "expression": {
                             "type": "StringLiteral",
-                            "value": "\\uD83D\\uDE02"
+                            "raw": "\\uD83D\\uDE02",
+                            "value": "��"
                         }
                     }
                 ]
@@ -89,14 +90,16 @@
                         "type": "Placeable",
                         "expression": {
                             "type": "StringLiteral",
-                            "value": "\\uD83D"
+                            "raw": "\\uD83D",
+                            "value": "�"
                         }
                     },
                     {
                         "type": "Placeable",
                         "expression": {
                             "type": "StringLiteral",
-                            "value": "\\uDE02"
+                            "raw": "\\uDE02",
+                            "value": "�"
                         }
                     }
                 ]
@@ -135,6 +138,7 @@
                         "type": "Placeable",
                         "expression": {
                             "type": "StringLiteral",
+                            "raw": "A face 😂 with tears of joy.",
                             "value": "A face 😂 with tears of joy."
                         }
                     }

fluent-syntax/test/fixtures_reference/call_expressions.ftl

@@ -1,3 +1,19 @@
+## Callees
+
+function-callee = {FUNCTION()}
+
+# ERROR Equivalent to a MessageReference callee.
+mixed-case-callee = {Function()}
+
+# ERROR MessageReference is not a valid callee.
+message-callee = {message()}
+# ERROR TermReference is not a valid callee.
+term-callee = {-term()}
+# ERROR VariableReference is not a valid callee.
+variable-callee = {$variable()}
+
+## Arguments
+
 positional-args = {FUN(1, "a", msg)}
 named-args = {FUN(x: 1, y: "Y")}
 dense-named-args = {FUN(x:1, y:"Y")}