From 30d13b4a5b7d6510e7de3df86072c29758020e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 22 Nov 2018 15:57:31 +0100 Subject: [PATCH 1/3] Add FunctionReference --- fluent-syntax/src/ast.js | 15 +- fluent-syntax/src/parser.js | 2 +- fluent-syntax/src/serializer.js | 15 +- .../fixtures_reference/call_expressions.ftl | 16 ++ .../fixtures_reference/call_expressions.json | 255 ++++++++++++++---- .../select_expressions.json | 7 +- .../expressions_call_args.json | 12 +- .../fixtures_reference/call_expressions.json | 40 +++ 8 files changed, 286 insertions(+), 76 deletions(-) diff --git a/fluent-syntax/src/ast.js b/fluent-syntax/src/ast.js index 55a3f251a..0751b44ce 100644 --- a/fluent-syntax/src/ast.js +++ b/fluent-syntax/src/ast.js @@ -135,6 +135,14 @@ export class VariableReference extends Expression { } } +export class FunctionReference extends Expression { + constructor(id) { + super(); + this.type = "FunctionReference"; + this.id = id; + } +} + export class SelectExpression extends Expression { constructor(selector, variants) { super(); @@ -236,13 +244,6 @@ export class ResourceComment extends BaseComment { } } -export class Function extends Identifier { - constructor(name) { - super(name); - this.type = "Function"; - } -} - export class Junk extends SyntaxNode { constructor(content) { super(); diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 965a45aa7..91e8d9e7c 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -718,7 +718,7 @@ export default class FluentParser { ps.expectChar(")"); - const func = new AST.Function(literal.id.name); + const func = new AST.FunctionReference(literal.id); if (this.withSpans) { func.addSpan(literal.span.start, literal.span.end); } diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index 618496b7f..33db27ae4 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -221,7 +221,8 @@ function serializeExpression(expr) { case "NumberLiteral": return serializeNumberLiteral(expr); case "MessageReference": - return serializeMessageReference(expr); + case "FunctionReference": + return serializeIdentifier(expr.id); case "TermReference": return serializeTermReference(expr); case "VariableReference": @@ -252,11 +253,6 @@ function serializeNumberLiteral(expr) { } -function serializeMessageReference(expr) { - return serializeIdentifier(expr.id); -} - - function serializeTermReference(expr) { return `-${serializeIdentifier(expr.id)}`; } @@ -296,7 +292,7 @@ function serializeVariantExpression(expr) { function serializeCallExpression(expr) { - const fun = serializeFunction(expr.callee); + const fun = serializeExpression(expr.callee); const positional = expr.positional.map(serializeExpression).join(", "); const named = expr.named.map(serializeNamedArgument).join(", "); if (expr.positional.length > 0 && expr.named.length > 0) { @@ -339,8 +335,3 @@ function serializeVariantKey(key) { throw new Error(`Unknown variant key type: ${key.type}`); } } - - -function serializeFunction(fun) { - return fun.name; -} diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.ftl b/fluent-syntax/test/fixtures_reference/call_expressions.ftl index 065626422..9ed69bd1b 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/call_expressions.ftl @@ -1,3 +1,19 @@ +## Callees + +function-callee = {FUNCTION()} + +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee = {Function()} + +# ERROR MessageReference is not a valid callee. +message-callee = {message()} +# ERROR TermReference is not a valid callee. +term-callee = {-term()} +# ERROR VariableReference is not a valid callee. +variable-callee = {$variable()} + +## Arguments + positional-args = {FUN(1, "a", msg)} named-args = {FUN(x: 1, y: "Y")} dense-named-args = {FUN(x:1, y:"Y")} diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index a860260fe..e5a451938 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -1,6 +1,79 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Callees" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee = {Function()}\n" + }, + { + "type": "Comment", + "content": "ERROR MessageReference is not a valid callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee = {message()}\n" + }, + { + "type": "Comment", + "content": "ERROR TermReference is not a valid callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-callee = {-term()}\n" + }, + { + "type": "Comment", + "content": "ERROR VariableReference is not a valid callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee = {$variable()}\n" + }, + { + "type": "GroupComment", + "content": "Arguments" + }, { "type": "Message", "id": { @@ -15,8 +88,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -57,8 +133,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -106,8 +185,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -155,8 +237,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -242,8 +327,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -292,8 +380,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [] @@ -318,8 +409,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -368,8 +462,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -418,8 +515,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [] @@ -444,8 +544,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -475,8 +578,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -506,8 +612,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -540,8 +649,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -574,8 +686,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -608,15 +723,21 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "CallExpression", "callee": { - "type": "Function", - "name": "OTHER" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "OTHER" + } }, "positional": [], "named": [] @@ -644,8 +765,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -682,8 +806,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -720,8 +847,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -751,8 +881,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -790,8 +923,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -829,8 +965,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -864,8 +1003,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -912,8 +1054,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -972,8 +1117,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -1010,8 +1158,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 4c782006a..6bd9a698c 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -17,8 +17,11 @@ "selector": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "BUILTIN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "BUILTIN" + } }, "positional": [], "named": [] diff --git a/fluent-syntax/test/fixtures_structure/expressions_call_args.json b/fluent-syntax/test/fixtures_structure/expressions_call_args.json index a4ea0b03a..c7be7f94e 100644 --- a/fluent-syntax/test/fixtures_structure/expressions_call_args.json +++ b/fluent-syntax/test/fixtures_structure/expressions_call_args.json @@ -20,8 +20,16 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FOO", + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FOO", + "span": { + "type": "Span", + "start": 8, + "end": 11 + } + }, "span": { "type": "Span", "start": 8, diff --git a/fluent/test/fixtures_reference/call_expressions.json b/fluent/test/fixtures_reference/call_expressions.json index 431a25b5a..5f7c8ad63 100644 --- a/fluent/test/fixtures_reference/call_expressions.json +++ b/fluent/test/fixtures_reference/call_expressions.json @@ -1,4 +1,44 @@ { + "function-callee": [ + { + "type": "call", + "callee": { + "type": "func", + "name": "FUNCTION" + }, + "args": [] + } + ], + "mixed-case-callee": [ + { + "type": "call", + "callee": { + "type": "func", + "name": "Function" + }, + "args": [] + } + ], + "message-callee": [ + { + "type": "call", + "callee": { + "type": "func", + "name": "message" + }, + "args": [] + } + ], + "term-callee": [ + { + "type": "call", + "callee": { + "type": "func", + "name": "-term" + }, + "args": [] + } + ], "positional-args": [ { "type": "call", From 50f226ffc8e1d688c5210971989587c34baedaf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 22 Nov 2018 16:10:27 +0100 Subject: [PATCH 2/3] Inline trivial serialization functions --- fluent-syntax/src/serializer.js | 74 +++++++-------------------------- 1 file changed, 14 insertions(+), 60 deletions(-) diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index 33db27ae4..dd9f499e1 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -96,7 +96,7 @@ function serializeMessage(message) { parts.push(serializeComment(message.comment)); } - parts.push(`${serializeIdentifier(message.id)} =`); + parts.push(`${message.id.name} =`); if (message.value) { parts.push(serializeValue(message.value)); @@ -118,7 +118,7 @@ function serializeTerm(term) { parts.push(serializeComment(term.comment)); } - parts.push(`-${serializeIdentifier(term.id)} =`); + parts.push(`-${term.id.name} =`); parts.push(serializeValue(term.value)); for (const attribute of term.attributes) { @@ -131,9 +131,8 @@ function serializeTerm(term) { function serializeAttribute(attribute) { - const id = serializeIdentifier(attribute.id); const value = indent(serializeValue(attribute.value)); - return `\n .${id} =${value}`; + return `\n .${attribute.id.name} =${value}`; } @@ -184,7 +183,7 @@ function serializeVariant(variant) { function serializeElement(element) { switch (element.type) { case "TextElement": - return serializeTextElement(element); + return element.value; case "Placeable": return serializePlaceable(element); default: @@ -193,11 +192,6 @@ function serializeElement(element) { } -function serializeTextElement(text) { - return text.value; -} - - function serializePlaceable(placeable) { const expr = placeable.expression; @@ -217,16 +211,16 @@ function serializePlaceable(placeable) { function serializeExpression(expr) { switch (expr.type) { case "StringLiteral": - return serializeStringLiteral(expr); + return `"${expr.value}"`; case "NumberLiteral": - return serializeNumberLiteral(expr); + return expr.value; case "MessageReference": case "FunctionReference": - return serializeIdentifier(expr.id); + return expr.id.name; case "TermReference": - return serializeTermReference(expr); + return `-${expr.id.name}`; case "VariableReference": - return serializeVariableReference(expr); + return `$${expr.id.name}`; case "AttributeExpression": return serializeAttributeExpression(expr); case "VariantExpression": @@ -243,26 +237,6 @@ function serializeExpression(expr) { } -function serializeStringLiteral(expr) { - return `"${expr.value}"`; -} - - -function serializeNumberLiteral(expr) { - return expr.value; -} - - -function serializeTermReference(expr) { - return `-${serializeIdentifier(expr.id)}`; -} - - -function serializeVariableReference(expr) { - return `$${serializeIdentifier(expr.id)}`; -} - - function serializeSelectExpression(expr) { const parts = []; const selector = `${serializeExpression(expr.selector)} ->`; @@ -279,8 +253,7 @@ function serializeSelectExpression(expr) { function serializeAttributeExpression(expr) { const ref = serializeExpression(expr.ref); - const name = serializeIdentifier(expr.name); - return `${ref}.${name}`; + return `${ref}.${expr.name.name}`; } @@ -303,35 +276,16 @@ function serializeCallExpression(expr) { function serializeNamedArgument(arg) { - const name = serializeIdentifier(arg.name); - const value = serializeArgumentValue(arg.value); - return `${name}: ${value}`; -} - - -function serializeArgumentValue(argval) { - switch (argval.type) { - case "StringLiteral": - return serializeStringLiteral(argval); - case "NumberLiteral": - return serializeNumberLiteral(argval); - default: - throw new Error(`Unknown argument type: ${argval.type}`); - } + const value = serializeExpression(arg.value); + return `${arg.name.name}: ${value}`; } -function serializeIdentifier(identifier) { - return identifier.name; -} - function serializeVariantKey(key) { switch (key.type) { case "Identifier": - return serializeIdentifier(key); - case "NumberLiteral": - return serializeNumberLiteral(key); + return key.name; default: - throw new Error(`Unknown variant key type: ${key.type}`); + return serializeExpression(key); } } From fb5cbeffdedb809413c3414fd96299e7425c26ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 22 Nov 2018 16:22:17 +0100 Subject: [PATCH 3/3] Store unescaped content in StringLiteral.value and raw content in .raw --- fluent-syntax/src/ast.js | 3 ++- fluent-syntax/src/parser.js | 25 +++++++++++++------ fluent-syntax/src/serializer.js | 2 +- .../test/fixtures_reference/astral.json | 10 +++++--- .../fixtures_reference/call_expressions.json | 9 +++++++ .../escaped_characters.json | 14 ++++++++--- .../test/fixtures_reference/leading_dots.json | 8 ++++++ .../literal_expressions.json | 1 + .../test/fixtures_reference/messages.json | 1 + .../fixtures_reference/multiline_values.json | 8 ++++++ .../select_expressions.json | 2 ++ .../test/fixtures_reference/terms.json | 1 + .../fixtures_structure/escape_sequences.json | 14 ++++++++--- .../test/fixtures_structure/leading_dots.json | 7 ++++++ .../variant_with_empty_pattern.json | 1 + .../whitespace_leading.json | 2 ++ .../whitespace_trailing.json | 1 + 17 files changed, 89 insertions(+), 20 deletions(-) diff --git a/fluent-syntax/src/ast.js b/fluent-syntax/src/ast.js index 0751b44ce..fea7823c7 100644 --- a/fluent-syntax/src/ast.js +++ b/fluent-syntax/src/ast.js @@ -96,9 +96,10 @@ export class Placeable extends PatternElement { export class Expression extends SyntaxNode {} export class StringLiteral extends Expression { - constructor(value) { + constructor(raw, value) { super(); this.type = "StringLiteral"; + this.raw = raw; this.value = value; } } diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 91e8d9e7c..89376f267 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -601,7 +601,7 @@ export default class FluentParser { if (next === "\\" || next === "\"") { ps.next(); - return `\\${next}`; + return [`\\${next}`, next]; } if (next === "u") { @@ -618,7 +618,15 @@ export default class FluentParser { sequence += ch; } - return `\\u${sequence}`; + const codepoint = parseInt(sequence, 16); + const unescaped = codepoint <= 0xD7FF || 0xE000 <= codepoint + // It's a Unicode scalar value. + ? String.fromCodePoint(codepoint) + // Escape sequences reresenting surrogate code points are well-formed + // but invalid in Fluent. Replace them with U+FFFD REPLACEMENT + // CHARACTER. + : "�"; + return [`\\u${sequence}`, unescaped]; } throw new ParseError("E0025", next); @@ -805,16 +813,20 @@ export default class FluentParser { } getString(ps) { - let val = ""; + let raw = ""; + let value = ""; ps.expectChar("\""); let ch; while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) { if (ch === "\\") { - val += this.getEscapeSequence(ps); + const [sequence, unescaped] = this.getEscapeSequence(ps); + raw += sequence; + value += unescaped; } else { - val += ch; + raw += ch; + value += ch; } } @@ -824,8 +836,7 @@ export default class FluentParser { ps.expectChar("\""); - return new AST.StringLiteral(val); - + return new AST.StringLiteral(raw, value); } getLiteral(ps) { diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index dd9f499e1..48ef8d745 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -211,7 +211,7 @@ function serializePlaceable(placeable) { function serializeExpression(expr) { switch (expr.type) { case "StringLiteral": - return `"${expr.value}"`; + return `"${expr.raw}"`; case "NumberLiteral": return expr.value; case "MessageReference": diff --git a/fluent-syntax/test/fixtures_reference/astral.json b/fluent-syntax/test/fixtures_reference/astral.json index 6056fb727..6f748c9c0 100644 --- a/fluent-syntax/test/fixtures_reference/astral.json +++ b/fluent-syntax/test/fixtures_reference/astral.json @@ -68,7 +68,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uD83D\\uDE02" + "raw": "\\uD83D\\uDE02", + "value": "��" } } ] @@ -89,14 +90,16 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uD83D" + "raw": "\\uD83D", + "value": "�" } }, { "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uDE02" + "raw": "\\uDE02", + "value": "�" } } ] @@ -135,6 +138,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "A face 😂 with tears of joy.", "value": "A face 😂 with tears of joy." } } diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index e5a451938..0a3ae0697 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -101,6 +101,7 @@ }, { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -160,6 +161,7 @@ }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -212,6 +214,7 @@ }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -250,6 +253,7 @@ }, { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -280,6 +284,7 @@ }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -336,6 +341,7 @@ "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -418,6 +424,7 @@ "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -471,6 +478,7 @@ "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -587,6 +595,7 @@ "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" } ], diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index 26b1974b0..91de49944 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -122,7 +122,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\"" + "raw": "\\\"", + "value": "\"" } } ] @@ -143,7 +144,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\" + "raw": "\\\\", + "value": "\\" } } ] @@ -186,7 +188,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\u0041" + "raw": "\\u0041", + "value": "A" } } ] @@ -207,7 +210,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\u0041" + "raw": "\\\\u0041", + "value": "\\u0041" } } ] @@ -236,6 +240,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "{", "value": "{" } }, @@ -265,6 +270,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "}", "value": "}" } }, diff --git a/fluent-syntax/test/fixtures_reference/leading_dots.json b/fluent-syntax/test/fixtures_reference/leading_dots.json index 6459cb57c..1787d0ee0 100644 --- a/fluent-syntax/test/fixtures_reference/leading_dots.json +++ b/fluent-syntax/test/fixtures_reference/leading_dots.json @@ -50,6 +50,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -75,6 +76,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -104,6 +106,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -133,6 +136,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -229,6 +233,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -316,6 +321,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -377,6 +383,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -463,6 +470,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, diff --git a/fluent-syntax/test/fixtures_reference/literal_expressions.json b/fluent-syntax/test/fixtures_reference/literal_expressions.json index a979bc623..da73e1311 100644 --- a/fluent-syntax/test/fixtures_reference/literal_expressions.json +++ b/fluent-syntax/test/fixtures_reference/literal_expressions.json @@ -14,6 +14,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "abc", "value": "abc" } } diff --git a/fluent-syntax/test/fixtures_reference/messages.json b/fluent-syntax/test/fixtures_reference/messages.json index 1a5b5c46f..fa8dcbd7b 100644 --- a/fluent-syntax/test/fixtures_reference/messages.json +++ b/fluent-syntax/test/fixtures_reference/messages.json @@ -218,6 +218,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } diff --git a/fluent-syntax/test/fixtures_reference/multiline_values.json b/fluent-syntax/test/fixtures_reference/multiline_values.json index 17fb6f8e4..4d3dd033e 100644 --- a/fluent-syntax/test/fixtures_reference/multiline_values.json +++ b/fluent-syntax/test/fixtures_reference/multiline_values.json @@ -126,6 +126,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "placeables", "value": "placeables" } }, @@ -137,6 +138,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "at", "value": "at" } }, @@ -148,6 +150,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "of lines", "value": "of lines" } }, @@ -155,6 +158,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } } @@ -176,6 +180,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "A multiline value", "value": "A multiline value" } }, @@ -187,6 +192,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "with a placeable", "value": "with a placeable" } } @@ -280,6 +286,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -309,6 +316,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } } diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 6bd9a698c..61e18fa5c 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -57,6 +57,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } }, @@ -169,6 +170,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } diff --git a/fluent-syntax/test/fixtures_reference/terms.json b/fluent-syntax/test/fixtures_reference/terms.json index f63122e57..d3eff8065 100644 --- a/fluent-syntax/test/fixtures_reference/terms.json +++ b/fluent-syntax/test/fixtures_reference/terms.json @@ -49,6 +49,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.json b/fluent-syntax/test/fixtures_structure/escape_sequences.json index e9e851c7b..dccf18cb5 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.json +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.json @@ -252,7 +252,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\"", + "raw": "\\\"", + "value": "\"", "span": { "type": "Span", "start": 262, @@ -298,7 +299,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\", + "raw": "\\\\", + "value": "\\", "span": { "type": "Span", "start": 291, @@ -419,7 +421,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\u0041", + "raw": "\\u0041", + "value": "A", "span": { "type": "Span", "start": 443, @@ -465,7 +468,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\u0041", + "raw": "\\\\u0041", + "value": "\\u0041", "span": { "type": "Span", "start": 479, @@ -529,6 +533,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "{", "value": "{", "span": { "type": "Span", @@ -593,6 +598,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "}", "value": "}", "span": { "type": "Span", diff --git a/fluent-syntax/test/fixtures_structure/leading_dots.json b/fluent-syntax/test/fixtures_structure/leading_dots.json index d2e234c38..f95994c10 100644 --- a/fluent-syntax/test/fixtures_structure/leading_dots.json +++ b/fluent-syntax/test/fixtures_structure/leading_dots.json @@ -95,6 +95,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -150,6 +151,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -214,6 +216,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -278,6 +281,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -489,6 +493,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -671,6 +676,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -797,6 +803,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", diff --git a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json index 2b61ee990..c1d77e6dd 100644 --- a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json @@ -47,6 +47,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "", "span": { "type": "Span", diff --git a/fluent-syntax/test/fixtures_structure/whitespace_leading.json b/fluent-syntax/test/fixtures_structure/whitespace_leading.json index 168eee92f..1ee9e575c 100644 --- a/fluent-syntax/test/fixtures_structure/whitespace_leading.json +++ b/fluent-syntax/test/fixtures_structure/whitespace_leading.json @@ -111,6 +111,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "", "span": { "type": "Span", @@ -166,6 +167,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": " ", "value": " ", "span": { "type": "Span", diff --git a/fluent-syntax/test/fixtures_structure/whitespace_trailing.json b/fluent-syntax/test/fixtures_structure/whitespace_trailing.json index 69e33deb4..0a58feab0 100644 --- a/fluent-syntax/test/fixtures_structure/whitespace_trailing.json +++ b/fluent-syntax/test/fixtures_structure/whitespace_trailing.json @@ -192,6 +192,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": " ", "value": " ", "span": { "type": "Span",