Add support for literals

Summary: This diff adds support for literals as described in the spec. These become useful with user-defined functions, which are introduced in future diffs. The lexer already supports lexing literals so this was pretty easy to implement. Reviewed By: createdbysk Differential Revision: D67608814 fbshipit-source-id: 12ff50723dfe16fd34a2d93a8d3b30f65cdc0e17
facebook · Dec 26, 2024 · 10e0583 · 10e0583
1 parent f245bdd
commit 10e0583
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 42 deletions.
diff --git a/third-party/thrift/src/thrift/compiler/whisker/ast.cc b/third-party/thrift/src/thrift/compiler/whisker/ast.cc
@@ -16,6 +16,7 @@
 
 #include <thrift/compiler/whisker/ast.h>
 #include <thrift/compiler/whisker/detail/overload.h>
+#include <thrift/compiler/whisker/detail/string.h>
 
 #include <fmt/core.h>
 #include <fmt/ranges.h>
@@ -67,6 +68,13 @@ std::string partial_apply::path_string() const {
 std::string expression::to_string() const {
   return detail::variant_match(
       which,
+      [](const string_literal& s) {
+        return fmt::format("\"{}\"", detail::escape(s.text));
+      },
+      [](const i64_literal& i) { return fmt::format("{}", i.value); },
+      [](const null_literal&) -> std::string { return "null"; },
+      [](const true_literal&) -> std::string { return "true"; },
+      [](const false_literal&) -> std::string { return "false"; },
       [](const variable_lookup& v) { return v.chain_string(); },
       [](const function_call& f) {
         std::string out = fmt::format("({}", f.name());

diff --git a/third-party/thrift/src/thrift/compiler/whisker/ast.h b/third-party/thrift/src/thrift/compiler/whisker/ast.h
@@ -154,6 +154,10 @@ struct variable_lookup {
  */
 struct expression {
   source_range loc;
+
+  /**
+   * A function call expression like `(foo.bar "hello" "world")`.
+   */
   struct function_call {
     /**
      * Base class for all built-in functions.
@@ -263,7 +267,61 @@ struct expression {
     // Remove in C++20 which introduces comparison operator synthesis
     WHISKER_DEFINE_OPERATOR_INEQUALITY(function_call)
   };
-  std::variant<variable_lookup, function_call> which;
+
+  struct string_literal {
+    std::string text;
+
+    friend bool operator==(
+        const string_literal& lhs, const string_literal& rhs) {
+      return lhs.text == rhs.text;
+    }
+    // Remove in C++20 which introduces comparison operator synthesis
+    WHISKER_DEFINE_OPERATOR_INEQUALITY(string_literal)
+  };
+
+  struct i64_literal {
+    std::int64_t value;
+
+    friend bool operator==(const i64_literal& lhs, const i64_literal& rhs) {
+      return lhs.value == rhs.value;
+    }
+    // Remove in C++20 which introduces comparison operator synthesis
+    WHISKER_DEFINE_OPERATOR_INEQUALITY(i64_literal)
+  };
+
+  struct null_literal {
+    friend bool operator==(const null_literal&, const null_literal&) {
+      return true;
+    }
+    // Remove in C++20 which introduces comparison operator synthesis
+    WHISKER_DEFINE_OPERATOR_INEQUALITY(null_literal)
+  };
+
+  struct true_literal {
+    friend bool operator==(const true_literal&, const true_literal&) {
+      return true;
+    }
+    // Remove in C++20 which introduces comparison operator synthesis
+    WHISKER_DEFINE_OPERATOR_INEQUALITY(true_literal)
+  };
+
+  struct false_literal {
+    friend bool operator==(const false_literal&, const false_literal&) {
+      return true;
+    }
+    // Remove in C++20 which introduces comparison operator synthesis
+    WHISKER_DEFINE_OPERATOR_INEQUALITY(false_literal)
+  };
+
+  std::variant<
+      string_literal,
+      i64_literal,
+      null_literal,
+      true_literal,
+      false_literal,
+      variable_lookup,
+      function_call>
+      which;
 
   /**
    * Determines if two expressions are syntactically equivalent, excluding their

diff --git a/third-party/thrift/src/thrift/compiler/whisker/parser.cc b/third-party/thrift/src/thrift/compiler/whisker/parser.cc
@@ -1029,16 +1029,46 @@ class parser {
         scan};
   }
 
-  // expression → { variable-lookup | function-call }
+  // expression → { literal | variable-lookup | function-call }
   parse_result<ast::expression> parse_expression(parser_scan_window scan) {
     assert(scan.empty());
     auto scan_start = scan.start;
-    using function_call = ast::expression::function_call;
+
+    using expression = ast::expression;
+    using function_call = expression::function_call;
+
+    // Parse literals
+    {
+      const token& t = scan.advance();
+      switch (t.kind) {
+        case tok::string_literal:
+          return {
+              expression{
+                  scan.range(),
+                  expression::string_literal{std::string(t.string_value())}},
+              scan};
+        case tok::i64_literal:
+          return {
+              expression{scan.range(), expression::i64_literal{t.i64_value()}},
+              scan};
+        case tok::kw_null:
+          return {expression{scan.range(), expression::null_literal{}}, scan};
+        case tok::kw_true:
+          return {expression{scan.range(), expression::true_literal{}}, scan};
+        case tok::kw_false:
+          return {expression{scan.range(), expression::false_literal{}}, scan};
+        default:
+          // Other tokens are not literals, so reset the scan.
+          scan = scan.with_head(scan_start);
+          break;
+      }
+      assert(scan.empty());
+    }
 
     if (parse_result lookup = parse_variable_lookup(scan)) {
       auto expr = std::move(lookup).consume_and_advance(&scan);
       return {
-          ast::expression{scan.with_start(scan_start).range(), std::move(expr)},
+          expression{scan.with_start(scan_start).range(), std::move(expr)},
           scan};
     }
 
@@ -1068,29 +1098,27 @@ class parser {
     // positional-argument → { expression }
     // named-argument → { identifier ~ "=" ~ expression }
     const auto parse_argument = [this, &func](parser_scan_window scan)
-        -> parse_result<std::variant<ast::expression, named_argument_entry>> {
+        -> parse_result<std::variant<expression, named_argument_entry>> {
       assert(scan.empty());
       const token& id = scan.peek();
       if (id.kind == tok::identifier && scan.next().peek().kind == tok::eq) {
         scan = scan.next(2).make_fresh();
-        if (parse_result expression = parse_expression(scan)) {
+        if (parse_result expr = parse_expression(scan)) {
           return {
               named_argument_entry{
                   id.string_value(),
                   function_call::named_argument{
                       ast::identifier{id.range, std::string(id.string_value())},
-                      std::make_unique<ast::expression>(
-                          std::move(expression).consume_and_advance(&scan))}},
+                      std::make_unique<expression>(
+                          std::move(expr).consume_and_advance(&scan))}},
               scan};
         }
         report_expected(scan, "expression in named argument");
       }
 
       assert(scan.empty());
-      if (parse_result expression = parse_expression(scan)) {
-        return {
-            ast::expression{std::move(expression).consume_and_advance(&scan)},
-            scan};
+      if (parse_result expr = parse_expression(scan)) {
+        return {expression{std::move(expr).consume_and_advance(&scan)}, scan};
       }
       if (scan.peek().kind == tok::eq) {
         report_fatal_error(

diff --git a/third-party/thrift/src/thrift/compiler/whisker/render.cc b/third-party/thrift/src/thrift/compiler/whisker/render.cc
@@ -352,6 +352,21 @@ class render_engine {
     using function_call = ast::expression::function_call;
     return detail::variant_match(
         expr.which,
+        [](const ast::expression::string_literal& s) -> object_ptr {
+          return std::make_shared<const object>(string(s.text));
+        },
+        [](const ast::expression::i64_literal& i) -> object_ptr {
+          return std::make_shared<const object>(i64(i.value));
+        },
+        [](const ast::expression::null_literal&) -> object_ptr {
+          return as_ref(whisker::make::null);
+        },
+        [](const ast::expression::true_literal&) -> object_ptr {
+          return as_ref(whisker::make::true_);
+        },
+        [](const ast::expression::false_literal&) -> object_ptr {
+          return as_ref(whisker::make::false_);
+        },
         [&](const ast::variable_lookup& variable_lookup) -> object_ptr {
           return as_ref(lookup_variable(variable_lookup));
         },

diff --git a/third-party/thrift/src/thrift/compiler/whisker/test/parser_test.cc b/third-party/thrift/src/thrift/compiler/whisker/test/parser_test.cc
@@ -561,6 +561,28 @@ TEST_F(ParserTest, conditional_block_not_wrong_arity) {
           1)));
 }
 
+TEST_F(ParserTest, literals) {
+  auto ast = parse_ast(
+      "{{null}}\n"
+      "{{true}}\n"
+      "{{false}}\n"
+      "{{\"hello\\tworld\"}}\n"
+      "{{-1234}}\n");
+  EXPECT_EQ(
+      to_string(*ast),
+      "root [path/to/test-1.whisker]\n"
+      "|- interpolation <line:1:1, col:9> 'null'\n"
+      "|- newline <line:1:9, line:2:1> '\\n'\n"
+      "|- interpolation <line:2:1, col:9> 'true'\n"
+      "|- newline <line:2:9, line:3:1> '\\n'\n"
+      "|- interpolation <line:3:1, col:10> 'false'\n"
+      "|- newline <line:3:10, line:4:1> '\\n'\n"
+      "|- interpolation <line:4:1, col:19> '\"hello\\tworld\"'\n"
+      "|- newline <line:4:19, line:5:1> '\\n'\n"
+      "|- interpolation <line:5:1, col:10> '-1234'\n"
+      "|- newline <line:5:10, line:6:1> '\\n'\n");
+}
+
 TEST_F(ParserTest, function_call) {
   auto ast = parse_ast("{{ (uppercase (lowercase hello\tspace) world) }}");
   EXPECT_EQ(
@@ -570,11 +592,11 @@ TEST_F(ParserTest, function_call) {
 }
 
 TEST_F(ParserTest, function_call_named_args) {
-  auto ast = parse_ast("{{ (str.concat hello world sep=comma) }}");
+  auto ast = parse_ast(R"({{ (str.concat hello world sep=",") }})");
   EXPECT_EQ(
       to_string(*ast),
       "root [path/to/test-1.whisker]\n"
-      "|- interpolation <line:1:1, col:41> '(str.concat hello world sep=comma)'\n");
+      "|- interpolation <line:1:1, col:39> '(str.concat hello world sep=\",\")'\n");
 }
 
 TEST_F(ParserTest, function_call_named_args_only) {
@@ -599,7 +621,7 @@ TEST_F(ParserTest, function_call_named_args_duplicate) {
 }
 
 TEST_F(ParserTest, function_call_named_args_without_lookup) {
-  auto ast = parse_ast("{{ (sep=comma) }}");
+  auto ast = parse_ast(R"({{ (sep=",") }})");
   // `sep` is parsed as the name
   EXPECT_FALSE(ast.has_value());
   EXPECT_THAT(
@@ -612,7 +634,7 @@ TEST_F(ParserTest, function_call_named_args_without_lookup) {
 }
 
 TEST_F(ParserTest, function_call_positional_args_after_named_args) {
-  auto ast = parse_ast("{{ (str.concat sep=comma hello world) }}");
+  auto ast = parse_ast(R"({{ (str.concat sep="," hello world) }})");
   EXPECT_FALSE(ast.has_value());
   EXPECT_THAT(
       diagnostics,
@@ -637,7 +659,7 @@ TEST_F(ParserTest, function_call_named_args_not_identifier) {
 
 TEST_F(ParserTest, function_call_named_args_for_builtins) {
   {
-    auto ast = parse_ast("{{ (not foo sep=comma) }}");
+    auto ast = parse_ast("{{ (not foo ignore=1234) }}");
     EXPECT_FALSE(ast.has_value());
     EXPECT_THAT(
         diagnostics,
@@ -648,7 +670,7 @@ TEST_F(ParserTest, function_call_named_args_for_builtins) {
             1)));
   }
   {
-    auto ast = parse_ast("{{ (and foo bar sep=comma) }}");
+    auto ast = parse_ast("{{ (and foo bar ignore=1234) }}");
     EXPECT_FALSE(ast.has_value());
     EXPECT_THAT(
         diagnostics,
@@ -659,7 +681,7 @@ TEST_F(ParserTest, function_call_named_args_for_builtins) {
             1)));
   }
   {
-    auto ast = parse_ast("{{ (or foo bar sep=comma) }}");
+    auto ast = parse_ast("{{ (or foo bar ignore=1234) }}");
     EXPECT_FALSE(ast.has_value());
     EXPECT_THAT(
         diagnostics,