Compile variable nodes into the Liquid::C::BlockBody VM code

ext/liquid_c/block.c

@@ -3,6 +3,7 @@
 #include "tokenizer.h"
 #include "stringutil.h"
 #include "vm.h"
+#include "variable.h"
 #include <stdio.h>
 
 static ID
@@ -16,6 +17,7 @@ static ID
     intern_ivar_nodelist;
 
 static VALUE tag_registry;
+static VALUE variable_placeholder = Qnil;
 
 typedef struct tag_markup {
     VALUE name;
@@ -137,10 +139,14 @@ static tag_markup_t internal_block_body_parse(block_body_t *body, parse_context_
             }
             case TOKEN_VARIABLE:
             {
-                VALUE args[2] = {rb_enc_str_new(token.str_trimmed, token.len_trimmed, utf8_encoding), parse_context->ruby_obj};
-                VALUE var = rb_class_new_instance(2, args, cLiquidVariable);
-
-                vm_assembler_add_write_node(&body->code, var);
+                variable_parse_args_t parse_args = {
+                    .markup = token.str_trimmed,
+                    .markup_end = token.str_trimmed + token.len_trimmed,
+                    .code = &body->code,
+                    .parse_context = parse_context->ruby_obj,
+                    .line_number = token_start_line_number,
+                };
+                internal_variable_parse(&parse_args);
                 render_score_increment += 1;
                 body->blank = false;
                 break;
@@ -284,13 +290,22 @@ static VALUE block_body_remove_blank_strings(VALUE self)
     return Qnil;
 }
 
+static void memoize_variable_placeholder()
+{
+    if (variable_placeholder == Qnil) {
+        VALUE cLiquidCVariablePlaceholder = rb_const_get(mLiquidC, rb_intern("VariablePlaceholder"));
+        variable_placeholder = rb_class_new_instance(0, NULL, cLiquidCVariablePlaceholder);
+    }
+}
+
 // Deprecated: avoid using this for the love of performance
 static VALUE block_body_nodelist(VALUE self)
 {
     block_body_t *body;
     BlockBody_Get_Struct(self, body);
 
     ensure_not_parsing(body);
+    memoize_variable_placeholder();
 
     // Use rb_attr_get insteaad of rb_ivar_get to avoid an instance variable not
     // initialized warning.
@@ -318,8 +333,10 @@ static VALUE block_body_nodelist(VALUE self)
                 rb_ary_push(nodelist, const_ptr[0]);
                 break;
             }
-            default:
-                rb_bug("invalid opcode: %u", ip[-1]);
+
+            case OP_POP_WRITE_VARIABLE:
+                rb_ary_push(nodelist, variable_placeholder);
+                break;
         }
         liquid_vm_next_instruction(&ip, &const_ptr);
     }
@@ -352,5 +369,7 @@ void init_liquid_block()
     rb_define_method(cLiquidCBlockBody, "remove_blank_strings", block_body_remove_blank_strings, 0);
     rb_define_method(cLiquidCBlockBody, "blank?", block_body_blank_p, 0);
     rb_define_method(cLiquidCBlockBody, "nodelist", block_body_nodelist, 0);
+
+    rb_global_variable(&variable_placeholder);
 }
 

ext/liquid_c/extconf.rb

@@ -7,5 +7,10 @@
 if ENV['DEBUG'] == 'true' && compiler =~ /gcc|g\+\+/
   $CFLAGS << ' -fbounds-check'
 end
+
+if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7.0") # added in 2.7
+  $CFLAGS << ' -DHAVE_RB_HASH_BULK_INSERT'
+end
+
 $warnflags.gsub!(/-Wdeclaration-after-statement/, "") if $warnflags
 create_makefile("liquid_c")

ext/liquid_c/lexer.h

@@ -42,5 +42,12 @@ inline static VALUE token_to_rstr(lexer_token_t token) {
     return rb_enc_str_new(token.val, token.val_end - token.val, utf8_encoding);
 }
 
+inline static VALUE token_to_rsym(lexer_token_t token) {
+    VALUE sym = rb_check_symbol_cstr(token.val, token.val_end - token.val, utf8_encoding);
+    if (RB_LIKELY(sym != Qnil))
+        return sym;
+    return rb_str_intern(token_to_rstr(token));
+}
+
 #endif
 

ext/liquid_c/parser.c

@@ -130,6 +130,17 @@ static VALUE parse_variable(parser_t *p)
     return rb_class_new_instance(4, args, cLiquidVariableLookup);
 }
 
+bool will_parse_constant_expression_next(parser_t *p)
+{
+    switch (p->cur.type) {
+        case TOKEN_IDENTIFIER:
+        case TOKEN_OPEN_SQUARE:
+            return false;
+        default:
+            return true;
+    }
+}
+
 VALUE parse_expression(parser_t *p)
 {
     switch (p->cur.type) {

ext/liquid_c/parser.h

@@ -15,6 +15,7 @@ lexer_token_t parser_consume(parser_t *parser, unsigned char type);
 lexer_token_t parser_consume_any(parser_t *parser);
 
 VALUE parse_expression(parser_t *parser);
+bool will_parse_constant_expression_next(parser_t *parser);
 
 void init_liquid_parser(void);
 

ext/liquid_c/variable.c

@@ -3,6 +3,142 @@
 #include "parser.h"
 #include <stdio.h>
 
+static ID id_rescue_strict_parse_syntax_error;
+
+static void compile_expression(vm_assembler_t *code, VALUE expression, bool const_expression)
+{
+    if (const_expression)
+        vm_assembler_add_push_const(code, expression);
+    else
+        vm_assembler_add_push_eval_expr(code, expression);
+}
+
+static int compile_each_keyword_arg(VALUE key, VALUE value, VALUE func_arg)
+{
+    vm_assembler_t *code = (vm_assembler_t *)func_arg;
+
+    vm_assembler_add_push_const(code, key);
+
+    bool is_const_expr = !rb_respond_to(value, id_evaluate);
+    compile_expression(code, value, is_const_expr);
+
+    return ST_CONTINUE;
+}
+
+static inline void parse_and_compile_expression(parser_t *p, vm_assembler_t *code)
+{
+    bool is_const = will_parse_constant_expression_next(p);
+    compile_expression(code, parse_expression(p), is_const);
+}
+
+static VALUE try_variable_strict_parse(VALUE uncast_args)
+{
+    variable_parse_args_t *parse_args = (void *)uncast_args;
+    parser_t p;
+    init_parser(&p, parse_args->markup, parse_args->markup_end);
+    vm_assembler_t *code = parse_args->code;
+
+    if (p.cur.type == TOKEN_EOS)
+        return Qnil;
+
+    vm_assembler_add_render_variable_rescue(code, parse_args->line_number);
+
+    parse_and_compile_expression(&p, code);
+
+    while (parser_consume(&p, TOKEN_PIPE).type) {
+        lexer_token_t filter_name_token = parser_must_consume(&p, TOKEN_IDENTIFIER);
+        VALUE filter_name = token_to_rsym(filter_name_token);
+
+        size_t arg_count = 0;
+        bool const_keyword_args = true;
+        VALUE keyword_args = Qnil;
+
+        if (parser_consume(&p, TOKEN_COLON).type) {
+            do {
+                if (p.cur.type == TOKEN_IDENTIFIER && p.next.type == TOKEN_COLON) {
+                    VALUE key = token_to_rstr(parser_consume_any(&p));
+                    parser_consume_any(&p);
+
+                    if (const_keyword_args && !will_parse_constant_expression_next(&p))
+                        const_keyword_args = false;
+                    if (keyword_args == Qnil)
+                        keyword_args = rb_hash_new();
+                    rb_hash_aset(keyword_args, key, parse_expression(&p));
+                } else {
+                    parse_and_compile_expression(&p, code);
+                    arg_count++;
+                }
+            } while (parser_consume(&p, TOKEN_COMMA).type);
+        }
+
+        if (keyword_args != Qnil) {
+            arg_count++;
+            if (RHASH_SIZE(keyword_args) > 255) {
+                rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Too many filter keyword arguments");
+            }
+            if (const_keyword_args) {
+                vm_assembler_add_push_const(code, keyword_args);
+            } else {
+                rb_hash_foreach(keyword_args, compile_each_keyword_arg, (VALUE)code);
+                vm_assembler_add_hash_new(code, RHASH_SIZE(keyword_args));
+            }
+        }
+        if (arg_count > 254) {
+            rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Too many filter arguments");
+        }
+        vm_assembler_add_filter(code, filter_name, arg_count);
+    }
+
+    vm_assembler_add_pop_write_variable(code);
+
+    parser_must_consume(&p, TOKEN_EOS);
+
+    return Qnil;
+}
+
+typedef struct variable_strict_parse_rescue {
+    variable_parse_args_t *parse_args;
+    size_t instructions_size;
+    size_t constants_size;
+    size_t stack_size;
+} variable_strict_parse_rescue_t;
+
+static VALUE variable_strict_parse_rescue(VALUE uncast_args, VALUE exception)
+{
+    variable_strict_parse_rescue_t *rescue_args = (void *)uncast_args;
+    variable_parse_args_t *parse_args = rescue_args->parse_args;
+    vm_assembler_t *code = parse_args->code;
+
+    // undo partial strict parse
+    code->instructions.data_end = code->instructions.data + rescue_args->instructions_size;
+    code->constants.data_end = code->constants.data + rescue_args->constants_size;
+    code->stack_size = rescue_args->stack_size;
+
+    if (rb_obj_is_kind_of(exception, cLiquidSyntaxError) == Qfalse)
+        rb_exc_raise(exception);
+
+    VALUE markup_obj = rb_enc_str_new(parse_args->markup, parse_args->markup_end - parse_args->markup, utf8_encoding);
+    VALUE variable_obj = rb_funcall(
+        cLiquidVariable, id_rescue_strict_parse_syntax_error, 3,
+        exception, markup_obj, parse_args->parse_context
+    );
+
+    vm_assembler_add_write_node(code, variable_obj);
+    return Qnil;
+}
+
+void internal_variable_parse(variable_parse_args_t *parse_args)
+{
+    vm_assembler_t *code = parse_args->code;
+    variable_strict_parse_rescue_t rescue_args = {
+        .parse_args = parse_args,
+        .instructions_size = c_buffer_size(&code->instructions),
+        .constants_size = c_buffer_size(&code->constants),
+        .stack_size = code->stack_size,
+    };
+    rb_rescue(try_variable_strict_parse, (VALUE)parse_args, variable_strict_parse_rescue, (VALUE)&rescue_args);
+}
+
 static VALUE rb_variable_parse(VALUE self, VALUE markup, VALUE filters)
 {
     StringValue(markup);
@@ -49,6 +185,8 @@ static VALUE rb_variable_parse(VALUE self, VALUE markup, VALUE filters)
 
 void init_liquid_variable(void)
 {
+    id_rescue_strict_parse_syntax_error = rb_intern("rescue_strict_parse_syntax_error");
+
     rb_define_singleton_method(cLiquidVariable, "c_strict_parse", rb_variable_parse, 2);
 }
 

ext/liquid_c/variable.h

@@ -1,7 +1,18 @@
 #if !defined(LIQUID_VARIABLE_H)
 #define LIQUID_VARIABLE_H
 
+#include "vm_assembler.h"
+
+typedef struct variable_parse_args {
+    const char *markup;
+    const char *markup_end;
+    vm_assembler_t *code;
+    VALUE parse_context;
+    unsigned int line_number;
+} variable_parse_args_t;
+
 void init_liquid_variable(void);
+void internal_variable_parse(variable_parse_args_t *parse_args);
 
 #endif