strange error, tokenizer?

[drsm]

>> 2.toString()
SyntaxError: Unexpected token "toString" in 1
>> (function weird() { return 2.toString(); })()
2

[hongzhidao]

Hi, try this patch.

diff -r d2cbea77122c njs/njs_parser.c
--- a/njs/njs_parser.c	Thu Aug 02 19:39:55 2018 +0300
+++ b/njs/njs_parser.c	Sun Aug 05 05:13:27 2018 -0400
@@ -311,12 +311,6 @@
     case NJS_TOKEN_FUNCTION:
         return njs_parser_function_declaration(vm, parser);

-    case NJS_TOKEN_RETURN:
-        return njs_parser_return_statement(vm, parser);
-
-    case NJS_TOKEN_VAR:
-        return njs_parser_var_statement(vm, parser);
-
     case NJS_TOKEN_IF:
         return njs_parser_if_statement(vm, parser);

@@ -341,9 +335,6 @@
     case NJS_TOKEN_TRY:
         return njs_parser_try_statement(vm, parser);

-    case NJS_TOKEN_THROW:
-        return njs_parser_throw_statement(vm, parser);
-
     case NJS_TOKEN_SEMICOLON:
         return njs_parser_token(parser);

@@ -360,7 +351,26 @@
         /* Fall through. */

     default:
-        token = njs_parser_expression(vm, parser, token);
+
+        switch (token) {
+
+        case NJS_TOKEN_RETURN:
+            token = njs_parser_return_statement(vm, parser);
+            break;
+
+        case NJS_TOKEN_VAR:
+            token = njs_parser_var_statement(vm, parser);
+            break;
+
+        case NJS_TOKEN_THROW:
+            token = njs_parser_throw_statement(vm, parser);
+            break;
+
+        default:
+            token = njs_parser_expression(vm, parser, token);
+            break;
+        }
+
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -794,10 +804,6 @@
         node->right = parser->node;
         parser->node = node;

-        if (token == NJS_TOKEN_SEMICOLON) {
-            return njs_parser_token(parser);
-        }
-
         return token;
     }
 }
@@ -888,26 +894,7 @@

     } while (token == NJS_TOKEN_COMMA);

-    /*
-     * A var statement must be terminated by semicolon,
-     * or by a close curly brace or by the end of line.
-     */
-    switch (token) {
-
-    case NJS_TOKEN_SEMICOLON:
-        return njs_parser_token(parser);
-
-    case NJS_TOKEN_CLOSE_BRACE:
-    case NJS_TOKEN_END:
-        return token;
-
-    default:
-        if (parser->lexer->prev_token == NJS_TOKEN_LINE_END) {
-            return token;
-        }
-
-        return NJS_TOKEN_ILLEGAL;
-    }
+    return token;
 }


diff -r d2cbea77122c njs/njs_parser_expression.c
--- a/njs/njs_parser_expression.c	Thu Aug 02 19:39:55 2018 +0300
+++ b/njs/njs_parser_expression.c	Sun Aug 05 05:13:27 2018 -0400
@@ -917,6 +917,7 @@
 }


+#include <stdio.h>
 static njs_token_t
 njs_parser_call_expression(njs_vm_t *vm, njs_parser_t *parser,
     njs_token_t token)
diff -r d2cbea77122c njs/test/njs_unit_test.c
--- a/njs/test/njs_unit_test.c	Thu Aug 02 19:39:55 2018 +0300
+++ b/njs/test/njs_unit_test.c	Sun Aug 05 05:13:27 2018 -0400
@@ -2042,7 +2042,7 @@
     { nxt_string("var a = [3], b; if (1==1||2==2) { b = '1'+'2'+a[0] }; b"),
       nxt_string("123") },

-    { nxt_string("(function(){ if(true) return 1 else return 0; })()"),
+    { nxt_string("(function(){ if(true) return 1; else return 0; })()"),
       nxt_string("1") },

     { nxt_string("(function(){ if(true) return 1; else return 0; })()"),
@@ -4884,7 +4884,7 @@

     { nxt_string("function f(a) {"
                  "    if (a > 1)"
-                 "        return a * f(a - 1)"
+                 "        return a * f(a - 1)\n"
                  "    return 1"
                  "}"
                  "f(10)"),
@@ -4969,7 +4969,7 @@

     { nxt_string("function fibo(n) {"
                  "    if (n > 1)"
-                 "        return fibo(n-1) + fibo(n-2)"
+                 "        return fibo(n-1) + fibo(n-2)\n"
                  "     return 1"
                  "}"
                  "fibo(10)"),
@@ -4977,7 +4977,7 @@

     { nxt_string("function fibo(n) {"
                  "    if (n > 1)"
-                 "        return fibo(n-1) + fibo(n-2)"
+                 "        return fibo(n-1) + fibo(n-2)\n"
                  "     return '.'"
                  "}"
                  "fibo(10).length"),
@@ -4985,7 +4985,7 @@

     { nxt_string("function fibo(n) {"
                  "    if (n > 1)"
-                 "        return fibo(n-1) + fibo(n-2)"
+                 "        return fibo(n-1) + fibo(n-2)\n"
                  "     return 1"
                  "}"
                  "fibo('10')"),

[hongzhidao]

1. I think var/return/throw/expression statements can be treated as the same which could generate value.
2. The statement of "if (1) return 1 else return 2" is invalid.
   @xeioex

[xeioex]

@hongzhidao Thank you for the patch, I will look at it this week.