Improving prefix-string parsing (fixes lcompilers#2339) (lcompilers#2511

)

src/lpython/parser/parser.yy

@@ -179,6 +179,9 @@ void yyerror(YYLTYPE *yyloc, LCompilers::LPython::Parser &p, const std::string &
 %token KW_MATCH
 %token KW_CASE
 
+%token KW_STR_PREFIX
+%type <string> KW_STR_PREFIX
+
 // Nonterminal tokens
 
 %type <ast> script_unit
@@ -1102,9 +1105,9 @@ subscript
 
 string
  : string TK_STRING { $$ = STRING2($1, $2, @$); } // TODO
- | string id TK_STRING { $$ = STRING4($1, STRING3($2, $3, @$), @$); }
+ | string KW_STR_PREFIX TK_STRING { $$ = STRING4($1, STRING3($2, $3, @$), @$); }
  | TK_STRING { $$ = STRING1($1, @$); }
- | id TK_STRING { $$ = STRING3($1, $2, @$); }
+ | KW_STR_PREFIX TK_STRING { $$ = STRING3($1, $2, @$); }
  ;
 
 lambda_parameter

src/lpython/parser/semantics.h

@@ -800,7 +800,7 @@ static inline ast_t* concat_string(Allocator &al, Location &l,
 #define INTEGER(x, l) make_ConstantInt_t(p.m_a, l, x, nullptr)
 #define STRING1(x, l) make_ConstantStr_t(p.m_a, l, str_unescape_c(p.m_a, x), nullptr)
 #define STRING2(x, y, l) concat_string(p.m_a, l, EXPR(x), str_unescape_c(p.m_a, y), nullptr)
-#define STRING3(id, x, l) PREFIX_STRING(p.m_a, l, name2char(id), x.c_str(p.m_a))
+#define STRING3(prefix, x, l) PREFIX_STRING(p.m_a, l, prefix.c_str(p.m_a), x.c_str(p.m_a))
 #define STRING4(x, s, l) concat_string(p.m_a, l, EXPR(x), "", EXPR(s))
 #define FLOAT(x, l) make_ConstantFloat_t(p.m_a, l, x, nullptr)
 #define COMPLEX(x, l) make_ConstantComplex_t(p.m_a, l, 0, x, nullptr)

src/lpython/parser/tokenizer.re

@@ -435,6 +435,19 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
  }
  }
 
+ [rR][bB] | [bB][rR]
+ | [fF][rR] | [rR][fF]
+ | [rR] | [bB] | [fF] | [uU]
+ {
+ if(cur[0] == '\'' || cur[0] == '"'){
+ KW(STR_PREFIX);
+ }
+ else {
+ token(yylval.string);
+ RET(TK_NAME);
+ }
+ }
+
  // Tokens
  newline {
  if(parenlevel) { continue; }
@@ -763,6 +776,7 @@ std::string token2text(const int token)
 
  T(KW_MATCH, "match")
  T(KW_CASE, "case")
+ T(KW_STR_PREFIX, "string prefix")
 
  default : {
  std::cout << "TOKEN: " << token << std::endl;

tests/errors/prefix_string_01.py

@@ -0,0 +1,5 @@
+def main():
+ # python2 syntax should result in a syntax error
+ print "Hello", "World" 
+
+main()

tests/errors/prefix_string_02.py

@@ -0,0 +1,9 @@
+from lpython import i32
+
+# fix difference between lpython and cpython in prefix string grammar
+# Prefix should be attached to the quote without any whitespace.
+
+def main():
+ print(r "Hello World")
+
+main()

tests/reference/ast-prefix_string_01-cf221fd.json

@@ -0,0 +1,13 @@
+{
+ "basename": "ast-prefix_string_01-cf221fd",
+ "cmd": "lpython --show-ast --no-color {infile} -o {outfile}",
+ "infile": "tests/errors/prefix_string_01.py",
+ "infile_hash": "0d83c0e32a78023fccb343a4d3358071792265c1ae357176fe0912eb",
+ "outfile": null,
+ "outfile_hash": null,
+ "stdout": null,
+ "stdout_hash": null,
+ "stderr": "ast-prefix_string_01-cf221fd.stderr",
+ "stderr_hash": "b600057f41f59ba7fdebe3971bfea0eadca972747ccf70d575c1cdcd",
+ "returncode": 1
+}

tests/reference/ast-prefix_string_01-cf221fd.stderr

@@ -0,0 +1,5 @@
+syntax error: Token '"Hello"' (of type 'string') is unexpected here
+ --> tests/errors/prefix_string_01.py:3:11
+ |
+3 | print "Hello", "World" 
+ | ^^^^^^^ 

tests/reference/ast-prefix_string_02-3d530b2.json

@@ -0,0 +1,13 @@
+{
+ "basename": "ast-prefix_string_02-3d530b2",
+ "cmd": "lpython --show-ast --no-color {infile} -o {outfile}",
+ "infile": "tests/errors/prefix_string_02.py",
+ "infile_hash": "5d0c279ea735e60d5243a4b33100832dc1564917d6ef83c9b32705f9",
+ "outfile": null,
+ "outfile_hash": null,
+ "stdout": null,
+ "stdout_hash": null,
+ "stderr": "ast-prefix_string_02-3d530b2.stderr",
+ "stderr_hash": "cd72affed29823c0364d52bfb3ba0674d9d7950390b7cd6b04f7538b",
+ "returncode": 1
+}

tests/reference/ast-prefix_string_02-3d530b2.stderr

@@ -0,0 +1,5 @@
+syntax error: Token '"Hello World"' (of type 'string') is unexpected here
+ --> tests/errors/prefix_string_02.py:7:13
+ |
+7 | print(r "Hello World")
+ | ^^^^^^^^^^^^^ 

tests/tests.toml

@@ -1286,6 +1286,14 @@ asr = true
 filename = "errors/unsigned_04.py"
 asr = true
 
+[[test]]
+filename = "errors/prefix_string_01.py"
+ast = true
+
+[[test]]
+filename = "errors/prefix_string_02.py"
+ast = true
+
 # tests/runtime_errors
 [[test]]
 filename = "runtime_errors/test_list_01.py"