Support lexing multibyte unicode chars (poorly, but it works)

Author: bleibig

lexer.l

@@ -28,13 +28,22 @@ static int saw_non_hash;
 %x doc_block
 %x suffix
 
+ident [a-zA-Z\x80-\xff_][a-zA-Z0-9\x80-\xff_]*
+
 %%
 
-<suffix>[a-zA-Z_][a-zA-Z0-9_]*      { BEGIN(INITIAL); }
-<suffix>(.|\n)                      { yyless(0); BEGIN(INITIAL); }
+<suffix>{ident}            { BEGIN(INITIAL); }
+<suffix>(.|\n)  { yyless(0); BEGIN(INITIAL); }
 
 [ \n\t\r]             { }
 
+\xef\xbb\xbf {
+  // UTF-8 byte order mark (BOM), ignore if in line 1, error otherwise
+  if (yyget_lineno() != 1) {
+    return -1;
+  }
+}
+
 \/\/(\/|\!)           { BEGIN(doc_line); yymore(); }
 <doc_line>\n          { BEGIN(INITIAL);
                         yyleng--;
@@ -104,7 +113,7 @@ use      { return USE; }
 where    { return WHERE; }
 while    { return WHILE; }
 
-[a-zA-Z_][a-zA-Z0-9_]*      { return IDENT; }
+{ident}  { return IDENT; }
 
 0x[0-9a-fA-F_]+                                    { BEGIN(suffix); return LIT_INTEGER; }
 0o[0-8_]+                                          { BEGIN(suffix); return LIT_INTEGER; }
@@ -172,11 +181,12 @@ while    { return WHILE; }
 
 \x27                                  { BEGIN(ltorchar); yymore(); }
 <ltorchar>static                      { BEGIN(INITIAL); return STATIC_LIFETIME; }
-<ltorchar>[a-zA-Z_][a-zA-Z0-9_]*      { BEGIN(INITIAL); return LIFETIME; }
+<ltorchar>{ident}                     { BEGIN(INITIAL); return LIFETIME; }
 <ltorchar>\\[nrt\\\x27\x220]\x27      { BEGIN(suffix); return LIT_CHAR; }
 <ltorchar>\\x[0-9a-fA-F]{2}\x27       { BEGIN(suffix); return LIT_CHAR; }
 <ltorchar>\\u\{[0-9a-fA-F]?{6}\}\x27  { BEGIN(suffix); return LIT_CHAR; }
 <ltorchar>.\x27                       { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>[\x80-\xff]{2,4}\x27        { BEGIN(suffix); return LIT_CHAR; }
 <ltorchar><<EOF>>                     { BEGIN(INITIAL); return -1; }
 
 b\x22              { BEGIN(bytestr); yymore(); }

rlex.rs

@@ -35,7 +35,7 @@ fn token_to_string(tok: token::Token) -> String {
                 format!("Float({})", c.as_str().to_string())
             },
             Lit::Str_(s) => {
-                format!("Str(\"{}\")", token::get_name(s).get().escape_default())
+                format!("Str(\"{}\")", token::get_name(s).get())
             },
             Lit::StrRaw(s, n) => {
                 format!("StrRaw(r{delim}\"{string}\"{delim})",

tokens.c

@@ -9,7 +9,6 @@ extern char *yytext;
 
 static char *binop_text(char*);
 static char *desugar_num(char*, char*);
-static char *escape_string(char*);
 
 void print_token(int token) {
   switch (token) {
@@ -58,7 +57,7 @@ void print_token(int token) {
   case LIT_CHAR: printf("Char(%s)", yytext); break;
   case LIT_INTEGER: printf("Integer(%s)", yytext); break;
   case LIT_FLOAT: printf("Float(%s)", yytext); break;
-  case LIT_STR: printf("Str(%s)", escape_string(yytext)); break;
+  case LIT_STR: printf("Str(%s)", yytext); break;
   case LIT_STR_RAW: printf("StrRaw(%s)", yytext); break;
   case LIT_BINARY: printf("Binary(%s)", yytext); break;
   case LIT_BINARY_RAW: printf("BinaryRaw(%s)", yytext); break;
@@ -225,44 +224,3 @@ static char *desugar_num(char *tok, char *default_suffix) {
     }
     return res;
 }
-
-static char *escape_string(char *str) {
-  int len = strlen(str);
-  char *res = malloc(sizeof(char) * len * 2 + 1);
-  int j = 0;
-  res[j++] = '"';
-  for (int i = 1; i < len - 1; ++i) {
-    char c = str[i];
-    switch (c) {
-    case '\t':
-      res[j++] = '\\';
-      res[j++] = 't';
-      break;
-    case '\r':
-      res[j++] = '\\';
-      res[j++] = 'r';
-      break;
-    case '\n':
-      res[j++] = '\\';
-      res[j++] = 'n';
-      break;
-    case '\\':
-      res[j++] = '\\';
-      res[j++] = '\\';
-      break;
-    case '\'':
-      res[j++] = '\\';
-      res[j++] = '\'';
-      break;
-    case '\"':
-      res[j++] = '\\';
-      res[j++] = '\"';
-      break;
-    default:
-      res[j++] = c;
-    }
-  }
-  res[j++] = '"';
-  res[j++] = '\0';
-  return res;
-}

verify-lexer.py

@@ -31,9 +31,7 @@ def compare(p):
     for f in filter(lambda p: p.endswith('.rs'), files):
         p = os.path.join(base, f)
         # compile-fail programs should be ignored
-        # also, the lexer doesn't work with multibyte characters so
-        # ignore programs that contain them
-        if "compile-fail" in p or not all(ord(c) < 128 for c in open(p).read()):
+        if "compile-fail" in p:
             print("skipping {}".format(p))
             continue
         print("comparing {}".format(p))