Fix non-ascii string/binary literals; Add multi-line binary literals

src/errors.ml

@@ -30,6 +30,7 @@ type lexer_error =
  | BlockClosedWithTooManyBackQuotes of Range.t
  | SeeBreakInStringLiteral of Range.t
  | NotASingleCodePoint of Range.t
+ | UnknownEscapeSequence of Range.t
 
 type syntax_error =
  | LexerError of lexer_error

src/lexer.mll

@@ -33,6 +33,14 @@
  Buffer.clear strbuf;
  FormatConst(s)
 
+
+ let escape_sequence c rngL = match c with
+ | 'n' -> '\n'
+ | 'r' -> '\r'
+ | 't' -> '\t'
+ | '\\' | '"' | '\'' -> c
+ | _ -> raise_error (UnknownEscapeSequence(rngL))
+
 }
 
 let space = [' ' '\t']
@@ -207,7 +215,17 @@ rule token = parse
  STRING(rng, s)
  }
 
- | ("`" +) {
+ | ("`"+ break) {
+ (* When first character in a string block is a line break,
+ ignore this line break *)
+ Lexing.new_line lexbuf;
+ let posL = Range.from_lexbuf lexbuf in
+ let num_start = String.length (String.trim (Lexing.lexeme lexbuf)) in
+ let strbuf = Buffer.create 128 in
+ string_block num_start posL strbuf lexbuf
+ }
+
+ | ("`"+) {
  let posL = Range.from_lexbuf lexbuf in
  let num_start = String.length (Lexing.lexeme lexbuf) in
  let strbuf = Buffer.create 128 in
@@ -219,15 +237,19 @@ rule token = parse
 and binary_literal posL strbuf = parse
  | break { raise_error (SeeBreakInStringLiteral(posL)) }
  | eof { raise_error (SeeEndOfFileInStringLiteral(posL)) }
+ | ("\\" (_ as c)) {
+ Buffer.add_char strbuf (escape_sequence c posL); binary_literal posL strbuf lexbuf
+ }
  | "\"" { let posR = Range.from_lexbuf lexbuf in (Range.unite posL posR, Buffer.contents strbuf) }
- | "\\\"" { Buffer.add_char strbuf '"'; binary_literal posL strbuf lexbuf }
  | _ as c { Buffer.add_char strbuf c; binary_literal posL strbuf lexbuf }
 
 and string_literal posL strbuf = parse
  | break { raise_error (SeeBreakInStringLiteral(posL)) }
  | eof { raise_error (SeeEndOfFileInStringLiteral(posL)) }
+ | ("\\" (_ as c)) {
+ Buffer.add_char strbuf (escape_sequence c posL); string_literal posL strbuf lexbuf
+ }
  | "\'" { let posR = Range.from_lexbuf lexbuf in (Range.unite posL posR, Buffer.contents strbuf) }
- | "\\\'" { Buffer.add_char strbuf '\''; string_literal posL strbuf lexbuf }
  | _ as c { Buffer.add_char strbuf c; string_literal posL strbuf lexbuf }
 
 and format_literal posL strbuf acc = parse

src/logging.ml

@@ -76,6 +76,10 @@ let report_lexer_error (e : lexer_error) : unit =
  Format.printf "%a: not a single code point\n"
  Range.pp rng
 
+ | UnknownEscapeSequence(rngL) ->
+ Format.printf "%a: unknown escape sequence \n"
+ Range.pp rngL
+
 
 let report_config_error (e : config_error) : unit =
  Format.printf "! [Build error] ";

src/outputErlangCode.ml

@@ -151,6 +151,20 @@ let stringify_format_element = function
  in
  (1, Printf.sprintf "~%s%s" s ch)
 
+let escape_character c =
+ match Uchar.to_int c with
+ | 10 -> [ Uchar.of_char '\\'; Uchar.of_char 'n' ]
+ | 13 -> [ Uchar.of_char '\\'; Uchar.of_char 'r' ]
+ | 9 -> [ Uchar.of_char '\\'; Uchar.of_char 't' ]
+ | 92 -> [ Uchar.of_char '\\'; Uchar.of_char '\\' ]
+ | 34 -> [ Uchar.of_char '\\'; Uchar.of_char '"' ]
+ | 39 -> [ Uchar.of_char '\\'; Uchar.of_char '\'' ]
+ | _ -> [c]
+
+let escape_string s =
+ let buffer = Buffer.create 0 in
+ s |> MyUtil.Utf.uchar_of_utf8 |> List.map escape_character |> List.flatten |> List.iter (Buffer.add_utf_8_uchar buffer);
+ Buffer.contents buffer
 
 let stringify_base_constant (bc : base_constant) =
  match bc with
@@ -167,9 +181,9 @@ let stringify_base_constant (bc : base_constant) =
  else
  assert false
 
- | BinaryByString(s) -> Printf.sprintf "<<\"%s\">>" (String.escaped s)
+ | BinaryByString(s) -> Printf.sprintf "<<\"%s\"/utf8>>" (escape_string s)
  | BinaryByInts(ns) -> Printf.sprintf "<<%s>>" (ns |> List.map string_of_int |> String.concat ", ")
- | String(s) -> Printf.sprintf "\"%s\"" (String.escaped s)
+ | String(s) -> Printf.sprintf "\"%s\"" (escape_string s)
  | Char(uchar) -> Printf.sprintf "%d" (Uchar.to_int uchar)
 
  | FormatString(fmtelems) ->

src/parser.mly

@@ -677,10 +677,14 @@ exprbot:
  let rng = make_range (Token(tokL)) (Token(tokR)) in
  (rng, BinaryByList(ns))
  }
- | strlit=BINARY {
- let (rng, s) = strlit in
+ | binlit=BINARY {
+ let (rng, s) = binlit in
  (rng, BaseConst(BinaryByString(s)))
  }
+ | strblock=STRING_BLOCK {
+ let (rng, s) = strblock in
+ (rng, BaseConst(BinaryByInts(s |> String.to_seq |> List.of_seq |> List.map Char.code)))
+ }
  | strlit=STRING {
  let (rng, s) = strlit in
  (rng, BaseConst(String(s)))

test/pass/test_binary.sest

@@ -0,0 +1,30 @@
+module TestBinary = struct
+
+ val check : fun({binary, binary, binary, binary, binary}) -> {binary, binary, binary, binary, binary} = external 1 ```
+ check({A, B, C, D, E}) ->
+ <<240,159,145,169,226,128,141,240,159,148,172>> = A,
+ <<"👩‍🔬"/utf8>> = A,
+ <<10,13,9,34,39,92>> = B,
+ <<"\n\r\t\"\'\\"/utf8>> = B,
+ <<39>> = C,
+ <<"\'"/utf8>> = C,
+ <<33,34,39,96,92,92>> = D,
+ <<"!\"\'`\\\\"/utf8>> = D,
+ <<111,110,101,10,116,119,111>> = E,
+ <<"one\ntwo"/utf8>> = E,
+ {A, B, C, D, E}.
+ ```
+
+ val main(args) =
+ let woman_scientist = "👩‍🔬" in
+ let escape_sequences = "\n\r\t\"\'\\" in
+ let single_quote = "'" in
+ let raw = ``!"'`\\`` in
+ let multiline = ```
+one
+two```
+ in
+ let examples = {woman_scientist, escape_sequences, single_quote, raw, multiline} in
+ print_debug(check(examples))
+
+end