Fix non-ascii string/binary literals; Add multi-line binary literals

src/lexer.mll

@@ -143,6 +143,8 @@ rule token = parse
  let (rng, s) = string_literal posL strbuf lexbuf in
  match MyUtil.Utf.uchar_of_utf8 s with
  | [ uchar ] -> CHAR(rng, uchar)
+ (* TODO: (?) allow escape sequences in char literal,
+ e.g. $'\n' (erlang $\n) or $'\x{10ffff}' (erlang $\x{10ffff}) *)
  | _ -> raise_error (NotASingleCodePoint(rng))
  }
 
@@ -204,10 +206,22 @@ rule token = parse
  let posL = Range.from_lexbuf lexbuf in
  let strbuf = Buffer.create 128 in
  let (rng, s) = string_literal posL strbuf lexbuf in
+ (* TODO: handle multi-codepoint unicode characters e.g. 🤷🏽‍♀️
+ (raise_error NotASingleCodePoint or convert somehow) *)
  STRING(rng, s)
  }
 
- | ("`" +) {
+ | ("`"+ break) {
+ (* When first character in a string block is a line break,
+ ignore this line break *)
+ Lexing.new_line lexbuf;
+ let posL = Range.from_lexbuf lexbuf in
+ let num_start = String.length (String.trim (Lexing.lexeme lexbuf)) in
+ let strbuf = Buffer.create 128 in
+ string_block num_start posL strbuf lexbuf
+ }
+
+ | ("`"+) {
  let posL = Range.from_lexbuf lexbuf in
  let num_start = String.length (Lexing.lexeme lexbuf) in
  let strbuf = Buffer.create 128 in
@@ -219,15 +233,17 @@ rule token = parse
 and binary_literal posL strbuf = parse
  | break { raise_error (SeeBreakInStringLiteral(posL)) }
  | eof { raise_error (SeeEndOfFileInStringLiteral(posL)) }
+ | "\\\"" { Buffer.add_char strbuf '\\'; Buffer.add_char strbuf '"'; binary_literal posL strbuf lexbuf }
  | "\"" { let posR = Range.from_lexbuf lexbuf in (Range.unite posL posR, Buffer.contents strbuf) }
- | "\\\"" { Buffer.add_char strbuf '"'; binary_literal posL strbuf lexbuf }
  | _ as c { Buffer.add_char strbuf c; binary_literal posL strbuf lexbuf }
 
 and string_literal posL strbuf = parse
  | break { raise_error (SeeBreakInStringLiteral(posL)) }
  | eof { raise_error (SeeEndOfFileInStringLiteral(posL)) }
+ | "\\\'" { Buffer.add_char strbuf '\\'; Buffer.add_char strbuf '\''; string_literal posL strbuf lexbuf }
+ | "\\\"" { Buffer.add_char strbuf '\\'; Buffer.add_char strbuf '\"'; string_literal posL strbuf lexbuf }
+ | "\"" { Buffer.add_char strbuf '\\'; Buffer.add_char strbuf '\"'; string_literal posL strbuf lexbuf }
  | "\'" { let posR = Range.from_lexbuf lexbuf in (Range.unite posL posR, Buffer.contents strbuf) }
- | "\\\'" { Buffer.add_char strbuf '\''; string_literal posL strbuf lexbuf }
  | _ as c { Buffer.add_char strbuf c; string_literal posL strbuf lexbuf }
 
 and format_literal posL strbuf acc = parse

src/outputErlangCode.ml

@@ -167,9 +167,9 @@ let stringify_base_constant (bc : base_constant) =
  else
  assert false
 
- | BinaryByString(s) -> Printf.sprintf "<<\"%s\">>" (String.escaped s)
+ | BinaryByString(s) -> Printf.sprintf "<<\"%s\"/utf8>>" s
  | BinaryByInts(ns) -> Printf.sprintf "<<%s>>" (ns |> List.map string_of_int |> String.concat ", ")
- | String(s) -> Printf.sprintf "\"%s\"" (String.escaped s)
+ | String(s) -> Printf.sprintf "\"%s\"" s
  | Char(uchar) -> Printf.sprintf "%d" (Uchar.to_int uchar)
 
  | FormatString(fmtelems) ->

src/parser.mly

@@ -677,10 +677,14 @@ exprbot:
  let rng = make_range (Token(tokL)) (Token(tokR)) in
  (rng, BinaryByList(ns))
  }
- | strlit=BINARY {
- let (rng, s) = strlit in
+ | binlit=BINARY {
+ let (rng, s) = binlit in
  (rng, BaseConst(BinaryByString(s)))
  }
+ | strblock=STRING_BLOCK {
+ let (rng, s) = strblock in
+ (rng, BaseConst(BinaryByInts(s |> String.to_seq |> List.of_seq |> List.map Char.code)))
+ }
  | strlit=STRING {
  let (rng, s) = strlit in
  (rng, BaseConst(String(s)))

test/pass/test_binary.sest

@@ -0,0 +1,20 @@
+module TestBinary = struct
+
+ val check : fun({binary, binary, binary, binary}) -> {binary, binary, binary, binary} = external 1 ```
+ check(X) ->
+ {<<"👩‍🔬"/utf8>>,<<"\n"/utf8>>,<<33, 96, 92, 92>>,<<111, 110, 101, 10, 116, 119, 111>>} = X,
+ X.
+ ```
+
+ val main(args) =
+ let woman_scientist = "👩‍🔬" in
+ let newline = "\n" in
+ let raw = ``!`\\`` in
+ let multiline = ```
+one
+two```
+ in
+ let result = {woman_scientist, newline, raw, multiline} in
+ print_debug(check({woman_scientist, newline, raw, multiline}))
+
+end