fix unicode characters error locations (#797)

Closes #742

.changeset/empty-lies-fetch.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": patch
+---
+
+fix source locations for unicode characters in error reports

crates/codegen/parser/runtime/src/parse_error.rs

@@ -1,7 +1,7 @@
 use std::collections::BTreeSet;
 
 use crate::kinds::TokenKind;
-use crate::text_index::{TextIndex, TextRange};
+use crate::text_index::{TextIndex, TextRange, TextRangeExtensions};
 
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub struct ParseError {
@@ -64,8 +64,6 @@ pub(crate) fn render_error_report(
 
     let kind = ReportKind::Error;
     let color = if with_color { Color::Red } else { Color::Unset };
-    let source_start = error.text_range.start.utf8;
-    let source_end = error.text_range.end.utf8;
 
     let tokens_that_would_have_allowed_more_progress =
         error.tokens_that_would_have_allowed_more_progress();
@@ -82,12 +80,14 @@ pub(crate) fn render_error_report(
         return format!("{kind}: {message}\n   ─[{source_id}:0:0]");
     }
 
-    let mut builder = Report::build(kind, source_id, source_start)
+    let range = error.text_range.char();
+
+    let mut builder = Report::build(kind, source_id, range.start)
         .with_config(Config::default().with_color(with_color))
         .with_message(message);
 
     builder.add_label(
-        Label::new((source_id, source_start..source_end))
+        Label::new((source_id, range))
             .with_color(color)
             .with_message("Error occurred here.".to_string()),
     );

crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml

@@ -0,0 +1,22 @@
+# This file is generated automatically by infrastructure scripts. Please don't edit by hand.
+
+Source: >
+  1  │ // ╒════════════════════════════════════════════════════════════════╕            │ 0..201
+  2  │ // │ More Info: https://github.com/NomicFoundation/slang/issues/742 │            │ 202..275
+  3  │ // ╘════════════════════════════════════════════════════════════════╛            │ 276..477
+  4  │ unexpected                                                                       │ 478..488
+
+Errors: # 1 total
+  - >
+    Error: Expected ContractKeyword.
+       ╭─[crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/input.sol:1:1]
+       │
+     1 │ ╭─▶ // ╒════════════════════════════════════════════════════════════════╕
+       ┆ ┆   
+     4 │ ├─▶ unexpected
+       │ │                
+       │ ╰──────────────── Error occurred here.
+    ───╯
+
+Tree:
+  - (SKIPPED): "// ╒══════════════════════════════════════════════..." # (0..489)

crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/input.sol

@@ -0,0 +1,4 @@
+// ╒════════════════════════════════════════════════════════════════╕
+// │ More Info: https://github.com/NomicFoundation/slang/issues/742 │
+// ╘════════════════════════════════════════════════════════════════╛
+unexpected

crates/solidity/testing/snapshots/cst_output/Expression/unicode_string_literal/generated/0.4.11-failure.yml

@@ -8,7 +8,9 @@ Errors: # 1 total
     Error: Expected Ampersand or AmpersandAmpersand or AmpersandEqual or Asterisk or AsteriskAsterisk or AsteriskEqual or BangEqual or Bar or BarBar or BarEqual or Caret or CaretEqual or Equal or EqualEqual or GreaterThan or GreaterThanEqual or GreaterThanGreaterThan or GreaterThanGreaterThanEqual or GreaterThanGreaterThanGreaterThan or GreaterThanGreaterThanGreaterThanEqual or LessThan or LessThanEqual or LessThanLessThan or LessThanLessThanEqual or Minus or MinusEqual or Percent or PercentEqual or Plus or PlusEqual or Slash or SlashEqual.
        ╭─[crates/solidity/testing/snapshots/cst_output/Expression/unicode_string_literal/input.sol:1:8]
        │
-     1 │ ╭─▶ unicode"This Emoji: 😃"
+     1 │ unicode"This Emoji: 😃"
+       │        ───────┬────────  
+       │               ╰────────── Error occurred here.
     ───╯
 
 Tree:

crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/multiple/generated/0.4.11-failure.yml

@@ -8,7 +8,9 @@ Errors: # 1 total
     Error: Expected end of file.
        ╭─[crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/multiple/input.sol:1:1]
        │
-     1 │ ╭─▶ unicode"happy 😃" unicode'sad 😔'
+     1 │ unicode"happy 😃" unicode'sad 😔'
+       │ ────────────────┬────────────────  
+       │                 ╰────────────────── Error occurred here.
     ───╯
 
 Tree:

crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/single/generated/0.4.11-failure.yml

@@ -8,7 +8,9 @@ Errors: # 1 total
     Error: Expected end of file.
        ╭─[crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/single/input.sol:1:1]
        │
-     1 │ ╭─▶ unicode"emoji 😃"
+     1 │ unicode"emoji 😃"
+       │ ────────┬────────  
+       │         ╰────────── Error occurred here.
     ───╯
 
 Tree:

crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/single_trailing_ident/generated/0.4.11-failure.yml

@@ -8,7 +8,9 @@ Errors: # 1 total
     Error: Expected end of file.
        ╭─[crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/single_trailing_ident/input.sol:1:1]
        │
-     1 │ ╭─▶ unicode"emoji 😃"happy
+     1 │ unicode"emoji 😃"happy
+       │ ───────────┬───────────  
+       │            ╰───────────── Error occurred here.
     ───╯
 
 Tree:

crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/single_trailing_ident/generated/0.7.0-failure.yml

@@ -6,9 +6,11 @@ Source: >
 Errors: # 1 total
   - >
     Error: Expected UnicodeStringLiteral.
-       ╭─[crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/single_trailing_ident/input.sol:1:20]
+       ╭─[crates/solidity/testing/snapshots/cst_output/UnicodeStringLiterals/single_trailing_ident/input.sol:1:17]
        │
-     1 │ ╭─▶ unicode"emoji 😃"happy
+     1 │ unicode"emoji 😃"happy
+       │                  ───┬──  
+       │                     ╰──── Error occurred here.
     ───╯
 
 Tree:

crates/solidity/testing/utils/src/cst_snapshots/mod.rs

@@ -1,4 +1,4 @@
-use std::cmp::max;
+use std::cmp::{max, min};
 use std::collections::HashSet;
 use std::fmt::Write;
 use std::ops::Range;
@@ -149,29 +149,29 @@ fn render_key(cursor: &mut CursorWithNames) -> String {
 }
 
 fn render_value(cursor: &mut CursorWithNames, source: &str) -> String {
-    let range = cursor.text_range().utf8();
-    let preview = render_preview(source, &range);
+    let utf8_range = cursor.text_range().utf8();
+    let char_range = cursor.text_range().char();
+    let preview = render_preview(source, &char_range);
 
     match cursor.node() {
-        Node::Rule(rule) if rule.children.is_empty() => format!("[] # ({range:?})"),
-        Node::Rule(_) => format!("# {preview} ({range:?})"),
-        Node::Token(_) => format!("{preview} # ({range:?})"),
+        Node::Rule(rule) if rule.children.is_empty() => format!("[] # ({utf8_range:?})"),
+        Node::Rule(_) => format!("# {preview} ({utf8_range:?})"),
+        Node::Token(_) => format!("{preview} # ({utf8_range:?})"),
     }
 }
 
-fn render_preview(source: &str, range: &Range<usize>) -> String {
-    let length = range.len();
+fn render_preview(source: &str, char_range: &Range<usize>) -> String {
+    let length = char_range.len();
 
     // Trim long values:
     let max_length = 50;
-    let contents = source
-        .bytes()
-        .skip(range.start)
-        .take(length.clamp(0, max_length))
+    let mut contents: String = source
+        .chars()
+        .skip(char_range.start)
+        .take(min(length, max_length))
         .collect();
 
     // Add terminator if trimmed:
-    let mut contents = String::from_utf8(contents).unwrap();
     if length > max_length {
         contents.push_str("...");
     }