Fully switch to byte based offsets

CHANGELOG.md

@@ -15,6 +15,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [0.25.0] - 2024-07-24
+
+### Added
+
+### Fixed
+
+* panics when ui_test tried to show diagnostics on multi-byte chars
+
+### Changed
+
+### Removed
+
 
 ## [0.24.0] - 2024-07-11
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ui_test"
-version = "0.24.0"
+version = "0.25.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "A test framework for testing rustc diagnostics output"
@@ -28,7 +28,7 @@ indicatif = "0.17.6"
 prettydiff = { version = "0.7", default-features = false }
 annotate-snippets = { version = "0.11.2" }
 levenshtein = "1.0.5"
-spanned = "0.2.1"
+spanned = "0.3.0"
 
 [dependencies.regex]
 version = "1.5.5"

src/aux_builds.rs

@@ -29,7 +29,6 @@ impl Flag for AuxBuilder {
     ) -> Result<(), Errored> {
         let aux = &self.aux_file;
         let aux_dir = config.aux_dir;
-        let line = aux.line();
         let aux_file = if aux.starts_with("..") {
             aux_dir.parent().unwrap().join(&aux.content)
         } else {
@@ -60,9 +59,8 @@ impl Flag for AuxBuilder {
                  }| Errored {
                     command,
                     errors: vec![Error::Aux {
-                        path: aux_file.to_path_buf(),
+                        path: Spanned::new(aux_file.to_path_buf(), aux.span()),
                         errors,
-                        line,
                     }],
                     stderr,
                     stdout,
@@ -84,14 +82,13 @@ pub struct AuxBuilder {
 impl Build for AuxBuilder {
     fn build(&self, build_manager: &BuildManager<'_>) -> Result<Vec<OsString>, Errored> {
         let mut config = build_manager.config().clone();
-        let file_contents = Spanned::read_from_file(&self.aux_file.content)
-            .map_err(|err| Errored {
+        let file_contents =
+            Spanned::read_from_file(&self.aux_file.content).map_err(|err| Errored {
                 command: format!("reading aux file `{}`", display(&self.aux_file)),
                 errors: vec![],
                 stderr: err.to_string().into_bytes(),
                 stdout: vec![],
-            })?
-            .map(|s| s.into_bytes());
+            })?;
         let comments = Comments::parse(file_contents.as_ref(), &config)
             .map_err(|errors| Errored::new(errors, "parse aux comments"))?;
         assert_eq!(

src/custom_flags/run.rs

@@ -1,11 +1,8 @@
 //! Types used for running tests after they pass compilation
 
 use bstr::ByteSlice;
-use spanned::{Span, Spanned};
-use std::{
-    path::PathBuf,
-    process::{Command, Output},
-};
+use spanned::Spanned;
+use std::process::{Command, Output};
 
 use crate::{
     build_manager::BuildManager, display, per_test_config::TestConfig, Error, Errored, TestOk,
@@ -116,19 +113,25 @@ fn get_panic_span(stderr: &[u8]) -> Spanned<String> {
             let Ok(filename) = filename.to_str() else {
                 continue;
             };
-            let Ok(line) = line.parse() else {
+            let Ok(line) = line.parse::<usize>() else {
+                continue;
+            };
+            let Ok(col) = col.parse::<usize>() else {
+                continue;
+            };
+            let Ok(file) = Spanned::read_from_file(filename) else {
+                continue;
+            };
+            let Some(line) = line.checked_sub(1) else {
                 continue;
             };
-            let Ok(col) = col.parse() else {
+            let Some(line) = file.lines().nth(line) else {
                 continue;
             };
-            let span = Span {
-                file: PathBuf::from(filename),
-                line_start: line,
-                line_end: line,
-                col_start: col,
-                col_end: col,
+            let Some(col) = col.checked_sub(1) else {
+                continue;
             };
+            let span = line.span.inc_col_start(col);
             return Spanned::new(message.into(), span);
         }
     }

src/custom_flags/rustfix.rs

@@ -234,7 +234,7 @@ fn compile_fixed(
                             .iter()
                             .flatten()
                             .chain(diagnostics.messages_from_unknown_file_or_line.iter())
-                            .find_map(|message| message.line_col.clone())
+                            .find_map(|message| message.span.clone())
                             .unwrap_or_default(),
                     ),
                 }],

src/diagnostics.rs

@@ -56,7 +56,9 @@ pub struct Message {
     /// The main message of the diagnostic (what will be matched for with `//~`)
     pub message: String,
     /// Information about where in the file the message was emitted
-    pub line_col: Option<spanned::Span>,
+    pub line: Option<usize>,
+    /// Exact span information of the message
+    pub span: Option<spanned::Span>,
     /// Identifier of the message (E0XXX for rustc errors, or lint names)
     pub code: Option<String>,
 }

src/error.rs

@@ -56,7 +56,7 @@ pub enum Error {
         /// The main message of the error.
         msgs: Vec<Message>,
         /// File and line information of the error.
-        path: Option<Spanned<PathBuf>>,
+        path: Option<(PathBuf, NonZeroUsize)>,
     },
     /// A comment failed to parse.
     InvalidComment {
@@ -72,7 +72,7 @@ pub enum Error {
         /// The comment being looked for
         kind: String,
         /// The lines where conflicts happened
-        lines: Vec<NonZeroUsize>,
+        lines: Vec<Span>,
     },
     /// A subcommand (e.g. rustfix) of a test failed.
     Command {
@@ -86,11 +86,9 @@ pub enum Error {
     /// An auxiliary build failed with its own set of errors.
     Aux {
         /// Path to the aux file.
-        path: PathBuf,
+        path: Spanned<PathBuf>,
         /// The errors that occurred during the build of the aux file.
         errors: Vec<Error>,
-        /// The line in which the aux file was requested to be built.
-        line: NonZeroUsize,
     },
     /// An error occured applying [`rustfix`] suggestions
     Rustfix(anyhow::Error),

src/lib.rs

@@ -139,8 +139,7 @@ pub fn test_command(mut config: Config, path: &Path) -> Result<Command> {
     config.fill_host_and_target()?;
 
     let content = Spanned::read_from_file(path)
-        .wrap_err_with(|| format!("failed to read {}", display(path)))?
-        .map(|s| s.into_bytes());
+        .wrap_err_with(|| format!("failed to read {}", display(path)))?;
     let comments = Comments::parse(content.as_ref(), &config)
         .map_err(|errors| color_eyre::eyre::eyre!("{errors:#?}"))?;
     let config = TestConfig {
@@ -234,9 +233,7 @@ pub fn run_tests_generic(
         |receive, finished_files_sender| -> Result<()> {
             for (status, build_manager) in receive {
                 let path = status.path();
-                let file_contents = Spanned::read_from_file(path)
-                    .unwrap()
-                    .map(|s| s.into_bytes());
+                let file_contents = Spanned::read_from_file(path).unwrap();
                 let mut config = build_manager.config().clone();
                 per_file_config(&mut config, &file_contents);
                 let result = match std::panic::catch_unwind(|| {

src/parser.rs

@@ -59,7 +59,7 @@ impl Comments {
             }
             if let Some(found) = f(rev).into_inner() {
                 if result.is_some() {
-                    errors.push(found.line());
+                    errors.push(found.span);
                 } else {
                     result = found.into();
                 }
@@ -911,7 +911,7 @@ impl CommentParser<&mut Revisioned> {
                         self.error(pattern.span(), format!(
                             "//~^ pattern is trying to refer to {} lines above, but there are only {} lines above",
                             offset,
-                            pattern.line().get() - 1,
+                            current_line.get() - 1,
                         ));
                         return ParsePatternResult::ErrorAbove {
                             match_line: current_line,

src/parser/tests.rs

@@ -7,6 +7,17 @@ use crate::{
 
 use super::Comments;
 
+macro_rules! line {
+    ($thing:expr, $s:expr) => {{
+        let file = Spanned::dummy($s.as_bytes());
+        let pos = file
+            .lines()
+            .position(|line| line.span.bytes.contains(&$thing.bytes.start))
+            .unwrap();
+        pos
+    }};
+}
+
 #[test]
 fn parse_simple_comment() {
     let s = r"
@@ -23,7 +34,7 @@ fn main() {
     let ErrorMatchKind::Pattern { pattern, .. } = &revisioned.error_matches[0].kind else {
         panic!("expected pattern matcher");
     };
-    assert_eq!(pattern.line().get(), 5);
+    assert_ne!(line!(&pattern.span, s), 5);
     match &**pattern {
         Pattern::SubString(s) => {
             assert_eq!(
@@ -49,7 +60,7 @@ fn main() {
     let ErrorMatchKind::Code(code) = &revisioned.error_matches[0].kind else {
         panic!("expected diagnostic code matcher");
     };
-    assert_eq!(code.line().get(), 3);
+    assert_eq!(line!(&code.span, s), 3);
     assert_eq!(**code, "E0308");
 }
 
@@ -66,7 +77,7 @@ fn main() {
     println!("parsed comments: {:#?}", errors);
     assert_eq!(errors.len(), 1);
     match &errors[0] {
-        Error::InvalidComment { msg, span } if span.line_start.get() == 5 => {
+        Error::InvalidComment { msg, span } if line!(span, s) == 5 => {
             assert_eq!(msg, "text found after error code `encountered`")
         }
         _ => unreachable!(),
@@ -86,7 +97,7 @@ use std::mem;
     let revisioned = &comments.revisioned[&vec![]];
     let pat = &revisioned.error_in_other_files[0];
     assert_eq!(format!("{:?}", **pat), r#"SubString("foomp")"#);
-    assert_eq!(pat.line().get(), 2);
+    assert_eq!(line!(pat.span, s), 2);
 }
 
 #[test]
@@ -102,7 +113,7 @@ use std::mem;
     let revisioned = &comments.revisioned[&vec![]];
     let pat = &revisioned.error_in_other_files[0];
     assert_eq!(format!("{:?}", **pat), r#"Regex(Regex("foomp"))"#);
-    assert_eq!(pat.line().get(), 2);
+    assert_eq!(line!(pat.span, s), 2);
 }
 
 #[test]
@@ -116,13 +127,13 @@ use std::mem;
     println!("parsed comments: {:#?}", errors);
     assert_eq!(errors.len(), 2);
     match &errors[0] {
-        Error::InvalidComment { msg, span } if span.line_start.get() == 2 => {
+        Error::InvalidComment { msg, span } if line!(span, s) == 2 => {
             assert!(msg.contains("must be followed by `:`"))
         }
         _ => unreachable!(),
     }
     match &errors[1] {
-        Error::InvalidComment { msg, span } if span.line_start.get() == 2 => {
+        Error::InvalidComment { msg, span } if line!(span, s) == 2 => {
             assert_eq!(msg, "`error-patttern` is not a command known to `ui_test`, did you mean `error-pattern`?");
         }
         _ => unreachable!(),
@@ -140,7 +151,7 @@ use std::mem;
     println!("parsed comments: {:#?}", errors);
     assert_eq!(errors.len(), 1);
     match &errors[0] {
-        Error::InvalidComment { msg, span } if span.line_start.get() == 2 => {
+        Error::InvalidComment { msg, span } if line!(span, s) == 2 => {
             assert!(msg.contains("must be followed by `:`"))
         }
         _ => unreachable!(),

src/per_test_config.rs

@@ -368,13 +368,7 @@ impl TestConfig<'_> {
                 if !msgs.is_empty() {
                     let line = NonZeroUsize::new(line).expect("line 0 is always empty");
                     errors.push(Error::ErrorsWithoutPattern {
-                        path: Some(Spanned::new(
-                            self.status.path().to_path_buf(),
-                            spanned::Span {
-                                line_start: line,
-                                ..spanned::Span::default()
-                            },
-                        )),
+                        path: Some((self.status.path().to_path_buf(), line)),
                         msgs,
                     });
                 }