Skip to content

Commit

Permalink
Improve spans when splitting multi-char operator tokens for proc macros.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nnethercote
nnethercote committed Oct 3, 2022
1 parent 177b3d2 commit 88dab8d
Show file tree
Hide file tree
Showing 14 changed files with 79 additions and 67 deletions.
16 changes: 14 additions & 2 deletions compiler/rustc_expand/src/proc_macro_server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,20 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
// before that get `joint = true`.
let mut op = |s: &str| {
assert!(s.is_ascii());
trees.extend(s.bytes().enumerate().map(|(idx, ch)| {
let is_final = idx == s.len() - 1;
trees.extend(s.bytes().enumerate().map(|(i, ch)| {
let is_final = i == s.len() - 1;
// Split the token span into single chars. Unless the span
// is an unusual one, e.g. due to proc macro expansion. We
// determine this by assuming any span with a length that
// matches the operator length is a normal one, and any
// span with a different length is an unusual one.
let span = if (span.hi() - span.lo()).to_usize() == s.len() {
let lo = span.lo() + BytePos::from_usize(i);
let hi = lo + BytePos::from_usize(1);
span.with_lo(lo).with_hi(hi)
} else {
span
};
TokenTree::Punct(Punct { ch, joint: if is_final { joint } else { true }, span })
}));
};
Expand Down
2 changes: 1 addition & 1 deletion src/test/ui/lint/redundant-semicolon/redundant-semi-proc-macro.stderr
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
TokenStream [Ident { ident: "fn", span: #0 bytes(198..200) }, Ident { ident: "span_preservation", span: #0 bytes(201..218) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(218..220) }, Group { delimiter: Brace, stream: TokenStream [Ident { ident: "let", span: #0 bytes(228..231) }, Ident { ident: "tst", span: #0 bytes(232..235) }, Punct { ch: '=', spacing: Alone, span: #0 bytes(236..237) }, Literal { kind: Integer, symbol: "123", suffix: None, span: #0 bytes(238..241) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(241..242) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(242..243) }, Ident { ident: "match", span: #0 bytes(289..294) }, Ident { ident: "tst", span: #0 bytes(295..298) }, Group { delimiter: Brace, stream: TokenStream [Literal { kind: Integer, symbol: "123", suffix: None, span: #0 bytes(483..486) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(487..489) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(487..489) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(490..492) }, Punct { ch: ',', spacing: Alone, span: #0 bytes(492..493) }, Ident { ident: "_", span: #0 bytes(502..503) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(504..506) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(504..506) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(507..509) }], span: #0 bytes(299..515) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(515..516) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(516..517) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(517..518) }], span: #0 bytes(222..562) }]
TokenStream [Ident { ident: "fn", span: #0 bytes(198..200) }, Ident { ident: "span_preservation", span: #0 bytes(201..218) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(218..220) }, Group { delimiter: Brace, stream: TokenStream [Ident { ident: "let", span: #0 bytes(228..231) }, Ident { ident: "tst", span: #0 bytes(232..235) }, Punct { ch: '=', spacing: Alone, span: #0 bytes(236..237) }, Literal { kind: Integer, symbol: "123", suffix: None, span: #0 bytes(238..241) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(241..242) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(242..243) }, Ident { ident: "match", span: #0 bytes(289..294) }, Ident { ident: "tst", span: #0 bytes(295..298) }, Group { delimiter: Brace, stream: TokenStream [Literal { kind: Integer, symbol: "123", suffix: None, span: #0 bytes(483..486) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(487..488) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(488..489) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(490..492) }, Punct { ch: ',', spacing: Alone, span: #0 bytes(492..493) }, Ident { ident: "_", span: #0 bytes(502..503) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(504..505) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(505..506) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(507..509) }], span: #0 bytes(299..515) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(515..516) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(516..517) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(517..518) }], span: #0 bytes(222..562) }]
error: unnecessary trailing semicolon
--> $DIR/redundant-semi-proc-macro.rs:9:19
|
Expand Down
4 changes: 2 additions & 2 deletions src/test/ui/proc-macro/attr-complex-fn.stdout
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,12 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
Punct {
ch: '>',
spacing: Joint,
span: $DIR/attr-complex-fn.rs:19:36: 19:38 (#0),
span: $DIR/attr-complex-fn.rs:19:36: 19:37 (#0),
},
Punct {
ch: '>',
spacing: Joint,
span: $DIR/attr-complex-fn.rs:19:36: 19:38 (#0),
span: $DIR/attr-complex-fn.rs:19:37: 19:38 (#0),
},
Punct {
ch: '>',
Expand Down
12 changes: 6 additions & 6 deletions src/test/ui/proc-macro/capture-macro-rules-invoke.stdout
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,12 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
Punct {
ch: ':',
spacing: Joint,
span: $DIR/capture-macro-rules-invoke.rs:45:16: 45:18 (#0),
span: $DIR/capture-macro-rules-invoke.rs:45:16: 45:17 (#0),
},
Punct {
ch: ':',
spacing: Alone,
span: $DIR/capture-macro-rules-invoke.rs:45:16: 45:18 (#0),
span: $DIR/capture-macro-rules-invoke.rs:45:17: 45:18 (#0),
},
Ident {
ident: "option",
Expand All @@ -191,12 +191,12 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
Punct {
ch: ':',
spacing: Joint,
span: $DIR/capture-macro-rules-invoke.rs:45:24: 45:26 (#0),
span: $DIR/capture-macro-rules-invoke.rs:45:24: 45:25 (#0),
},
Punct {
ch: ':',
spacing: Alone,
span: $DIR/capture-macro-rules-invoke.rs:45:24: 45:26 (#0),
span: $DIR/capture-macro-rules-invoke.rs:45:25: 45:26 (#0),
},
Ident {
ident: "Option",
Expand Down Expand Up @@ -231,12 +231,12 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
Punct {
ch: ':',
spacing: Joint,
span: $DIR/capture-macro-rules-invoke.rs:46:24: 46:26 (#0),
span: $DIR/capture-macro-rules-invoke.rs:46:24: 46:25 (#0),
},
Punct {
ch: ':',
spacing: Alone,
span: $DIR/capture-macro-rules-invoke.rs:46:24: 46:26 (#0),
span: $DIR/capture-macro-rules-invoke.rs:46:25: 46:26 (#0),
},
Ident {
ident: "path",
Expand Down
30 changes: 15 additions & 15 deletions src/test/ui/proc-macro/debug/dump-debug-span-debug.stderr
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
TokenStream [Ident { ident: "ident", span: $DIR/dump-debug-span-debug.rs:10:5: 10:10 (#0) }, Ident { ident: "r#ident", span: $DIR/dump-debug-span-debug.rs:11:5: 11:12 (#0) }, Punct { ch: ',', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:12:5: 12:6 (#0) }, Punct { ch: '&', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:13:5: 13:7 (#0) }, Punct { ch: '&', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:13:5: 13:7 (#0) }, Punct { ch: '|', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:14:5: 14:7 (#0) }, Punct { ch: '|', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:14:5: 14:7 (#0) }, Punct { ch: '>', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:14:7: 14:8 (#0) }, Punct { ch: '|', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:15:5: 15:7 (#0) }, Punct { ch: '|', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:15:5: 15:7 (#0) }, Punct { ch: '<', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:15:7: 15:9 (#0) }, Punct { ch: '<', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:15:7: 15:9 (#0) }, Punct { ch: '.', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:16:5: 16:8 (#0) }, Punct { ch: '.', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:16:5: 16:8 (#0) }, Punct { ch: '=', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:16:5: 16:8 (#0) }, Punct { ch: '<', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:17:5: 17:8 (#0) }, Punct { ch: '<', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:17:5: 17:8 (#0) }, Punct { ch: '=', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:17:5: 17:8 (#0) }, Punct { ch: '!', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:17:8: 17:9 (#0) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: $DIR/dump-debug-span-debug.rs:18:5: 18:7 (#0) }, Group { delimiter: Bracket, stream: TokenStream [Ident { ident: "_", span: $DIR/dump-debug-span-debug.rs:19:6: 19:7 (#0) }], span: $DIR/dump-debug-span-debug.rs:19:5: 19:8 (#0) }, Literal { kind: Integer, symbol: "0", suffix: None, span: $DIR/dump-debug-span-debug.rs:22:5: 22:6 (#0) }, Literal { kind: Float, symbol: "1.0", suffix: None, span: $DIR/dump-debug-span-debug.rs:23:5: 23:8 (#0) }, Literal { kind: Str, symbol: "S", suffix: None, span: $DIR/dump-debug-span-debug.rs:24:5: 24:8 (#0) }, Literal { kind: ByteStr, symbol: "B", suffix: None, span: $DIR/dump-debug-span-debug.rs:25:5: 25:9 (#0) }, Literal { kind: StrRaw(0), symbol: "R", suffix: None, span: $DIR/dump-debug-span-debug.rs:26:5: 26:9 (#0) }, Literal { kind: StrRaw(2), symbol: "R", suffix: None, span: $DIR/dump-debug-span-debug.rs:27:5: 27:13 (#0) }, Literal { kind: ByteStrRaw(0), symbol: "BR", suffix: None, span: $DIR/dump-debug-span-debug.rs:28:5: 28:11 (#0) }, Literal { kind: ByteStrRaw(2), symbol: "BR", suffix: None, span: $DIR/dump-debug-span-debug.rs:29:5: 29:15 (#0) }, Literal { kind: Char, symbol: "C", suffix: None, span: $DIR/dump-debug-span-debug.rs:30:5: 30:8 (#0) }, Literal { kind: Byte, symbol: "B", suffix: None, span: $DIR/dump-debug-span-debug.rs:31:5: 31:9 (#0) }, Literal { kind: Integer, symbol: "0", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:34:5: 34:7 (#0) }, Literal { kind: Float, symbol: "1.0", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:35:5: 35:9 (#0) }, Literal { kind: Str, symbol: "S", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:36:5: 36:9 (#0) }, Literal { kind: ByteStr, symbol: "B", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:37:5: 37:10 (#0) }, Literal { kind: StrRaw(0), symbol: "R", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:38:5: 38:10 (#0) }, Literal { kind: StrRaw(2), symbol: "R", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:39:5: 39:14 (#0) }, Literal { kind: ByteStrRaw(0), symbol: "BR", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:40:5: 40:12 (#0) }, Literal { kind: ByteStrRaw(2), symbol: "BR", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:41:5: 41:16 (#0) }, Literal { kind: Char, symbol: "C", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:42:5: 42:9 (#0) }, Literal { kind: Byte, symbol: "B", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:43:5: 43:10 (#0) }]
TokenStream [Ident { ident: "ident", span: $DIR/dump-debug-span-debug.rs:10:5: 10:10 (#0) }, Ident { ident: "r#ident", span: $DIR/dump-debug-span-debug.rs:11:5: 11:12 (#0) }, Punct { ch: ',', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:12:5: 12:6 (#0) }, Punct { ch: '&', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:13:5: 13:6 (#0) }, Punct { ch: '&', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:13:6: 13:7 (#0) }, Punct { ch: '|', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:14:5: 14:6 (#0) }, Punct { ch: '|', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:14:6: 14:7 (#0) }, Punct { ch: '>', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:14:7: 14:8 (#0) }, Punct { ch: '|', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:15:5: 15:6 (#0) }, Punct { ch: '|', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:15:6: 15:7 (#0) }, Punct { ch: '<', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:15:7: 15:8 (#0) }, Punct { ch: '<', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:15:8: 15:9 (#0) }, Punct { ch: '.', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:16:5: 16:6 (#0) }, Punct { ch: '.', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:16:6: 16:7 (#0) }, Punct { ch: '=', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:16:7: 16:8 (#0) }, Punct { ch: '<', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:17:5: 17:6 (#0) }, Punct { ch: '<', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:17:6: 17:7 (#0) }, Punct { ch: '=', spacing: Joint, span: $DIR/dump-debug-span-debug.rs:17:7: 17:8 (#0) }, Punct { ch: '!', spacing: Alone, span: $DIR/dump-debug-span-debug.rs:17:8: 17:9 (#0) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: $DIR/dump-debug-span-debug.rs:18:5: 18:7 (#0) }, Group { delimiter: Bracket, stream: TokenStream [Ident { ident: "_", span: $DIR/dump-debug-span-debug.rs:19:6: 19:7 (#0) }], span: $DIR/dump-debug-span-debug.rs:19:5: 19:8 (#0) }, Literal { kind: Integer, symbol: "0", suffix: None, span: $DIR/dump-debug-span-debug.rs:22:5: 22:6 (#0) }, Literal { kind: Float, symbol: "1.0", suffix: None, span: $DIR/dump-debug-span-debug.rs:23:5: 23:8 (#0) }, Literal { kind: Str, symbol: "S", suffix: None, span: $DIR/dump-debug-span-debug.rs:24:5: 24:8 (#0) }, Literal { kind: ByteStr, symbol: "B", suffix: None, span: $DIR/dump-debug-span-debug.rs:25:5: 25:9 (#0) }, Literal { kind: StrRaw(0), symbol: "R", suffix: None, span: $DIR/dump-debug-span-debug.rs:26:5: 26:9 (#0) }, Literal { kind: StrRaw(2), symbol: "R", suffix: None, span: $DIR/dump-debug-span-debug.rs:27:5: 27:13 (#0) }, Literal { kind: ByteStrRaw(0), symbol: "BR", suffix: None, span: $DIR/dump-debug-span-debug.rs:28:5: 28:11 (#0) }, Literal { kind: ByteStrRaw(2), symbol: "BR", suffix: None, span: $DIR/dump-debug-span-debug.rs:29:5: 29:15 (#0) }, Literal { kind: Char, symbol: "C", suffix: None, span: $DIR/dump-debug-span-debug.rs:30:5: 30:8 (#0) }, Literal { kind: Byte, symbol: "B", suffix: None, span: $DIR/dump-debug-span-debug.rs:31:5: 31:9 (#0) }, Literal { kind: Integer, symbol: "0", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:34:5: 34:7 (#0) }, Literal { kind: Float, symbol: "1.0", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:35:5: 35:9 (#0) }, Literal { kind: Str, symbol: "S", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:36:5: 36:9 (#0) }, Literal { kind: ByteStr, symbol: "B", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:37:5: 37:10 (#0) }, Literal { kind: StrRaw(0), symbol: "R", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:38:5: 38:10 (#0) }, Literal { kind: StrRaw(2), symbol: "R", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:39:5: 39:14 (#0) }, Literal { kind: ByteStrRaw(0), symbol: "BR", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:40:5: 40:12 (#0) }, Literal { kind: ByteStrRaw(2), symbol: "BR", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:41:5: 41:16 (#0) }, Literal { kind: Char, symbol: "C", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:42:5: 42:9 (#0) }, Literal { kind: Byte, symbol: "B", suffix: Some("q"), span: $DIR/dump-debug-span-debug.rs:43:5: 43:10 (#0) }]
TokenStream [
Ident {
ident: "ident",
Expand All @@ -16,22 +16,22 @@ TokenStream [
Punct {
ch: '&',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:13:5: 13:7 (#0),
span: $DIR/dump-debug-span-debug.rs:13:5: 13:6 (#0),
},
Punct {
ch: '&',
spacing: Alone,
span: $DIR/dump-debug-span-debug.rs:13:5: 13:7 (#0),
span: $DIR/dump-debug-span-debug.rs:13:6: 13:7 (#0),
},
Punct {
ch: '|',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:14:5: 14:7 (#0),
span: $DIR/dump-debug-span-debug.rs:14:5: 14:6 (#0),
},
Punct {
ch: '|',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:14:5: 14:7 (#0),
span: $DIR/dump-debug-span-debug.rs:14:6: 14:7 (#0),
},
Punct {
ch: '>',
Expand All @@ -41,52 +41,52 @@ TokenStream [
Punct {
ch: '|',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:15:5: 15:7 (#0),
span: $DIR/dump-debug-span-debug.rs:15:5: 15:6 (#0),
},
Punct {
ch: '|',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:15:5: 15:7 (#0),
span: $DIR/dump-debug-span-debug.rs:15:6: 15:7 (#0),
},
Punct {
ch: '<',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:15:7: 15:9 (#0),
span: $DIR/dump-debug-span-debug.rs:15:7: 15:8 (#0),
},
Punct {
ch: '<',
spacing: Alone,
span: $DIR/dump-debug-span-debug.rs:15:7: 15:9 (#0),
span: $DIR/dump-debug-span-debug.rs:15:8: 15:9 (#0),
},
Punct {
ch: '.',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:16:5: 16:8 (#0),
span: $DIR/dump-debug-span-debug.rs:16:5: 16:6 (#0),
},
Punct {
ch: '.',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:16:5: 16:8 (#0),
span: $DIR/dump-debug-span-debug.rs:16:6: 16:7 (#0),
},
Punct {
ch: '=',
spacing: Alone,
span: $DIR/dump-debug-span-debug.rs:16:5: 16:8 (#0),
span: $DIR/dump-debug-span-debug.rs:16:7: 16:8 (#0),
},
Punct {
ch: '<',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:17:5: 17:8 (#0),
span: $DIR/dump-debug-span-debug.rs:17:5: 17:6 (#0),
},
Punct {
ch: '<',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:17:5: 17:8 (#0),
span: $DIR/dump-debug-span-debug.rs:17:6: 17:7 (#0),
},
Punct {
ch: '=',
spacing: Joint,
span: $DIR/dump-debug-span-debug.rs:17:5: 17:8 (#0),
span: $DIR/dump-debug-span-debug.rs:17:7: 17:8 (#0),
},
Punct {
ch: '!',
Expand Down
Loading

0 comments on commit 88dab8d

Please sign in to comment.