Skip to content

Commit

Permalink
Rollup merge of rust-lang#102639 - nnethercote:improve-spans-splittin…
Browse files Browse the repository at this point in the history 
…g, r=Aaron1011

Improve spans when splitting multi-char operator tokens for proc macros.

When a two-char (or three-char) operator token is split into single-char operator tokens before being passed to a proc macro, the single-char tokens are given the original span of length two (or three). This PR gives them more accurate spans.

r? `@Aaron1011`
cc `@petrochenkov`
  • Loading branch information
@matthiaskrgr
matthiaskrgr authored Oct 4, 2022
2 parents f86ee78 + 88dab8d commit 185ca0f
Show file tree
Hide file tree
Showing 15 changed files with 166 additions and 84 deletions.
16 changes: 14 additions & 2 deletions compiler/rustc_expand/src/proc_macro_server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,20 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
// before that get `joint = true`.
let mut op = |s: &str| {
assert!(s.is_ascii());
trees.extend(s.bytes().enumerate().map(|(idx, ch)| {
let is_final = idx == s.len() - 1;
trees.extend(s.bytes().enumerate().map(|(i, ch)| {
let is_final = i == s.len() - 1;
// Split the token span into single chars. Unless the span
// is an unusual one, e.g. due to proc macro expansion. We
// determine this by assuming any span with a length that
// matches the operator length is a normal one, and any
// span with a different length is an unusual one.
let span = if (span.hi() - span.lo()).to_usize() == s.len() {
let lo = span.lo() + BytePos::from_usize(i);
let hi = lo + BytePos::from_usize(1);
span.with_lo(lo).with_hi(hi)
} else {
span
};
TokenTree::Punct(Punct { ch, joint: if is_final { joint } else { true }, span })
}));
};
Expand Down
2 changes: 1 addition & 1 deletion src/test/ui/lint/redundant-semicolon/redundant-semi-proc-macro.stderr
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
TokenStream [Ident { ident: "fn", span: #0 bytes(198..200) }, Ident { ident: "span_preservation", span: #0 bytes(201..218) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(218..220) }, Group { delimiter: Brace, stream: TokenStream [Ident { ident: "let", span: #0 bytes(228..231) }, Ident { ident: "tst", span: #0 bytes(232..235) }, Punct { ch: '=', spacing: Alone, span: #0 bytes(236..237) }, Literal { kind: Integer, symbol: "123", suffix: None, span: #0 bytes(238..241) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(241..242) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(242..243) }, Ident { ident: "match", span: #0 bytes(289..294) }, Ident { ident: "tst", span: #0 bytes(295..298) }, Group { delimiter: Brace, stream: TokenStream [Literal { kind: Integer, symbol: "123", suffix: None, span: #0 bytes(483..486) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(487..489) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(487..489) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(490..492) }, Punct { ch: ',', spacing: Alone, span: #0 bytes(492..493) }, Ident { ident: "_", span: #0 bytes(502..503) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(504..506) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(504..506) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(507..509) }], span: #0 bytes(299..515) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(515..516) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(516..517) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(517..518) }], span: #0 bytes(222..562) }]
TokenStream [Ident { ident: "fn", span: #0 bytes(198..200) }, Ident { ident: "span_preservation", span: #0 bytes(201..218) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(218..220) }, Group { delimiter: Brace, stream: TokenStream [Ident { ident: "let", span: #0 bytes(228..231) }, Ident { ident: "tst", span: #0 bytes(232..235) }, Punct { ch: '=', spacing: Alone, span: #0 bytes(236..237) }, Literal { kind: Integer, symbol: "123", suffix: None, span: #0 bytes(238..241) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(241..242) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(242..243) }, Ident { ident: "match", span: #0 bytes(289..294) }, Ident { ident: "tst", span: #0 bytes(295..298) }, Group { delimiter: Brace, stream: TokenStream [Literal { kind: Integer, symbol: "123", suffix: None, span: #0 bytes(483..486) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(487..488) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(488..489) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(490..492) }, Punct { ch: ',', spacing: Alone, span: #0 bytes(492..493) }, Ident { ident: "_", span: #0 bytes(502..503) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(504..505) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(505..506) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(507..509) }], span: #0 bytes(299..515) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(515..516) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(516..517) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(517..518) }], span: #0 bytes(222..562) }]
error: unnecessary trailing semicolon
--> $DIR/redundant-semi-proc-macro.rs:9:19
|
Expand Down
4 changes: 2 additions & 2 deletions src/test/ui/proc-macro/attr-complex-fn.stdout
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,12 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
Punct {
ch: '>',
spacing: Joint,
span: $DIR/attr-complex-fn.rs:19:36: 19:38 (#0),
span: $DIR/attr-complex-fn.rs:19:36: 19:37 (#0),
},
Punct {
ch: '>',
spacing: Joint,
span: $DIR/attr-complex-fn.rs:19:36: 19:38 (#0),
span: $DIR/attr-complex-fn.rs:19:37: 19:38 (#0),
},
Punct {
ch: '>',
Expand Down
12 changes: 6 additions & 6 deletions src/test/ui/proc-macro/capture-macro-rules-invoke.stdout
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,12 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
Punct {
ch: ':',
spacing: Joint,
span: $DIR/capture-macro-rules-invoke.rs:45:16: 45:18 (#0),
span: $DIR/capture-macro-rules-invoke.rs:45:16: 45:17 (#0),
},
Punct {
ch: ':',
spacing: Alone,
span: $DIR/capture-macro-rules-invoke.rs:45:16: 45:18 (#0),
span: $DIR/capture-macro-rules-invoke.rs:45:17: 45:18 (#0),
},
Ident {
ident: "option",
Expand All @@ -191,12 +191,12 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
Punct {
ch: ':',
spacing: Joint,
span: $DIR/capture-macro-rules-invoke.rs:45:24: 45:26 (#0),
span: $DIR/capture-macro-rules-invoke.rs:45:24: 45:25 (#0),
},
Punct {
ch: ':',
spacing: Alone,
span: $DIR/capture-macro-rules-invoke.rs:45:24: 45:26 (#0),
span: $DIR/capture-macro-rules-invoke.rs:45:25: 45:26 (#0),
},
Ident {
ident: "Option",
Expand Down Expand Up @@ -231,12 +231,12 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
Punct {
ch: ':',
spacing: Joint,
span: $DIR/capture-macro-rules-invoke.rs:46:24: 46:26 (#0),
span: $DIR/capture-macro-rules-invoke.rs:46:24: 46:25 (#0),
},
Punct {
ch: ':',
spacing: Alone,
span: $DIR/capture-macro-rules-invoke.rs:46:24: 46:26 (#0),
span: $DIR/capture-macro-rules-invoke.rs:46:25: 46:26 (#0),
},
Ident {
ident: "path",
Expand Down
7 changes: 6 additions & 1 deletion src/test/ui/proc-macro/debug/dump-debug-span-debug.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,19 @@
// aux-build:macro-dump-debug.rs
// compile-flags: -Z span-debug


extern crate macro_dump_debug;
use macro_dump_debug::dump_debug;

dump_debug! {
ident // ident
r#ident // raw ident
, // alone punct
==> // joint punct
&& // joint punct, two-char op
||> // joint punct, two-char op + one-char op
||<< // joint punct, two-char op + two-char op
..= // joint punct, three-char op
<<=! // joint punct, three-char op + one-char-op
() // empty group
[_] // nonempty group

Expand Down
Loading

0 comments on commit 185ca0f

Please sign in to comment.