77#![ feature( or_patterns) ]
88
99use rustc_ast as ast;
10- use rustc_ast:: token:: { self , DelimToken , Nonterminal , Token } ;
11- use rustc_ast:: tokenstream:: { self , TokenStream , TokenTree } ;
10+ use rustc_ast:: token:: { self , DelimToken , Nonterminal , Token , TokenKind } ;
11+ use rustc_ast:: tokenstream:: { self , IsJoint , TokenStream , TokenTree } ;
1212use rustc_ast_pretty:: pprust;
1313use rustc_data_structures:: sync:: Lrc ;
1414use rustc_errors:: { Diagnostic , FatalError , Level , PResult } ;
@@ -309,7 +309,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
309309 // modifications, including adding/removing typically non-semantic
310310 // tokens such as extra braces and commas, don't happen.
311311 if let Some ( tokens) = tokens {
312- if tokenstream_probably_equal_for_proc_macro ( & tokens, & tokens_for_real) {
312+ if tokenstream_probably_equal_for_proc_macro ( & tokens, & tokens_for_real, sess ) {
313313 return tokens;
314314 }
315315 info ! (
@@ -327,7 +327,11 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
327327//
328328// This is otherwise the same as `eq_unspanned`, only recursing with a
329329// different method.
330- pub fn tokenstream_probably_equal_for_proc_macro ( first : & TokenStream , other : & TokenStream ) -> bool {
330+ pub fn tokenstream_probably_equal_for_proc_macro (
331+ first : & TokenStream ,
332+ other : & TokenStream ,
333+ sess : & ParseSess ,
334+ ) -> bool {
331335 // When checking for `probably_eq`, we ignore certain tokens that aren't
332336 // preserved in the AST. Because they are not preserved, the pretty
333337 // printer arbitrarily adds or removes them when printing as token
@@ -408,20 +412,39 @@ pub fn tokenstream_probably_equal_for_proc_macro(first: &TokenStream, other: &To
408412 }
409413 }
410414 token_trees = out. into_iter ( ) . map ( TokenTree :: Token ) . collect ( ) ;
411- if token_trees. len ( ) != 1 {
412- debug ! ( "break_tokens: broke {:?} to {:?}" , tree, token_trees) ;
413- }
414415 } else {
415416 token_trees = SmallVec :: new ( ) ;
416417 token_trees. push ( tree) ;
417418 }
418419 token_trees. into_iter ( )
419420 }
420421
421- let mut t1 = first. trees ( ) . filter ( semantic_tree) . flat_map ( break_tokens) ;
422- let mut t2 = other. trees ( ) . filter ( semantic_tree) . flat_map ( break_tokens) ;
422+ let expand_nt = |tree : TokenTree | {
423+ if let TokenTree :: Token ( Token { kind : TokenKind :: Interpolated ( nt) , span } ) = & tree {
424+ // When checking tokenstreams for 'probable equality', we are comparing
425+ // a captured (from parsing) `TokenStream` to a reparsed tokenstream.
426+ // The reparsed Tokenstream will never have `None`-delimited groups,
427+ // since they are only ever inserted as a result of macro expansion.
428+ // Therefore, inserting a `None`-delimtied group here (when we
429+ // convert a nested `Nonterminal` to a tokenstream) would cause
430+ // a mismatch with the reparsed tokenstream.
431+ //
432+ // Note that we currently do not handle the case where the
433+ // reparsed stream has a `Parenthesis`-delimited group
434+ // inserted. This will cause a spurious mismatch:
435+ // issue #75734 tracks resolving this.
436+ nt_to_tokenstream ( nt, sess, * span) . into_trees ( )
437+ } else {
438+ TokenStream :: new ( vec ! [ ( tree, IsJoint :: NonJoint ) ] ) . into_trees ( )
439+ }
440+ } ;
441+
442+ // Break tokens after we expand any nonterminals, so that we break tokens
443+ // that are produced as a result of nonterminal expansion.
444+ let mut t1 = first. trees ( ) . filter ( semantic_tree) . flat_map ( expand_nt) . flat_map ( break_tokens) ;
445+ let mut t2 = other. trees ( ) . filter ( semantic_tree) . flat_map ( expand_nt) . flat_map ( break_tokens) ;
423446 for ( t1, t2) in t1. by_ref ( ) . zip ( t2. by_ref ( ) ) {
424- if !tokentree_probably_equal_for_proc_macro ( & t1, & t2) {
447+ if !tokentree_probably_equal_for_proc_macro ( & t1, & t2, sess ) {
425448 return false ;
426449 }
427450 }
@@ -433,13 +456,17 @@ pub fn tokenstream_probably_equal_for_proc_macro(first: &TokenStream, other: &To
433456//
434457// This is otherwise the same as `eq_unspanned`, only recursing with a
435458// different method.
436- fn tokentree_probably_equal_for_proc_macro ( first : & TokenTree , other : & TokenTree ) -> bool {
459+ pub fn tokentree_probably_equal_for_proc_macro (
460+ first : & TokenTree ,
461+ other : & TokenTree ,
462+ sess : & ParseSess ,
463+ ) -> bool {
437464 match ( first, other) {
438465 ( TokenTree :: Token ( token) , TokenTree :: Token ( token2) ) => {
439466 token_probably_equal_for_proc_macro ( token, token2)
440467 }
441468 ( TokenTree :: Delimited ( _, delim, tts) , TokenTree :: Delimited ( _, delim2, tts2) ) => {
442- delim == delim2 && tokenstream_probably_equal_for_proc_macro ( & tts, & tts2)
469+ delim == delim2 && tokenstream_probably_equal_for_proc_macro ( & tts, & tts2, sess )
443470 }
444471 _ => false ,
445472 }
@@ -498,7 +525,7 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool {
498525 b == d && ( a == c || a == kw:: DollarCrate || c == kw:: DollarCrate )
499526 }
500527
501- ( & Interpolated ( ..) , & Interpolated ( ..) ) => false ,
528+ ( & Interpolated ( ..) , & Interpolated ( ..) ) => panic ! ( "Unexpanded Interpolated!" ) ,
502529
503530 _ => panic ! ( "forgot to add a token?" ) ,
504531 }
0 commit comments