Do not remove tokens before AST json serialization

compiler/rustc_ast/src/tokenstream.rs

@@ -153,8 +153,9 @@ impl fmt::Debug for LazyTokenStream {
 }
 
 impl<S: Encoder> Encodable<S> for LazyTokenStream {
-    fn encode(&self, _s: &mut S) -> Result<(), S::Error> {
-        panic!("Attempted to encode LazyTokenStream");
+    fn encode(&self, s: &mut S) -> Result<(), S::Error> {
+        // Used by AST json printing.
+        Encodable::encode(&self.create_token_stream(), s)
     }
 }
 

compiler/rustc_interface/src/passes.rs

@@ -2,9 +2,8 @@ use crate::interface::{Compiler, Result};
 use crate::proc_macro_decls;
 use crate::util;
 
-use rustc_ast::mut_visit::{self, MutVisitor};
-use rustc_ast::ptr::P;
-use rustc_ast::{self as ast, token, visit};
+use rustc_ast::mut_visit::MutVisitor;
+use rustc_ast::{self as ast, visit};
 use rustc_codegen_ssa::back::link::emit_metadata;
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_data_structures::sync::{par_iter, Lrc, OnceCell, ParallelIterator, WorkerLocal};
@@ -37,7 +36,6 @@ use rustc_span::symbol::Symbol;
 use rustc_span::{FileName, RealFileName};
 use rustc_trait_selection::traits;
 use rustc_typeck as typeck;
-use smallvec::SmallVec;
 use tracing::{info, warn};
 
 use rustc_serialize::json;
@@ -52,82 +50,6 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::{env, fs, iter, mem};
 
-/// Remove alls `LazyTokenStreams` from an AST struct
-/// Normally, this is done during AST lowering. However,
-/// printing the AST JSON requires us to serialize
-/// the entire AST, and we don't want to serialize
-/// a `LazyTokenStream`.
-struct TokenStripper;
-impl mut_visit::MutVisitor for TokenStripper {
-    fn flat_map_item(&mut self, mut i: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
-        i.tokens = None;
-        mut_visit::noop_flat_map_item(i, self)
-    }
-    fn flat_map_foreign_item(
-        &mut self,
-        mut i: P<ast::ForeignItem>,
-    ) -> SmallVec<[P<ast::ForeignItem>; 1]> {
-        i.tokens = None;
-        mut_visit::noop_flat_map_foreign_item(i, self)
-    }
-    fn flat_map_trait_item(
-        &mut self,
-        mut i: P<ast::AssocItem>,
-    ) -> SmallVec<[P<ast::AssocItem>; 1]> {
-        i.tokens = None;
-        mut_visit::noop_flat_map_assoc_item(i, self)
-    }
-    fn flat_map_impl_item(&mut self, mut i: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
-        i.tokens = None;
-        mut_visit::noop_flat_map_assoc_item(i, self)
-    }
-    fn visit_block(&mut self, b: &mut P<ast::Block>) {
-        b.tokens = None;
-        mut_visit::noop_visit_block(b, self);
-    }
-    fn flat_map_stmt(&mut self, mut stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> {
-        stmt.tokens = None;
-        mut_visit::noop_flat_map_stmt(stmt, self)
-    }
-    fn visit_pat(&mut self, p: &mut P<ast::Pat>) {
-        p.tokens = None;
-        mut_visit::noop_visit_pat(p, self);
-    }
-    fn visit_ty(&mut self, ty: &mut P<ast::Ty>) {
-        ty.tokens = None;
-        mut_visit::noop_visit_ty(ty, self);
-    }
-    fn visit_attribute(&mut self, attr: &mut ast::Attribute) {
-        attr.tokens = None;
-        if let ast::AttrKind::Normal(ast::AttrItem { tokens, .. }) = &mut attr.kind {
-            *tokens = None;
-        }
-        mut_visit::noop_visit_attribute(attr, self);
-    }
-
-    fn visit_interpolated(&mut self, nt: &mut token::Nonterminal) {
-        if let token::Nonterminal::NtMeta(meta) = nt {
-            meta.tokens = None;
-        }
-        // Handles all of the other cases
-        mut_visit::noop_visit_interpolated(nt, self);
-    }
-
-    fn visit_path(&mut self, p: &mut ast::Path) {
-        p.tokens = None;
-        mut_visit::noop_visit_path(p, self);
-    }
-    fn visit_vis(&mut self, vis: &mut ast::Visibility) {
-        vis.tokens = None;
-        mut_visit::noop_visit_vis(vis, self);
-    }
-    fn visit_expr(&mut self, e: &mut P<ast::Expr>) {
-        e.tokens = None;
-        mut_visit::noop_visit_expr(e, self);
-    }
-    fn visit_mac(&mut self, _mac: &mut ast::MacCall) {}
-}
-
 pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
     let krate = sess.time("parse_crate", || match input {
         Input::File(file) => parse_crate_from_file(file, &sess.parse_sess),
@@ -137,10 +59,6 @@ pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
     })?;
 
     if sess.opts.debugging_opts.ast_json_noexpand {
-        // Set any `token` fields to `None` before
-        // we display the AST.
-        let mut krate = krate.clone();
-        TokenStripper.visit_crate(&mut krate);
         println!("{}", json::as_json(&krate));
     }
 
@@ -461,10 +379,6 @@ fn configure_and_expand_inner<'a>(
     }
 
     if sess.opts.debugging_opts.ast_json {
-        // Set any `token` fields to `None` before
-        // we display the AST.
-        let mut krate = krate.clone();
-        TokenStripper.visit_crate(&mut krate);
         println!("{}", json::as_json(&krate));
     }
 

src/test/ui/ast-json/ast-json-ice.rs

@@ -39,3 +39,26 @@ fn main() {
 
     struct A;
 }
+
+// Regressions tests for issues #78398 and #78510 (captured tokens in associated and foreign items)
+
+struct S;
+
+macro_rules! mac_extern {
+    ($i:item) => {
+        extern "C" { $i }
+    }
+}
+macro_rules! mac_assoc {
+    ($i:item) => {
+        impl S { $i }
+        trait Bar { $i }
+    }
+}
+
+mac_extern! {
+    fn foo();
+}
+mac_assoc! {
+    fn foo() {}
+}

src/test/ui/ast-json/ast-json-noexpand-output.stdout

@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":null}],"span":{"lo":0,"hi":0},"proc_macros":[]}
+{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}}],"span":{"lo":0,"hi":0},"proc_macros":[]}

src/test/ui/ast-json/ast-json-output.stdout

@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":null}],"span":{"lo":0,"hi":0},"proc_macros":[]}
+{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}}],"span":{"lo":0,"hi":0},"proc_macros":[]}