syntax: Move most of the TokenKind methods to Token

Author: petrochenkov

src/librustdoc/html/highlight.rs

@@ -257,7 +257,7 @@ impl<'a> Classifier<'a> {
             token::Question => Class::QuestionMark,
 
             token::Dollar => {
-                if self.lexer.peek().kind.is_ident() {
+                if self.lexer.peek().is_ident() {
                     self.in_macro_nonterminal = true;
                     Class::MacroNonTerminal
                 } else {

src/libsyntax/attr/mod.rs

@@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned};
 use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
 use crate::parse::parser::Parser;
 use crate::parse::{self, ParseSess, PResult};
-use crate::parse::token::{self, Token, TokenKind};
+use crate::parse::token::{self, Token};
 use crate::ptr::P;
 use crate::symbol::{sym, Symbol};
 use crate::ThinVec;
@@ -467,8 +467,7 @@ impl MetaItem {
                                              segment.ident.span.ctxt());
                 idents.push(TokenTree::token(token::ModSep, mod_sep_span).into());
             }
-            idents.push(TokenTree::token(TokenKind::from_ast_ident(segment.ident),
-                                         segment.ident.span).into());
+            idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
             last_pos = segment.ident.span.hi();
         }
         self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);

src/libsyntax/ext/tt/macro_parser.rs

@@ -428,13 +428,13 @@ pub fn parse_failure_msg(tok: TokenKind) -> String {
 }
 
 /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
-fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
-    if let (Some((name1, is_raw1)), Some((name2, is_raw2))) = (t1.ident_name(), t2.ident_name()) {
-        name1 == name2 && is_raw1 == is_raw2
-    } else if let (Some(name1), Some(name2)) = (t1.lifetime_name(), t2.lifetime_name()) {
-        name1 == name2
+fn token_name_eq(t1: &Token, t2: &Token) -> bool {
+    if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) {
+        ident1.name == ident2.name && is_raw1 == is_raw2
+    } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) {
+        ident1.name == ident2.name
     } else {
-        *t1 == *t2
+        t1.kind == t2.kind
     }
 }
 
@@ -712,7 +712,7 @@ pub fn parse(
 
         // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
         // either the parse is ambiguous (which should never happen) or there is a syntax error.
-        if token_name_eq(&parser.token, &token::Eof) {
+        if parser.token == token::Eof {
             if eof_items.len() == 1 {
                 let matches = eof_items[0]
                     .matches

src/libsyntax/ext/tt/transcribe.rs

@@ -4,7 +4,7 @@ use crate::ext::expand::Marker;
 use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
 use crate::ext::tt::quoted;
 use crate::mut_visit::noop_visit_tt;
-use crate::parse::token::{self, NtTT, Token, TokenKind};
+use crate::parse::token::{self, NtTT, Token};
 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
 
 use smallvec::{smallvec, SmallVec};
@@ -237,7 +237,7 @@ pub fn transcribe(
                         Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
                     sp = sp.apply_mark(cx.current_expansion.mark);
                     result.push(TokenTree::token(token::Dollar, sp).into());
-                    result.push(TokenTree::token(TokenKind::from_ast_ident(ident), sp).into());
+                    result.push(TokenTree::Token(Token::from_ast_ident(ident)).into());
                 }
             }
 

src/libsyntax/parse/lexer/mod.rs

@@ -1501,7 +1501,7 @@ fn char_at(s: &str, byte: usize) -> char {
 mod tests {
     use super::*;
 
-    use crate::ast::{Ident, CrateConfig};
+    use crate::ast::CrateConfig;
     use crate::symbol::Symbol;
     use crate::source_map::{SourceMap, FilePathMapping};
     use crate::feature_gate::UnstableFeatures;
@@ -1562,7 +1562,7 @@ mod tests {
             assert_eq!(string_reader.next_token(), token::Whitespace);
             let tok1 = string_reader.next_token();
             let tok2 = Token::new(
-                token::Ident(Symbol::intern("fn"), false),
+                mk_ident("fn"),
                 Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
             );
             assert_eq!(tok1.kind, tok2.kind);
@@ -1593,7 +1593,7 @@ mod tests {
 
     // make the identifier by looking up the string in the interner
     fn mk_ident(id: &str) -> TokenKind {
-        TokenKind::from_ast_ident(Ident::from_str(id))
+        token::Ident(Symbol::intern(id), false)
     }
 
     fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {

src/libsyntax/parse/parser.rs

@@ -2627,9 +2627,11 @@ impl<'a> Parser<'a> {
                     token::Ident(name, _) => name,
                     _ => unreachable!()
                 };
-                let mut err = self.fatal(&format!("unknown macro variable `{}`", name));
-                err.span_label(self.token.span, "unknown macro variable");
-                err.emit();
+                let span = self.prev_span.to(self.token.span);
+                self.diagnostic()
+                    .struct_span_fatal(span, &format!("unknown macro variable `{}`", name))
+                    .span_label(span, "unknown macro variable")
+                    .emit();
                 self.bump();
                 return
             }