Rollup merge of rust-lang#65916 - Centril:split-syntax-3, r=davidtwco

syntax: move stuff around

Part of rust-lang#65324.

r? @davidtwco
cc @estebank @petrochenkov

Author: Centril

src/librustc/hir/lowering.rs

@@ -66,7 +66,7 @@ use syntax::ptr::P as AstP;
 use syntax::ast::*;
 use syntax::errors;
 use syntax::print::pprust;
-use syntax::parse::token::{self, Nonterminal, Token};
+use syntax::token::{self, Nonterminal, Token};
 use syntax::tokenstream::{TokenStream, TokenTree};
 use syntax::sess::ParseSess;
 use syntax::source_map::{respan, ExpnData, ExpnKind, DesugaringKind, Spanned};

src/librustc/hir/map/def_collector.rs

@@ -4,7 +4,7 @@ use crate::hir::def_id::DefIndex;
 use syntax::ast::*;
 use syntax::visit;
 use syntax::symbol::{kw, sym};
-use syntax::parse::token::{self, Token};
+use syntax::token::{self, Token};
 use syntax_pos::hygiene::ExpnId;
 use syntax_pos::Span;
 

src/librustc/ich/impls_syntax.rs

@@ -8,9 +8,9 @@ use std::mem;
 
 use syntax::ast;
 use syntax::feature_gate;
-use syntax::parse::token;
-use syntax::symbol::SymbolStr;
+use syntax::token;
 use syntax::tokenstream;
+use syntax_pos::symbol::SymbolStr;
 use syntax_pos::SourceFile;
 
 use crate::hir::def_id::{DefId, CrateNum, CRATE_DEF_INDEX};

src/librustc_driver/lib.rs

@@ -42,6 +42,7 @@ use rustc::ty::TyCtxt;
 use rustc::util::common::{set_time_depth, time, print_time_passes_entry, ErrorReported};
 use rustc_metadata::locator;
 use rustc_codegen_utils::codegen_backend::CodegenBackend;
+use errors::PResult;
 use rustc_interface::interface;
 use rustc_interface::util::get_codegen_sysroot;
 use rustc_data_structures::sync::SeqCst;
@@ -64,7 +65,7 @@ use std::time::Instant;
 use syntax::ast;
 use syntax::source_map::FileLoader;
 use syntax::feature_gate::{GatedCfg, UnstableFeatures};
-use syntax::parse::{self, PResult};
+use syntax::parse;
 use syntax::symbol::sym;
 use syntax_pos::{DUMMY_SP, FileName};
 

src/librustc_errors/lib.rs

@@ -15,7 +15,8 @@ use Level::*;
 
 use emitter::{Emitter, EmitterWriter, is_case_difference};
 use registry::Registry;
-
+#[cfg(target_arch = "x86_64")]
+use rustc_data_structures::static_assert_size;
 use rustc_data_structures::sync::{self, Lrc, Lock};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_data_structures::stable_hasher::StableHasher;
@@ -48,6 +49,13 @@ use syntax_pos::{
     SpanSnippetError,
 };
 
+pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;
+
+// `PResult` is used a lot. Make sure it doesn't unintentionally get bigger.
+// (See also the comment on `DiagnosticBuilderInner`.)
+#[cfg(target_arch = "x86_64")]
+static_assert_size!(PResult<'_, bool>, 16);
+
 /// Indicates the confidence in the correctness of a suggestion.
 ///
 /// All suggestions are marked with an `Applicability`. Tools use the applicability of a suggestion

src/librustc_interface/interface.rs

@@ -16,7 +16,7 @@ use std::result;
 use std::sync::{Arc, Mutex};
 use syntax::{self, parse};
 use syntax::ast::{self, MetaItemKind};
-use syntax::parse::token;
+use syntax::token;
 use syntax::source_map::{FileName, FileLoader, SourceMap};
 use syntax::sess::ParseSess;
 use syntax_pos::edition;

src/librustc_interface/passes.rs

@@ -22,6 +22,7 @@ use rustc_codegen_utils::codegen_backend::CodegenBackend;
 use rustc_codegen_utils::link::filename_for_metadata;
 use rustc_data_structures::{box_region_allow_access, declare_box_region_type, parallel};
 use rustc_data_structures::sync::{Lrc, ParallelIterator, par_iter};
+use rustc_errors::PResult;
 use rustc_incremental;
 use rustc_metadata::cstore;
 use rustc_mir as mir;
@@ -36,7 +37,7 @@ use syntax::{self, ast, visit};
 use syntax::early_buffered_lints::BufferedEarlyLint;
 use syntax_expand::base::{NamedSyntaxExtension, ExtCtxt};
 use syntax::mut_visit::MutVisitor;
-use syntax::parse::{self, PResult};
+use syntax::parse;
 use syntax::util::node_count::NodeCounter;
 use syntax::symbol::Symbol;
 use syntax_pos::FileName;

src/librustc_resolve/build_reduced_graph.rs

@@ -33,7 +33,7 @@ use syntax::attr;
 use syntax::ast::{self, Block, ForeignItem, ForeignItemKind, Item, ItemKind, NodeId};
 use syntax::ast::{MetaItemKind, StmtKind, TraitItem, TraitItemKind};
 use syntax::feature_gate::is_builtin_attr;
-use syntax::parse::token::{self, Token};
+use syntax::token::{self, Token};
 use syntax::print::pprust;
 use syntax::{span_err, struct_span_err};
 use syntax::source_map::{respan, Spanned};

src/librustc_save_analysis/dump_visitor.rs

@@ -24,7 +24,7 @@ use std::path::Path;
 use std::env;
 
 use syntax::ast::{self, Attribute, NodeId, PatKind};
-use syntax::parse::token;
+use syntax::token;
 use syntax::visit::{self, Visitor};
 use syntax::print::pprust::{
     bounds_to_string,

src/librustc_save_analysis/lib.rs

@@ -29,7 +29,7 @@ use std::path::{Path, PathBuf};
 
 use syntax::ast::{self, Attribute, DUMMY_NODE_ID, NodeId, PatKind};
 use syntax::source_map::Spanned;
-use syntax::parse::lexer::comments::strip_doc_comment_decoration;
+use syntax::util::comments::strip_doc_comment_decoration;
 use syntax::print::pprust;
 use syntax::visit::{self, Visitor};
 use syntax::print::pprust::{param_to_string, ty_to_string};

src/librustc_save_analysis/span_utils.rs

@@ -3,7 +3,7 @@ use rustc::session::Session;
 use crate::generated_code;
 
 use syntax::parse::lexer::{self, StringReader};
-use syntax::parse::token::{self, TokenKind};
+use syntax::token::{self, TokenKind};
 use syntax_pos::*;
 
 #[derive(Clone)]

src/librustdoc/clean/mod.rs

@@ -28,7 +28,7 @@ use rustc::ty::layout::VariantIdx;
 use rustc::util::nodemap::{FxHashMap, FxHashSet};
 use syntax::ast::{self, Attribute, AttrStyle, AttrKind, Ident};
 use syntax::attr;
-use syntax::parse::lexer::comments;
+use syntax::util::comments;
 use syntax::source_map::DUMMY_SP;
 use syntax_pos::symbol::{Symbol, kw, sym};
 use syntax_pos::hygiene::MacroKind;

src/librustdoc/html/highlight.rs

@@ -13,7 +13,7 @@ use std::io::prelude::*;
 
 use syntax::source_map::SourceMap;
 use syntax::parse::lexer;
-use syntax::parse::token::{self, Token};
+use syntax::token::{self, Token};
 use syntax::sess::ParseSess;
 use syntax::symbol::{kw, sym};
 use syntax_pos::{Span, FileName};

src/librustdoc/passes/check_code_block_syntax.rs

@@ -1,6 +1,6 @@
 use errors::Applicability;
 use syntax::parse::lexer::{StringReader as Lexer};
-use syntax::parse::token;
+use syntax::token;
 use syntax::sess::ParseSess;
 use syntax::source_map::FilePathMapping;
 use syntax_pos::{InnerSpan, FileName};

src/libsyntax/ast.rs

@@ -24,9 +24,9 @@ pub use crate::util::parser::ExprPrecedence;
 
 pub use syntax_pos::symbol::{Ident, Symbol as Name};
 
-use crate::parse::token::{self, DelimToken};
 use crate::ptr::P;
 use crate::source_map::{dummy_spanned, respan, Spanned};
+use crate::token::{self, DelimToken};
 use crate::tokenstream::TokenStream;
 
 use syntax_pos::symbol::{kw, sym, Symbol};

src/libsyntax/attr/mod.rs

@@ -14,17 +14,17 @@ use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem};
 use crate::ast::{Lit, LitKind, Expr, Item, Local, Stmt, StmtKind, GenericParam};
 use crate::mut_visit::visit_clobber;
 use crate::source_map::{BytePos, Spanned};
-use crate::parse::lexer::comments::doc_comment_style;
 use crate::parse;
-use crate::parse::PResult;
-use crate::parse::token::{self, Token};
+use crate::token::{self, Token};
 use crate::ptr::P;
 use crate::sess::ParseSess;
 use crate::symbol::{sym, Symbol};
 use crate::ThinVec;
 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
 use crate::GLOBALS;
 
+use errors::PResult;
+
 use log::debug;
 use syntax_pos::Span;
 
@@ -400,11 +400,11 @@ pub fn mk_attr_outer(item: MetaItem) -> Attribute {
     mk_attr(AttrStyle::Outer, item.path, item.kind.tokens(item.span), item.span)
 }
 
-pub fn mk_doc_comment(comment: Symbol, span: Span) -> Attribute {
+pub fn mk_doc_comment(style: AttrStyle, comment: Symbol, span: Span) -> Attribute {
     Attribute {
         kind: AttrKind::DocComment(comment),
         id: mk_attr_id(),
-        style: doc_comment_style(&comment.as_str()),
+        style,
         span,
     }
 }

src/libsyntax/feature_gate/check.rs

@@ -11,7 +11,7 @@ use crate::attr::{self, check_builtin_attribute};
 use crate::source_map::Spanned;
 use crate::edition::{ALL_EDITIONS, Edition};
 use crate::visit::{self, FnKind, Visitor};
-use crate::parse::token;
+use crate::token;
 use crate::sess::ParseSess;
 use crate::symbol::{Symbol, sym};
 use crate::tokenstream::TokenTree;

src/libsyntax/lib.rs

@@ -85,7 +85,10 @@ pub mod diagnostics {
 pub mod error_codes;
 
 pub mod util {
+    crate mod classify;
+    pub mod comments;
     pub mod lev_distance;
+    crate mod literal;
     pub mod node_count;
     pub mod parser;
     pub mod map_in_place;
@@ -107,6 +110,7 @@ pub mod show_span;
 pub use syntax_pos::edition;
 pub use syntax_pos::symbol;
 pub mod sess;
+pub mod token;
 pub mod tokenstream;
 pub mod visit;
 

src/libsyntax/mut_visit.rs

@@ -9,7 +9,7 @@
 
 use crate::ast::*;
 use crate::source_map::{Spanned, respan};
-use crate::parse::token::{self, Token};
+use crate::token::{self, Token};
 use crate::ptr::P;
 use crate::ThinVec;
 use crate::tokenstream::*;

src/libsyntax/parse/lexer/mod.rs

@@ -1,7 +1,7 @@
-use crate::parse::token::{self, Token, TokenKind};
+use crate::token::{self, Token, TokenKind};
 use crate::sess::ParseSess;
 use crate::symbol::{sym, Symbol};
-use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char};
+use crate::util::comments;
 
 use errors::{FatalError, DiagnosticBuilder};
 use syntax_pos::{BytePos, Pos, Span};
@@ -16,9 +16,10 @@ use log::debug;
 #[cfg(test)]
 mod tests;
 
-pub mod comments;
 mod tokentrees;
 mod unicode_chars;
+mod unescape_error_reporting;
+use unescape_error_reporting::{emit_unescape_error, push_escaped_char};
 
 #[derive(Clone, Debug)]
 pub struct UnmatchedBrace {
@@ -178,7 +179,7 @@ impl<'a> StringReader<'a> {
             rustc_lexer::TokenKind::LineComment => {
                 let string = self.str_from(start);
                 // comments with only more "/"s are not doc comments
-                let tok = if is_doc_comment(string) {
+                let tok = if comments::is_line_doc_comment(string) {
                     self.forbid_bare_cr(start, string, "bare CR not allowed in doc-comment");
                     token::DocComment(Symbol::intern(string))
                 } else {
@@ -191,7 +192,7 @@ impl<'a> StringReader<'a> {
                 let string = self.str_from(start);
                 // block comments starting with "/**" or "/*!" are doc-comments
                 // but comments with only "*"s between two "/"s are not
-                let is_doc_comment = is_block_doc_comment(string);
+                let is_doc_comment = comments::is_block_doc_comment(string);
 
                 if !terminated {
                     let msg = if is_doc_comment {
@@ -642,18 +643,3 @@ impl<'a> StringReader<'a> {
         }
     }
 }
-
-fn is_doc_comment(s: &str) -> bool {
-    let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/') ||
-              s.starts_with("//!");
-    debug!("is {:?} a doc comment? {}", s, res);
-    res
-}
-
-fn is_block_doc_comment(s: &str) -> bool {
-    // Prevent `/**/` from being parsed as a doc comment
-    let res = ((s.starts_with("/**") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'*') ||
-               s.starts_with("/*!")) && s.len() >= 5;
-    debug!("is {:?} a doc comment? {}", s, res);
-    res
-}

src/libsyntax/parse/lexer/tests.rs

@@ -2,7 +2,8 @@ use super::*;
 
 use crate::symbol::Symbol;
 use crate::source_map::{SourceMap, FilePathMapping};
-use crate::parse::token;
+use crate::token;
+use crate::util::comments::is_doc_comment;
 use crate::with_default_globals;
 
 use errors::{Handler, emitter::EmitterWriter};

src/libsyntax/parse/lexer/tokentrees.rs

@@ -4,10 +4,11 @@ use syntax_pos::Span;
 use super::{StringReader, UnmatchedBrace};
 
 use crate::print::pprust::token_to_string;
-use crate::parse::token::{self, Token};
-use crate::parse::PResult;
+use crate::token::{self, Token};
 use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint};
 
+use errors::PResult;
+
 impl<'a> StringReader<'a> {
     crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
         let mut tt_reader = TokenTreesReader {

src/libsyntax/parse/unescape_error_reporting.rs src/libsyntax/parse/lexer/unescape_error_reporting.rs

@@ -4,7 +4,7 @@
 use super::StringReader;
 use errors::{Applicability, DiagnosticBuilder};
 use syntax_pos::{BytePos, Pos, Span, symbol::kw};
-use crate::parse::token;
+use crate::token;
 
 #[rustfmt::skip] // for line breaks
 const UNICODE_ARRAY: &[(char, &str, char)] = &[

src/libsyntax/parse/lexer/unicode_chars.rs

@@ -2,14 +2,12 @@
 
 use crate::ast;
 use crate::parse::parser::{Parser, emit_unclosed_delims, make_unclosed_delims_error};
-use crate::parse::token::Nonterminal;
+use crate::token::{self, Nonterminal};
 use crate::tokenstream::{self, TokenStream, TokenTree};
 use crate::print::pprust;
 use crate::sess::ParseSess;
 
-use errors::{FatalError, Level, Diagnostic, DiagnosticBuilder};
-#[cfg(target_arch = "x86_64")]
-use rustc_data_structures::static_assert_size;
+use errors::{PResult, FatalError, Level, Diagnostic};
 use rustc_data_structures::sync::Lrc;
 use syntax_pos::{Span, SourceFile, FileName};
 
@@ -25,18 +23,6 @@ mod tests;
 #[macro_use]
 pub mod parser;
 pub mod lexer;
-pub mod token;
-
-crate mod classify;
-crate mod literal;
-crate mod unescape_error_reporting;
-
-pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;
-
-// `PResult` is used a lot. Make sure it doesn't unintentionally get bigger.
-// (See also the comment on `DiagnosticBuilderInner`.)
-#[cfg(target_arch = "x86_64")]
-static_assert_size!(PResult<'_, bool>, 16);
 
 #[derive(Clone)]
 pub struct Directory<'a> {