Rollup merge of rust-lang#33041 - petrochenkov:path, r=nrc,Manishearth

 Paths are mostly parsed without taking whitespaces into account, e.g. `std :: vec :: Vec :: new ()` parses successfully, however, there are some special cases involving keywords `super`, `self` and `Self`. For example, `self::` is considered a path start only if there are no spaces between `self` and `::`. These restrictions probably made sense when `self` and friends weren't keywords, but now they are unnecessary.

The first two commits remove this special treatment of whitespaces by removing `token::IdentStyle` entirely and therefore fix rust-lang#14109.
This change also affects naked `self` and `super` (which are not tightly followed by `::`, obviously) they can now be parsed as paths, however they are still not resolved correctly in imports (cc @jseyfried, see `compile-fail/use-keyword.rs`), so rust-lang#29036 is not completely fixed.

The third commit also makes `super`, `self`, `Self` and `static` keywords nominally (before this they acted as keywords for all purposes) and removes most of remaining \"special idents\".

The last commit (before tests) contains some small improvements - some qualified paths with type parameters are parsed correctly, `parse_path` is not used for parsing single identifiers, imports are sanity checked for absence of type parameters - such type parameters can be generated by syntax extensions or by macros when rust-lang#10415 is fixed (~~soon!~~already!).

This patch changes some pretty basic things in `libsyntax`, like `token::Token` and the keyword list, so it's a plugin-[breaking-change].

r? @eddyb

src/etc/generate-keyword-tests.py

@@ -34,15 +34,17 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// compile-flags: -Z parse-only
+
 // This file was auto-generated using 'src/etc/generate-keyword-tests.py %s'
 
 fn main() {
- let %s = "foo"; //~ error: ident
+ let %s = "foo"; //~ error: expected pattern, found keyword `%s`
 }
 """
 
 test_dir = os.path.abspath(
- os.path.join(os.path.dirname(__file__), '../test/compile-fail')
+ os.path.join(os.path.dirname(__file__), '../test/parse-fail')
 )
 
 for kw in sys.argv[1:]:
@@ -53,7 +55,7 @@
  os.chmod(test_file, stat.S_IWUSR)
 
  with open(test_file, 'wt') as f:
- f.write(template % (datetime.datetime.now().year, kw, kw))
+ f.write(template % (datetime.datetime.now().year, kw, kw, kw))
 
  # mark file read-only
  os.chmod(test_file, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)

src/librustc/hir/fold.rs

@@ -18,7 +18,7 @@ use syntax::attr::ThinAttributesExt;
 use hir;
 use syntax::codemap::{respan, Span, Spanned};
 use syntax::ptr::P;
-use syntax::parse::token;
+use syntax::parse::token::keywords;
 use syntax::util::move_map::MoveMap;
 
 pub trait Folder : Sized {
@@ -867,7 +867,7 @@ pub fn noop_fold_crate<T: Folder>(Crate { module, attrs, config, span,
  let config = folder.fold_meta_items(config);
 
  let crate_mod = folder.fold_item(hir::Item {
- name: token::special_idents::invalid.name,
+ name: keywords::Invalid.name(),
  attrs: attrs,
  id: DUMMY_NODE_ID,
  vis: hir::Public,

src/librustc/hir/print.rs

@@ -14,9 +14,8 @@ use syntax::abi::Abi;
 use syntax::ast;
 use syntax::codemap::{self, CodeMap, BytePos, Spanned};
 use syntax::errors;
-use syntax::parse::token::{self, BinOpToken};
+use syntax::parse::token::{self, keywords, BinOpToken};
 use syntax::parse::lexer::comments;
-use syntax::parse;
 use syntax::print::pp::{self, break_offset, word, space, hardbreak};
 use syntax::print::pp::{Breaks, eof};
 use syntax::print::pp::Breaks::{Consistent, Inconsistent};
@@ -2209,9 +2208,8 @@ impl<'a> State<'a> {
  hir::TyInfer if is_closure => self.print_pat(&input.pat)?,
  _ => {
  match input.pat.node {
- PatKind::Ident(_, ref path1, _) if
- path1.node.name ==
- parse::token::special_idents::invalid.name => {
+ PatKind::Ident(_, ref path1, _)
+ if path1.node.name == keywords::Invalid.name() => {
  // Do nothing.
  }
  _ => {

src/librustc/middle/liveness.rs

@@ -125,7 +125,7 @@ use std::io;
 use std::rc::Rc;
 use syntax::ast::{self, NodeId};
 use syntax::codemap::{BytePos, original_sp, Span};
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 use syntax::ptr::P;
 
 use hir::Expr;
@@ -1578,7 +1578,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
  let var = self.variable(p_id, sp);
  // Ignore unused self.
  let name = path1.node;
- if name != special_idents::self_.name {
+ if name != keywords::SelfValue.name() {
  if !self.warn_about_unused(sp, p_id, entry_ln, var) {
  if self.live_on_entry(entry_ln, var).is_none() {
  self.report_dead_assign(p_id, sp, var, true);

src/librustc/middle/resolve_lifetime.rs

@@ -29,7 +29,7 @@ use std::fmt;
 use std::mem::replace;
 use syntax::ast;
 use syntax::codemap::Span;
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 use util::nodemap::NodeMap;
 
 use hir;
@@ -245,7 +245,7 @@ impl<'a, 'v> Visitor<'v> for LifetimeContext<'a> {
  }
 
  fn visit_lifetime(&mut self, lifetime_ref: &hir::Lifetime) {
- if lifetime_ref.name == special_idents::static_lifetime.name {
+ if lifetime_ref.name == keywords::StaticLifetime.name() {
  self.insert_lifetime(lifetime_ref, DefStaticRegion);
  return;
  }
@@ -672,9 +672,8 @@ impl<'a> LifetimeContext<'a> {
  for i in 0..lifetimes.len() {
  let lifetime_i = &lifetimes[i];
 
- let special_idents = [special_idents::static_lifetime];
  for lifetime in lifetimes {
- if special_idents.iter().any(|&i| i.name == lifetime.lifetime.name) {
+ if lifetime.lifetime.name == keywords::StaticLifetime.name() {
  span_err!(self.sess, lifetime.lifetime.span, E0262,
  "invalid lifetime parameter name: `{}`", lifetime.lifetime.name);
  }

src/librustc/mir/repr.rs

@@ -200,7 +200,7 @@ pub struct ArgDecl<'tcx> {
  /// and has to be collected from multiple actual arguments.
  pub spread: bool,
 
- /// Either special_idents::invalid or the name of a single-binding
+ /// Either keywords::Invalid or the name of a single-binding
  /// pattern associated with this argument. Useful for debuginfo.
  pub debug_name: Name
 }

src/librustc/ty/context.rs

@@ -44,7 +44,7 @@ use std::hash::{Hash, Hasher};
 use std::rc::Rc;
 use syntax::ast::{self, Name, NodeId};
 use syntax::attr;
-use syntax::parse::token::{self, special_idents};
+use syntax::parse::token::{self, keywords};
 
 use hir;
 
@@ -1069,7 +1069,7 @@ impl<'tcx> TyCtxt<'tcx> {
  }
 
  pub fn mk_self_type(&self) -> Ty<'tcx> {
- self.mk_param(subst::SelfSpace, 0, special_idents::type_self.name)
+ self.mk_param(subst::SelfSpace, 0, keywords::SelfType.name())
  }
 
  pub fn mk_param_from_def(&self, def: &ty::TypeParameterDef) -> Ty<'tcx> {

src/librustc/ty/sty.rs

@@ -24,7 +24,7 @@ use std::ops;
 use std::mem;
 use syntax::abi;
 use syntax::ast::{self, Name};
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 
 use serialize::{Decodable, Decoder};
 
@@ -533,7 +533,7 @@ impl ParamTy {
  }
 
  pub fn for_self() -> ParamTy {
- ParamTy::new(subst::SelfSpace, 0, special_idents::type_self.name)
+ ParamTy::new(subst::SelfSpace, 0, keywords::SelfType.name())
  }
 
  pub fn for_def(def: &ty::TypeParameterDef) -> ParamTy {

src/librustc_mir/build/mod.rs

@@ -18,7 +18,7 @@ use rustc::hir::pat_util::pat_is_binding;
 use std::ops::{Index, IndexMut};
 use syntax::ast;
 use syntax::codemap::Span;
-use syntax::parse::token;
+use syntax::parse::token::keywords;
 
 pub struct Builder<'a, 'tcx: 'a> {
  hir: Cx<'a, 'tcx>,
@@ -238,7 +238,7 @@ pub fn construct<'a,'tcx>(hir: Cx<'a,'tcx>,
  ty::UpvarCapture::ByRef(..) => true
  });
  let mut decl = UpvarDecl {
- debug_name: token::special_idents::invalid.name,
+ debug_name: keywords::Invalid.name(),
  by_ref: by_ref
  };
  if let Some(hir::map::NodeLocal(pat)) = tcx.map.find(fv.def.var_id()) {
@@ -296,7 +296,7 @@ impl<'a,'tcx> Builder<'a,'tcx> {
  self.schedule_drop(pattern.as_ref().map_or(ast_block.span, |pat| pat.span),
  argument_extent, &lvalue, ty);
 
- let mut name = token::special_idents::invalid.name;
+ let mut name = keywords::Invalid.name();
  if let Some(pat) = pattern {
  if let hir::PatKind::Ident(_, ref ident, _) = pat.node {
  if pat_is_binding(&self.hir.tcx().def_map.borrow(), pat) {

src/librustc_resolve/build_reduced_graph.rs

@@ -112,15 +112,14 @@ impl<'b, 'tcx:'b> Resolver<'b, 'tcx> {
  !segment.parameters.bindings().is_empty()
  });
  if found_param {
- self.session.span_err(path.span,
- "type or lifetime parameter is found in import path");
+ self.session.span_err(path.span, "type or lifetime parameters in import path");
  }
 
  // Checking for special identifiers in path
  // prevent `self` or `super` at beginning of global path
  if path.global && path.segments.len() > 0 {
  let first = path.segments[0].identifier.name;
- if first == keywords::Super.to_name() || first == keywords::SelfValue.to_name() {
+ if first == keywords::Super.name() || first == keywords::SelfValue.name() {
  self.session.add_lint(
  lint::builtin::SUPER_OR_SELF_IN_GLOBAL_PATH, id, path.span,
  format!("expected identifier, found keyword `{}`", first)

src/librustc_resolve/lib.rs

@@ -62,7 +62,7 @@ use syntax::ast::{CRATE_NODE_ID, Name, NodeId, CrateNum, IntTy, UintTy};
 use syntax::attr::AttrMetaMethods;
 use syntax::codemap::{self, Span, Pos};
 use syntax::errors::DiagnosticBuilder;
-use syntax::parse::token::{self, special_names, special_idents};
+use syntax::parse::token::{self, keywords};
 use syntax::util::lev_distance::find_best_match_for_name;
 
 use rustc::hir::intravisit::{self, FnKind, Visitor};
@@ -1954,8 +1954,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
  let mut self_type_rib = Rib::new(NormalRibKind);
 
  // plain insert (no renaming, types are not currently hygienic....)
- let name = special_names::type_self;
- self_type_rib.bindings.insert(name, self_def);
+ self_type_rib.bindings.insert(keywords::SelfType.name(), self_def);
  self.type_ribs.push(self_type_rib);
  f(self);
  if !self.resolved {
@@ -2195,11 +2194,10 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
  "type name"
  };
 
- let self_type_name = special_idents::type_self.name;
  let is_invalid_self_type_name = path.segments.len() > 0 &&
  maybe_qself.is_none() &&
  path.segments[0].identifier.name ==
- self_type_name;
+ keywords::SelfType.name();
  if is_invalid_self_type_name {
  resolve_error(self,
  ty.span,
@@ -2643,7 +2641,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
  namespace: Namespace,
  record_used: bool)
  -> Option<LocalDef> {
- if identifier.name == special_idents::invalid.name {
+ if identifier.unhygienic_name == keywords::Invalid.name() {
  return Some(LocalDef::from_def(Def::Err));
  }
 
@@ -3074,7 +3072,8 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
  false // Stop advancing
  });
 
- if method_scope && special_names::self_.as_str() == &path_name[..] {
+ if method_scope &&
+ &path_name[..] == keywords::SelfValue.name().as_str() {
  resolve_error(self,
  expr.span,
  ResolutionError::SelfNotAvailableInStaticMethod);
@@ -3612,7 +3611,7 @@ fn module_to_string(module: Module) -> String {
  }
  BlockParentLink(ref module, _) => {
  // danger, shouldn't be ident?
- names.push(special_idents::opaque.name);
+ names.push(token::intern("<opaque>"));
  collect_mod(names, module);
  }
  }

src/librustc_save_analysis/dump_visitor.rs

@@ -1011,7 +1011,7 @@ impl<'v, 'l, 'tcx: 'l, 'll, D: Dump +'ll> Visitor<'v> for DumpVisitor<'l, 'tcx,
  span: sub_span.expect("No span found for use"),
  id: item.id,
  mod_id: mod_id,
- name: ident.name.to_string(),
+ name: ident.to_string(),
  scope: self.cur_scope
  }.normalize(&self.tcx));
  }
@@ -1075,7 +1075,7 @@ impl<'v, 'l, 'tcx: 'l, 'll, D: Dump +'ll> Visitor<'v> for DumpVisitor<'l, 'tcx,
  if !self.span.filter_generated(alias_span, item.span) {
  self.dumper.extern_crate(item.span, ExternCrateData {
  id: item.id,
- name: item.ident.name.to_string(),
+ name: item.ident.to_string(),
  crate_num: cnum,
  location: location,
  span: alias_span.expect("No span found for extern crate"),

src/librustc_trans/mir/mod.rs

@@ -22,7 +22,7 @@ use machine;
 use type_of;
 
 use syntax::codemap::DUMMY_SP;
-use syntax::parse::token;
+use syntax::parse::token::keywords;
 
 use std::ops::Deref;
 use std::rc::Rc;
@@ -286,7 +286,7 @@ fn arg_value_refs<'bcx, 'tcx>(bcx: &BlockAndBuilder<'bcx, 'tcx>,
  alloca: lltemp,
  address_operations: &ops
  };
- declare_local(bcx, token::special_idents::invalid.name,
+ declare_local(bcx, keywords::Invalid.name(),
  tupled_arg_ty, scope, variable_access,
  VariableKind::ArgumentVariable(arg_index + i + 1),
  bcx.fcx().span.unwrap_or(DUMMY_SP));

src/librustc_typeck/astconv.rs

@@ -73,7 +73,7 @@ use syntax::{abi, ast};
 use syntax::codemap::{Span, Pos};
 use syntax::errors::DiagnosticBuilder;
 use syntax::feature_gate::{GateIssue, emit_feature_err};
-use syntax::parse::token;
+use syntax::parse::token::{self, keywords};
 
 use rustc::hir::print as pprust;
 use rustc::hir;
@@ -1313,7 +1313,7 @@ fn associated_path_def_to_ty<'tcx>(this: &AstConv<'tcx>,
  let trait_node_id = tcx.map.as_local_node_id(trait_did).unwrap();
  match find_bound_for_assoc_item(this,
  trait_node_id,
- token::special_idents::type_self.name,
+ keywords::SelfType.name(),
  assoc_name,
  span) {
  Ok(bound) => bound,

src/librustc_typeck/check/mod.rs

@@ -120,7 +120,7 @@ use syntax::attr;
 use syntax::attr::AttrMetaMethods;
 use syntax::codemap::{self, Span, Spanned};
 use syntax::errors::DiagnosticBuilder;
-use syntax::parse::token::{self, InternedString, special_idents};
+use syntax::parse::token::{self, InternedString, keywords};
 use syntax::ptr::P;
 use syntax::util::lev_distance::find_best_match_for_name;
 
@@ -2851,7 +2851,7 @@ fn check_expr_with_expectation_and_lvalue_pref<'a, 'tcx>(fcx: &FnCtxt<'a, 'tcx>,
  method_ty
  }
  Err(error) => {
- if method_name.node != special_idents::invalid.name {
+ if method_name.node != keywords::Invalid.name() {
  method::report_error(fcx, method_name.span, expr_t,
  method_name.node, Some(rcvr), error);
  }
@@ -2990,7 +2990,7 @@ fn check_expr_with_expectation_and_lvalue_pref<'a, 'tcx>(fcx: &FnCtxt<'a, 'tcx>,
  let msg = format!("field `{}` of struct `{}` is private", field.node, struct_path);
  fcx.tcx().sess.span_err(expr.span, &msg);
  fcx.write_ty(expr.id, field_ty);
- } else if field.node == special_idents::invalid.name {
+ } else if field.node == keywords::Invalid.name() {
  fcx.write_error(expr.id);
  } else if method::exists(fcx, field.span, field.node, expr_t, expr.id) {
  fcx.type_error_struct(field.span,
@@ -3780,7 +3780,7 @@ pub fn resolve_ty_and_def_ufcs<'a, 'b, 'tcx>(fcx: &FnCtxt<'b, 'tcx>,
  method::MethodError::PrivateMatch(def) => Some(def),
  _ => None,
  };
- if item_name != special_idents::invalid.name {
+ if item_name != keywords::Invalid.name() {
  method::report_error(fcx, span, ty, item_name, None, error);
  }
  def

src/librustc_typeck/check/wfcheck.rs

@@ -24,7 +24,7 @@ use std::collections::HashSet;
 use syntax::ast;
 use syntax::codemap::{Span};
 use syntax::errors::DiagnosticBuilder;
-use syntax::parse::token::{special_idents};
+use syntax::parse::token::keywords;
 use rustc::hir::intravisit::{self, Visitor};
 use rustc::hir;
 
@@ -472,7 +472,7 @@ impl<'ccx, 'tcx> CheckTypeWellFormedVisitor<'ccx, 'tcx> {
  {
  let name = match space {
  TypeSpace => ast_generics.ty_params[index].name,
- SelfSpace => special_idents::type_self.name,
+ SelfSpace => keywords::SelfType.name(),
  FnSpace => bug!("Fn space occupied?"),
  };
 

src/librustc_typeck/collect.rs

@@ -93,7 +93,7 @@ use syntax::abi;
 use syntax::ast;
 use syntax::attr;
 use syntax::codemap::Span;
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 use syntax::ptr::P;
 use rustc::hir::{self, PatKind};
 use rustc::hir::intravisit;
@@ -1655,7 +1655,7 @@ fn ty_generics_for_trait<'a, 'tcx>(ccx: &CrateCtxt<'a, 'tcx>,
  let def = ty::TypeParameterDef {
  space: SelfSpace,
  index: 0,
- name: special_idents::type_self.name,
+ name: keywords::SelfType.name(),
  def_id: ccx.tcx.map.local_def_id(param_id),
  default_def_id: ccx.tcx.map.local_def_id(parent),
  default: None,