diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 3e85565beb6d1..495ad176542c1 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -18,8 +18,7 @@ use errors::DiagnosticBuilder; use ext::expand::{self, Invocation, Expansion}; use ext::hygiene::Mark; use fold::{self, Folder}; -use parse; -use parse::parser::{self, Parser}; +use parse::{self, parser}; use parse::token; use parse::token::{InternedString, str_to_ident}; use ptr::P; @@ -188,146 +187,6 @@ impl AttrProcMacro for F } } -pub struct TokResult<'a> { - pub parser: Parser<'a>, - pub span: Span, -} - -impl<'a> TokResult<'a> { - // There is quite a lot of overlap here with ParserAnyMacro in ext/tt/macro_rules.rs - // We could probably share more code. - // FIXME(#36641) Unify TokResult and ParserAnyMacro. - fn ensure_complete_parse(&mut self, allow_semi: bool) { - let macro_span = &self.span; - self.parser.ensure_complete_parse(allow_semi, |parser| { - let token_str = parser.this_token_to_string(); - let msg = format!("macro expansion ignores token `{}` and any following", token_str); - let span = parser.span; - parser.diagnostic() - .struct_span_err(span, &msg) - .span_note(*macro_span, "caused by the macro expansion here") - .emit(); - }); - } -} - -impl<'a> MacResult for TokResult<'a> { - fn make_items(mut self: Box) -> Option>> { - if self.parser.sess.span_diagnostic.has_errors() { - return Some(SmallVector::zero()); - } - - let mut items = SmallVector::zero(); - loop { - match self.parser.parse_item() { - Ok(Some(item)) => items.push(item), - Ok(None) => { - self.ensure_complete_parse(false); - return Some(items); - } - Err(mut e) => { - e.emit(); - return Some(SmallVector::zero()); - } - } - } - } - - fn make_impl_items(mut self: Box) -> Option> { - let mut items = SmallVector::zero(); - loop { - if self.parser.token == token::Eof { - break; - } - match self.parser.parse_impl_item() { - Ok(item) => items.push(item), - Err(mut e) => { - e.emit(); - return Some(SmallVector::zero()); - } - } - } - self.ensure_complete_parse(false); - Some(items) - } - - fn make_trait_items(mut self: Box) -> Option> { - let mut items = SmallVector::zero(); - loop { - if self.parser.token == token::Eof { - break; - } - match self.parser.parse_trait_item() { - Ok(item) => items.push(item), - Err(mut e) => { - e.emit(); - return Some(SmallVector::zero()); - } - } - } - self.ensure_complete_parse(false); - Some(items) - } - - fn make_expr(mut self: Box) -> Option> { - match self.parser.parse_expr() { - Ok(e) => { - self.ensure_complete_parse(true); - Some(e) - } - Err(mut e) => { - e.emit(); - Some(DummyResult::raw_expr(self.span)) - } - } - } - - fn make_pat(mut self: Box) -> Option> { - match self.parser.parse_pat() { - Ok(e) => { - self.ensure_complete_parse(false); - Some(e) - } - Err(mut e) => { - e.emit(); - Some(P(DummyResult::raw_pat(self.span))) - } - } - } - - fn make_stmts(mut self: Box) -> Option> { - let mut stmts = SmallVector::zero(); - loop { - if self.parser.token == token::Eof { - break; - } - match self.parser.parse_full_stmt(false) { - Ok(Some(stmt)) => stmts.push(stmt), - Ok(None) => { /* continue */ } - Err(mut e) => { - e.emit(); - return Some(SmallVector::zero()); - } - } - } - self.ensure_complete_parse(false); - Some(stmts) - } - - fn make_ty(mut self: Box) -> Option> { - match self.parser.parse_ty() { - Ok(e) => { - self.ensure_complete_parse(false); - Some(e) - } - Err(mut e) => { - e.emit(); - Some(DummyResult::raw_ty(self.span)) - } - } - } -} - /// Represents a thing that maps token trees to Macro Results pub trait TTMacroExpander { fn expand<'cx>(&self, diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 7359c21ecccac..7c43f1f307698 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -9,7 +9,7 @@ // except according to those terms. use ast::{Block, Crate, Ident, Mac_, PatKind}; -use ast::{MacStmtStyle, StmtKind, ItemKind}; +use ast::{Name, MacStmtStyle, StmtKind, ItemKind}; use ast; use ext::hygiene::Mark; use ext::placeholders::{placeholder, PlaceholderExpander}; @@ -21,9 +21,9 @@ use ext::base::*; use feature_gate::{self, Features}; use fold; use fold::*; -use parse::{ParseSess, lexer}; +use parse::{ParseSess, PResult, lexer}; use parse::parser::Parser; -use parse::token::{intern, keywords}; +use parse::token::{self, intern, keywords}; use print::pprust; use ptr::P; use tokenstream::{TokenTree, TokenStream}; @@ -38,12 +38,12 @@ macro_rules! expansions { ($($kind:ident: $ty:ty [$($vec:ident, $ty_elt:ty)*], $kind_name:expr, .$make:ident, $(.$fold:ident)* $(lift .$fold_elt:ident)*, $(.$visit:ident)* $(lift .$visit_elt:ident)*;)*) => { - #[derive(Copy, Clone)] + #[derive(Copy, Clone, PartialEq, Eq)] pub enum ExpansionKind { OptExpr, $( $kind, )* } pub enum Expansion { OptExpr(Option>), $( $kind($ty), )* } impl ExpansionKind { - fn name(self) -> &'static str { + pub fn name(self) -> &'static str { match self { ExpansionKind::OptExpr => "expression", $( ExpansionKind::$kind => $kind_name, )* @@ -106,6 +106,12 @@ macro_rules! expansions { self.expand(Expansion::$kind(SmallVector::one(node))).$make() })*)* } + + impl<'a> MacResult for ::ext::tt::macro_rules::ParserAnyMacro<'a> { + $(fn $make(self: Box<::ext::tt::macro_rules::ParserAnyMacro<'a>>) -> Option<$ty> { + Some(self.make(ExpansionKind::$kind).$make()) + })* + } } } @@ -293,10 +299,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> { }; attr::mark_used(&attr); + let name = intern(&attr.name()); self.cx.bt_push(ExpnInfo { call_site: attr.span, callee: NameAndSpan { - format: MacroAttribute(intern(&attr.name())), + format: MacroAttribute(name), span: Some(attr.span), allow_internal_unstable: false, } @@ -319,14 +326,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { let item_toks = TokenStream::from_tts(tts_for_item(&item, &self.cx.parse_sess)); let tok_result = mac.expand(self.cx, attr.span, attr_toks, item_toks); - let parser = self.cx.new_parser_from_tts(&tok_result.to_tts()); - let result = Box::new(TokResult { parser: parser, span: attr.span }); - - kind.make_from(result).unwrap_or_else(|| { - let msg = format!("macro could not be expanded into {} position", kind.name()); - self.cx.span_err(attr.span, &msg); - kind.dummy(attr.span) - }) + self.parse_expansion(tok_result, kind, name, attr.span) } _ => unreachable!(), } @@ -423,14 +423,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> { }, }); - - let tok_result = expandfun.expand(self.cx, - span, - TokenStream::from_tts(marked_tts)); - let parser = self.cx.new_parser_from_tts(&tok_result.to_tts()); - let result = Box::new(TokResult { parser: parser, span: span }); - // FIXME better span info. - kind.make_from(result).map(|i| i.fold_with(&mut ChangeSpan { span: span })) + let toks = TokenStream::from_tts(marked_tts); + let tok_result = expandfun.expand(self.cx, span, toks); + Some(self.parse_expansion(tok_result, kind, extname, span)) } }; @@ -448,6 +443,75 @@ impl<'a, 'b> MacroExpander<'a, 'b> { expn_id: Some(self.cx.backtrace()), }) } + + fn parse_expansion(&mut self, toks: TokenStream, kind: ExpansionKind, name: Name, span: Span) + -> Expansion { + let mut parser = self.cx.new_parser_from_tts(&toks.to_tts()); + let expansion = match parser.parse_expansion(kind, false) { + Ok(expansion) => expansion, + Err(mut err) => { + err.emit(); + return kind.dummy(span); + } + }; + parser.ensure_complete_parse(name, kind.name(), span); + // FIXME better span info + expansion.fold_with(&mut ChangeSpan { span: span }) + } +} + +impl<'a> Parser<'a> { + pub fn parse_expansion(&mut self, kind: ExpansionKind, macro_legacy_warnings: bool) + -> PResult<'a, Expansion> { + Ok(match kind { + ExpansionKind::Items => { + let mut items = SmallVector::zero(); + while let Some(item) = self.parse_item()? { + items.push(item); + } + Expansion::Items(items) + } + ExpansionKind::TraitItems => { + let mut items = SmallVector::zero(); + while self.token != token::Eof { + items.push(self.parse_trait_item()?); + } + Expansion::TraitItems(items) + } + ExpansionKind::ImplItems => { + let mut items = SmallVector::zero(); + while self.token != token::Eof { + items.push(self.parse_impl_item()?); + } + Expansion::ImplItems(items) + } + ExpansionKind::Stmts => { + let mut stmts = SmallVector::zero(); + while self.token != token::Eof { + if let Some(stmt) = self.parse_full_stmt(macro_legacy_warnings)? { + stmts.push(stmt); + } + } + Expansion::Stmts(stmts) + } + ExpansionKind::Expr => Expansion::Expr(self.parse_expr()?), + ExpansionKind::OptExpr => Expansion::OptExpr(Some(self.parse_expr()?)), + ExpansionKind::Ty => Expansion::Ty(self.parse_ty()?), + ExpansionKind::Pat => Expansion::Pat(self.parse_pat()?), + }) + } + + pub fn ensure_complete_parse(&mut self, macro_name: ast::Name, kind_name: &str, span: Span) { + if self.token != token::Eof { + let msg = format!("macro expansion ignores token `{}` and any following", + self.this_token_to_string()); + let mut err = self.diagnostic().struct_span_err(self.span, &msg); + let msg = format!("caused by the macro expansion here; the usage \ + of `{}!` is likely invalid in {} context", + macro_name, kind_name); + err.span_note(span, &msg).emit(); + } + } } struct InvocationCollector<'a, 'b: 'a> { diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 0eed3e5898c00..d222de2dd36c6 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -12,6 +12,7 @@ use {ast, attr}; use syntax_pos::{Span, DUMMY_SP}; use ext::base::{DummyResult, ExtCtxt, MacEager, MacResult, SyntaxExtension}; use ext::base::{IdentMacroExpander, NormalTT, TTMacroExpander}; +use ext::expand::{Expansion, ExpansionKind}; use ext::placeholders; use ext::tt::macro_parser::{Success, Error, Failure}; use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal}; @@ -22,18 +23,14 @@ use parse::parser::{Parser, Restrictions}; use parse::token::{self, gensym_ident, NtTT, Token}; use parse::token::Token::*; use print; -use ptr::P; use tokenstream::{self, TokenTree}; -use util::small_vector::SmallVector; - -use std::cell::RefCell; use std::collections::{HashMap}; use std::collections::hash_map::{Entry}; use std::rc::Rc; -struct ParserAnyMacro<'a> { - parser: RefCell>, +pub struct ParserAnyMacro<'a> { + parser: Parser<'a>, /// Span of the expansion site of the macro this parser is for site_span: Span, @@ -42,106 +39,20 @@ struct ParserAnyMacro<'a> { } impl<'a> ParserAnyMacro<'a> { - /// Make sure we don't have any tokens left to parse, so we don't - /// silently drop anything. `allow_semi` is so that "optional" - /// semicolons at the end of normal expressions aren't complained - /// about e.g. the semicolon in `macro_rules! kapow { () => { - /// panic!(); } }` doesn't get picked up by .parse_expr(), but it's - /// allowed to be there. - fn ensure_complete_parse(&self, allow_semi: bool, context: &str) { - let mut parser = self.parser.borrow_mut(); - parser.ensure_complete_parse(allow_semi, |parser| { - let token_str = parser.this_token_to_string(); - let msg = format!("macro expansion ignores token `{}` and any \ - following", - token_str); - let span = parser.span; - let mut err = parser.diagnostic().struct_span_err(span, &msg); - let msg = format!("caused by the macro expansion here; the usage \ - of `{}!` is likely invalid in {} context", - self.macro_ident, context); - err.span_note(self.site_span, &msg) - .emit(); - }); - } -} - -impl<'a> MacResult for ParserAnyMacro<'a> { - fn make_expr(self: Box>) -> Option> { - let ret = panictry!(self.parser.borrow_mut().parse_expr()); - self.ensure_complete_parse(true, "expression"); - Some(ret) - } - fn make_pat(self: Box>) -> Option> { - let ret = panictry!(self.parser.borrow_mut().parse_pat()); - self.ensure_complete_parse(false, "pattern"); - Some(ret) - } - fn make_items(self: Box>) -> Option>> { - let mut ret = SmallVector::zero(); - while let Some(item) = panictry!(self.parser.borrow_mut().parse_item()) { - ret.push(item); - } - self.ensure_complete_parse(false, "item"); - Some(ret) - } - - fn make_impl_items(self: Box>) - -> Option> { - let mut ret = SmallVector::zero(); - loop { - let mut parser = self.parser.borrow_mut(); - match parser.token { - token::Eof => break, - _ => ret.push(panictry!(parser.parse_impl_item())) - } - } - self.ensure_complete_parse(false, "item"); - Some(ret) - } - - fn make_trait_items(self: Box>) - -> Option> { - let mut ret = SmallVector::zero(); - loop { - let mut parser = self.parser.borrow_mut(); - match parser.token { - token::Eof => break, - _ => ret.push(panictry!(parser.parse_trait_item())) - } + pub fn make(mut self: Box>, kind: ExpansionKind) -> Expansion { + let ParserAnyMacro { site_span, macro_ident, ref mut parser } = *self; + let expansion = panictry!(parser.parse_expansion(kind, true)); + + // We allow semicolons at the end of expressions -- e.g. the semicolon in + // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, + // but `m!()` is allowed in expression positions (c.f. issue #34706). + if kind == ExpansionKind::Expr && parser.token == token::Semi { + parser.bump(); } - self.ensure_complete_parse(false, "item"); - Some(ret) - } - - - fn make_stmts(self: Box>) - -> Option> { - let mut ret = SmallVector::zero(); - loop { - let mut parser = self.parser.borrow_mut(); - match parser.token { - token::Eof => break, - _ => match parser.parse_full_stmt(true) { - Ok(maybe_stmt) => match maybe_stmt { - Some(stmt) => ret.push(stmt), - None => (), - }, - Err(mut e) => { - e.emit(); - break; - } - } - } - } - self.ensure_complete_parse(false, "statement"); - Some(ret) - } - fn make_ty(self: Box>) -> Option> { - let ret = panictry!(self.parser.borrow_mut().parse_ty()); - self.ensure_complete_parse(false, "type"); - Some(ret) + // Make sure we don't have any tokens left to parse so we don't silently drop anything. + parser.ensure_complete_parse(macro_ident.name, kind.name(), site_span); + expansion } } @@ -219,7 +130,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, // Let the context choose how to interpret the result. // Weird, but useful for X-macros. return Box::new(ParserAnyMacro { - parser: RefCell::new(p), + parser: p, // Pass along the original expansion site and the name of the macro // so we can print a useful error message if the parse of the expanded diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 23085fadc5e60..e83d003ab74dc 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -6171,15 +6171,4 @@ impl<'a> Parser<'a> { _ => Err(self.fatal("expected string literal")) } } - - pub fn ensure_complete_parse(&mut self, allow_semi: bool, on_err: F) - where F: FnOnce(&Parser) - { - if allow_semi && self.token == token::Semi { - self.bump(); - } - if self.token != token::Eof { - on_err(self); - } - } }