diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index 856cc0884ac14..e613263a52c81 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -79,6 +79,11 @@ fn mk_uniq_vec_e(cx: ext_ctxt, sp: span, exprs: ~[@ast::expr]) -> @ast::expr { mk_vstore_e(cx, sp, mk_base_vec_e(cx, sp, exprs), ast::expr_vstore_uniq) } +fn mk_slice_vec_e(cx: ext_ctxt, sp: span, exprs: ~[@ast::expr]) -> + @ast::expr { + mk_vstore_e(cx, sp, mk_base_vec_e(cx, sp, exprs), + ast::expr_vstore_slice) +} fn mk_fixed_vec_e(cx: ext_ctxt, sp: span, exprs: ~[@ast::expr]) -> @ast::expr { mk_vstore_e(cx, sp, mk_base_vec_e(cx, sp, exprs), diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 69d067f1ddb0a..443a937d4eb7a 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -5,7 +5,7 @@ use ast::{crate, expr_, expr_mac, mac_invoc, mac_invoc_tt, use fold::*; use ext::base::*; use ext::qquote::{qq_helper}; -use parse::{parser, parse_expr_from_source_str, new_parser_from_tt}; +use parse::{parser, parse_expr_from_source_str, new_parser_from_tts}; use codemap::{span, ExpandedFrom}; diff --git a/src/libsyntax/ext/pipes/pipec.rs b/src/libsyntax/ext/pipes/pipec.rs index d03a0fde66c97..907a193d05c67 100644 --- a/src/libsyntax/ext/pipes/pipec.rs +++ b/src/libsyntax/ext/pipes/pipec.rs @@ -473,6 +473,7 @@ trait ext_ctxt_parse_utils { fn parse_item(s: ~str) -> @ast::item; fn parse_expr(s: ~str) -> @ast::expr; fn parse_stmt(s: ~str) -> @ast::stmt; + fn parse_tts(s: ~str) -> ~[ast::token_tree]; } impl ext_ctxt: ext_ctxt_parse_utils { @@ -508,4 +509,12 @@ impl ext_ctxt: ext_ctxt_parse_utils { self.cfg(), self.parse_sess()) } + + fn parse_tts(s: ~str) -> ~[ast::token_tree] { + parse::parse_tts_from_source_str( + ~"***protocol expansion***", + @(copy s), + self.cfg(), + self.parse_sess()) + } } diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 3cca48c750891..80a0616e732be 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -19,7 +19,7 @@ use token::*; pub mod rt { pub use ast::*; pub use parse::token::*; - pub use parse::new_parser_from_tt; + pub use parse::new_parser_from_tts; pub use codemap::BytePos; pub use codemap::span; } @@ -27,7 +27,7 @@ pub mod rt { pub fn expand_quote_tokens(cx: ext_ctxt, sp: span, tts: ~[ast::token_tree]) -> base::mac_result { - base::mr_expr(expand_tt(cx, sp, tts)) + base::mr_expr(expand_tts(cx, sp, tts)) } pub fn expand_quote_expr(cx: ext_ctxt, @@ -121,8 +121,7 @@ fn mk_span(cx: ext_ctxt, qsp: span, sp: span) -> @ast::expr { } }; - let span_path = ids_ext( - cx, ~[~"syntax", ~"ext", ~"quote", ~"rt", ~"span"]); + let span_path = ids_ext(cx, ~[~"span"]); build::mk_struct_e(cx, qsp, span_path, @@ -150,7 +149,7 @@ fn mk_ident(cx: ext_ctxt, sp: span, ident: ast::ident) -> @ast::expr { } fn mk_bytepos(cx: ext_ctxt, sp: span, bpos: BytePos) -> @ast::expr { - let path = ids_ext(cx, ~[~"syntax", ~"ext", ~"quote", ~"rt", ~"BytePos"]); + let path = ids_ext(cx, ~[~"BytePos"]); let arg = build::mk_uint(cx, sp, bpos.to_uint()); build::mk_call(cx, sp, path, ~[arg]) } @@ -318,17 +317,21 @@ fn mk_token(cx: ext_ctxt, sp: span, tok: token::Token) -> @ast::expr { fn mk_tt(cx: ext_ctxt, sp: span, tt: &ast::token_tree) -> @ast::expr { match *tt { - ast::tt_tok(sp, tok) => - build::mk_call(cx, sp, - ids_ext(cx, ~[~"tt_tok"]), - ~[mk_span(cx, sp, sp), - mk_token(cx, sp, tok)]), + ast::tt_tok(sp, tok) => { + let e_tok = + build::mk_call(cx, sp, + ids_ext(cx, ~[~"tt_tok"]), + ~[mk_span(cx, sp, sp), + mk_token(cx, sp, tok)]); + build::mk_uniq_vec_e(cx, sp, ~[e_tok]) + } ast::tt_delim(tts) => { - let e_tts = tts.map(|tt| mk_tt(cx, sp, tt)); - build::mk_call(cx, sp, - ids_ext(cx, ~[~"tt_delim"]), - ~[build::mk_uniq_vec_e(cx, sp, e_tts)]) + let e_delim = + build::mk_call(cx, sp, + ids_ext(cx, ~[~"tt_delim"]), + ~[mk_tts(cx, sp, tts)]); + build::mk_uniq_vec_e(cx, sp, ~[e_delim]) } ast::tt_seq(*) => fail ~"tt_seq in quote!", @@ -338,22 +341,25 @@ fn mk_tt(cx: ext_ctxt, sp: span, tt: &ast::token_tree) -> @ast::expr { } } +fn mk_tts(cx: ext_ctxt, sp: span, tts: &[ast::token_tree]) -> @ast::expr { + let e_tts = tts.map(|tt| mk_tt(cx, sp, tt)); + build::mk_call(cx, sp, + ids_ext(cx, ~[~"vec", ~"concat"]), + ~[build::mk_slice_vec_e(cx, sp, e_tts)]) +} -fn expand_tt(cx: ext_ctxt, - sp: span, - tts: ~[ast::token_tree]) -> @ast::expr { +fn expand_tts(cx: ext_ctxt, + sp: span, + tts: ~[ast::token_tree]) -> @ast::expr { // NB: It appears that the main parser loses its mind if we consider // $foo as a tt_nonterminal during the main parse, so we have to re-parse // under quote_depth > 0. This is silly and should go away; the _guess_ is // it has to do with transition away from supporting old-style macros, so // try removing it when enough of them are gone. - let p = parse::new_parser_from_tt(cx.parse_sess(), cx.cfg(), tts); + let p = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(), tts); p.quote_depth += 1u; - let tq = dvec::DVec(); - while p.token != token::EOF { - tq.push(p.parse_token_tree()); - } - let tts = tq.get(); + let tts = p.parse_all_token_trees(); + p.abort_if_errors(); // We want to emit a block expression that does a sequence of 'use's to // import the runtime module, followed by a tt expression. @@ -361,8 +367,7 @@ fn expand_tt(cx: ext_ctxt, ~"ext", ~"quote", ~"rt"])) ]; - build::mk_block(cx, sp, uses, ~[], - Some(mk_tt(cx, sp, &ast::tt_delim(tts)))) + build::mk_block(cx, sp, uses, ~[], Some(mk_tts(cx, sp, tts))) } fn expand_parse_call(cx: ext_ctxt, @@ -370,7 +375,7 @@ fn expand_parse_call(cx: ext_ctxt, parse_method: ~str, arg_exprs: ~[@ast::expr], tts: ~[ast::token_tree]) -> @ast::expr { - let tt_expr = expand_tt(cx, sp, tts); + let tts_expr = expand_tts(cx, sp, tts); let cfg_call = || build::mk_call_( cx, sp, build::mk_access(cx, sp, ids_ext(cx, ~[~"ext_cx"]), @@ -386,10 +391,10 @@ fn expand_parse_call(cx: ext_ctxt, ~"ext", ~"quote", ~"rt", - ~"new_parser_from_tt"]), + ~"new_parser_from_tts"]), ~[parse_sess_call(), cfg_call(), - build::mk_uniq_vec_e(cx, sp, ~[tt_expr])]); + tts_expr]); build::mk_call_(cx, sp, build::mk_access_(cx, sp, new_parser_call, diff --git a/src/libsyntax/parse.rs b/src/libsyntax/parse.rs index 2e0b204df8d04..3ddb03547cbb5 100644 --- a/src/libsyntax/parse.rs +++ b/src/libsyntax/parse.rs @@ -5,12 +5,13 @@ export new_parse_sess, new_parse_sess_special_handler; export next_node_id; export new_parser_from_file, new_parser_etc_from_file; export new_parser_from_source_str; -export new_parser_from_tt; +export new_parser_from_tts; export new_sub_parser_from_file; export parse_crate_from_file, parse_crate_from_crate_file; export parse_crate_from_source_str; export parse_expr_from_source_str, parse_item_from_source_str; export parse_stmt_from_source_str; +export parse_tts_from_source_str; export parse_from_source_str; use parser::Parser; @@ -127,6 +128,16 @@ fn parse_stmt_from_source_str(name: ~str, source: @~str, cfg: ast::crate_cfg, return r; } +fn parse_tts_from_source_str(name: ~str, source: @~str, cfg: ast::crate_cfg, + sess: parse_sess) -> ~[ast::token_tree] { + let p = new_parser_from_source_str(sess, cfg, name, + codemap::FssNone, source); + p.quote_depth += 1u; + let r = p.parse_all_token_trees(); + p.abort_if_errors(); + return r; +} + fn parse_from_source_str(f: fn (p: Parser) -> T, name: ~str, ss: codemap::FileSubstr, source: @~str, cfg: ast::crate_cfg, @@ -199,9 +210,9 @@ fn new_sub_parser_from_file(sess: parse_sess, cfg: ast::crate_cfg, } } -fn new_parser_from_tt(sess: parse_sess, cfg: ast::crate_cfg, - tt: ~[ast::token_tree]) -> Parser { +fn new_parser_from_tts(sess: parse_sess, cfg: ast::crate_cfg, + tts: ~[ast::token_tree]) -> Parser { let trdr = lexer::new_tt_reader(sess.span_diagnostic, sess.interner, - None, tt); + None, tts); return Parser(sess, cfg, trdr as reader) } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 0df61a7f044e9..925934d165f68 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -1321,6 +1321,14 @@ impl Parser { }; } + fn parse_all_token_trees() -> ~[token_tree] { + let tts = DVec(); + while self.token != token::EOF { + tts.push(self.parse_token_tree()); + } + tts.get() + } + fn parse_matchers() -> ~[matcher] { // unification of matchers and token_trees would vastly improve // the interpolation of matchers diff --git a/src/test/run-pass-fulldeps/quote-tokens.rs b/src/test/run-pass-fulldeps/quote-tokens.rs index 7149c73be8076..2e71d213428d1 100644 --- a/src/test/run-pass-fulldeps/quote-tokens.rs +++ b/src/test/run-pass-fulldeps/quote-tokens.rs @@ -1,10 +1,12 @@ +#[allow(non_implicitly_copyable_typarams)]; + extern mod syntax; use syntax::ext::base::ext_ctxt; fn syntax_extension(ext_cx: @ext_ctxt) { - let e_toks : syntax::ast::token_tree = quote_tokens!(1 + 2); - let p_toks : syntax::ast::token_tree = quote_tokens!((x, 1 .. 4, *)); + let e_toks : ~[syntax::ast::token_tree] = quote_tokens!(1 + 2); + let p_toks : ~[syntax::ast::token_tree] = quote_tokens!((x, 1 .. 4, *)); let _a: @syntax::ast::expr = quote_expr!(1 + 2); let _b: Option<@syntax::ast::item> = quote_item!( const foo : int = $e_toks; ); @@ -14,6 +16,6 @@ fn syntax_extension(ext_cx: @ext_ctxt) { } fn main() { - let _x: syntax::ast::token_tree = quote_tokens!(a::Foo::foo()); + let _x: ~[syntax::ast::token_tree] = quote_tokens!(a::Foo::foo()); }