From 898e9a27230750f4c172761bc68119ea20337f98 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 30 May 2017 21:49:42 -0700 Subject: [PATCH] Finish porting to the `Synom` trait --- src/attr.rs | 249 +++--- src/constant.rs | 182 ++-- src/data.rs | 283 +++---- src/derive.rs | 181 +++- src/expr.rs | 1689 +++++++++++++++++++++---------------- src/generics.rs | 343 ++++---- src/ident.rs | 75 +- src/item.rs | 1074 +++++++++++------------ src/krate.rs | 75 +- src/lib.rs | 290 +------ src/lit.rs | 316 +------ src/mac.rs | 211 +---- src/op.rs | 147 ++-- src/ty.rs | 779 +++++++++-------- synom/src/delimited.rs | 69 +- synom/src/helper.rs | 209 +++-- synom/src/lib.rs | 719 +++------------- synom/src/tokens.rs | 40 +- tests/test_expr.rs | 2 +- tests/test_generics.rs | 6 +- tests/test_macro_input.rs | 20 +- tests/test_meta_item.rs | 14 +- tests/test_round_trip.rs | 2 +- tests/test_token_trees.rs | 7 +- 24 files changed, 3150 insertions(+), 3832 deletions(-) diff --git a/src/attr.rs b/src/attr.rs index 9e498d29c2..76b1503bbc 100644 --- a/src/attr.rs +++ b/src/attr.rs @@ -292,137 +292,123 @@ impl<'a, T> FilterAttrs<'a> for T #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use mac::TokenTree; - use mac::parsing::token_trees; - use ty::parsing::mod_style_path; - use proc_macro2::{self, TokenKind, OpKind, Literal}; + use synom::IResult; + use synom::tokens::*; + use proc_macro2::{TokenKind, OpKind, TokenTree}; fn eq() -> TokenTree { - TokenTree(proc_macro2::TokenTree { + TokenTree { span: Default::default(), kind: TokenKind::Op('=', OpKind::Alone), - }) + } } - fn doc(s: &str) -> TokenTree { - TokenTree(proc_macro2::TokenTree { - span: Default::default(), - kind: TokenKind::Literal(Literal::doccomment(s)), - }) + impl Attribute { + #[cfg(feature = "full")] + pub fn parse_inner(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + pound: syn!(Pound) >> + bang: syn!(Bang) >> + path_and_tts: brackets!(tuple!( + call!(::Path::parse_mod_style), + call!(::TokenTree::parse_list) + )) >> + ({ + let ((path, tts), bracket) = path_and_tts; + + Attribute { + style: AttrStyle::Inner(bang), + path: path, + tts: tts, + is_sugared_doc: false, + pound_token: pound, + bracket_token: bracket, + } + }) + ) + | + map!( + lit_doc_comment, + |lit| Attribute { + style: AttrStyle::Inner(tokens::Bang::default()), + path: "doc".into(), + tts: vec![ + ::TokenTree(eq()), + ::TokenTree(lit), + ], + is_sugared_doc: true, + pound_token: tokens::Pound::default(), + bracket_token: tokens::Bracket::default(), + } + ) + } + } + + pub fn parse_outer(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + pound: syn!(Pound) >> + path_and_tts: brackets!(tuple!( + call!(::Path::parse_mod_style), + call!(::TokenTree::parse_list) + )) >> + ({ + let ((path, tts), bracket) = path_and_tts; + + Attribute { + style: AttrStyle::Outer, + path: path, + tts: tts, + is_sugared_doc: false, + pound_token: pound, + bracket_token: bracket, + } + }) + ) + | + map!( + lit_doc_comment, + |lit| Attribute { + style: AttrStyle::Outer, + path: "doc".into(), + tts: vec![ + ::TokenTree(eq()), + ::TokenTree(lit), + ], + is_sugared_doc: true, + pound_token: tokens::Pound::default(), + bracket_token: tokens::Bracket::default(), + } + ) + } + } } - #[cfg(feature = "full")] - named!(pub inner_attr -> Attribute, alt!( - do_parse!( - punct!("#") >> - punct!("!") >> - path_and_tts: delim!(Bracket, tuple!(mod_style_path, token_trees)) >> - ({ - let (path, tts) = path_and_tts; - - Attribute { - style: AttrStyle::Inner(tokens::Bang::default()), - path: path, - tts: tts, - is_sugared_doc: false, - pound_token: tokens::Pound::default(), - bracket_token: tokens::Bracket::default(), - } - }) - ) - | - do_parse!( - comment: inner_doc_comment >> - (Attribute { - style: AttrStyle::Inner(tokens::Bang::default()), - path: "doc".into(), - tts: vec![ - eq(), - doc(&format!("//!{}", content)), - ], - is_sugared_doc: true, - pound_token: tokens::Pound::default(), - bracket_token: tokens::Bracket::default(), - }) - ) - | - do_parse!( - option!(whitespace) >> - peek!(tag!("/*!")) >> - com: block_comment >> - (Attribute { - style: AttrStyle::Inner(tokens::Bang::default()), - path: "doc".into(), - tts: vec![ - eq(), - doc(com), - ], - is_sugared_doc: true, - pound_token: tokens::Pound::default(), - bracket_token: tokens::Bracket::default(), - }) - ) - )); - - named!(pub outer_attr -> Attribute, alt!( - do_parse!( - punct!("#") >> - path_and_tts: delim!(Bracket, tuple!(mod_style_path, token_trees)) >> - ({ - let (path, tts) = path_and_tts; - - Attribute { - style: AttrStyle::Outer, - path: path, - tts: tts, - is_sugared_doc: false, - pound_token: tokens::Pound::default(), - bracket_token: tokens::Bracket::default(), - } - }) - ) - | - do_parse!( - punct!("///") >> - not!(tag!("/")) >> - content: take_until!("\n") >> - (Attribute { - style: AttrStyle::Outer, - path: "doc".into(), - tts: vec![ - eq(), - doc(&format!("///{}", content)), - ], - is_sugared_doc: true, - pound_token: tokens::Pound::default(), - bracket_token: tokens::Bracket::default(), - }) - ) - | - do_parse!( - option!(whitespace) >> - peek!(tuple!(tag!("/**"), not!(tag!("*")))) >> - com: block_comment >> - (Attribute { - style: AttrStyle::Outer, - path: "doc".into(), - tts: vec![ - eq(), - doc(com), - ], - is_sugared_doc: true, - pound_token: tokens::Pound::default(), - bracket_token: tokens::Bracket::default(), - }) - ) - )); + fn lit_doc_comment(input: &[TokenTree]) -> IResult<&[TokenTree], TokenTree> { + let mut tokens = input.iter(); + let tok = match tokens.next() { + Some(tok) => tok, + None => return IResult::Error, + }; + let literal = match tok.kind { + TokenKind::Literal(ref l) => l.to_string(), + _ => return IResult::Error, + }; + if literal.starts_with("//") || literal.starts_with("/*") { + IResult::Done(tokens.as_slice(), tok.clone()) + } else { + IResult::Error + } + } } #[cfg(feature = "printing")] mod printing { use super::*; use quote::{Tokens, ToTokens}; - use proc_macro2::{Literal, TokenTree}; impl ToTokens for Attribute { fn to_tokens(&self, tokens: &mut Tokens) { @@ -431,32 +417,9 @@ mod printing { if let Some(MetaItem::NameValue(ref pair)) = self.meta_item() { if pair.ident == "doc" { let value = pair.lit.value.to_string(); - match self.style { - AttrStyle::Inner(_) if value.starts_with("//!") => { - let doc = Literal::doccomment(&format!("{}\n", value)); - tokens.append(TokenTree { - span: pair.lit.span.0, - kind: TokenKind::Literal(doc), - }); - return; - } - AttrStyle::Inner(_) if value.starts_with("/*!") => { - pair.lit.to_tokens(tokens); - return; - } - AttrStyle::Outer if value.starts_with("///") => { - let doc = Literal::doccomment(&format!("{}\n", value)); - tokens.append(TokenTree { - span: pair.lit.span.0, - kind: TokenKind::Literal(doc), - }); - return; - } - AttrStyle::Outer if value.starts_with("/**") => { - pair.lit.to_tokens(tokens); - return; - } - _ => {} + if value.starts_with('/') { + pair.lit.to_tokens(tokens); + return } } } diff --git a/src/constant.rs b/src/constant.rs index 1a2818c7d4..d76a82761d 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -89,103 +89,103 @@ pub type Other = Expr; #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use {BinOp, Ty}; - use lit::parsing::lit; - use op::parsing::{binop, unop}; - use ty::parsing::{path, ty}; - - named!(pub const_expr -> ConstExpr, do_parse!( - mut e: alt!( - expr_unary - | - expr_lit - | - expr_path - | - expr_paren - // Cannot handle ConstExpr::Other here because for example - // `[u32; n!()]` would end up successfully parsing `n` as - // ConstExpr::Path and then fail to parse `!()`. Instead, callers - // are required to handle Other. See ty::parsing::array_len and - // data::parsing::discriminant. - ) >> - many0!(alt!( - tap!(args: and_call => { - let (args, paren) = args; - e = ConstCall { - func: Box::new(e), - args: args, - paren_token: paren, - }.into(); - }) - | - tap!(more: and_binary => { - let (op, other) = more; - e = ConstBinary { op: op, left: Box::new(e), right: Box::new(other) }.into(); - }) - | - tap!(ty: and_cast => { - let (ty, token) = ty; - e = ConstCast { - expr: Box::new(e), - ty: Box::new(ty), - as_token: token, - }.into(); - }) - | - tap!(i: and_index => { - let (i, bracket) = i; - e = ConstIndex { - expr: Box::new(e), - index: Box::new(i), - bracket_token: bracket, - }.into(); - }) - )) >> - (e) - )); - - named!(and_call -> (Delimited, tokens::Paren), do_parse!( - punct!("(") >> - args: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - const_expr) >> - punct!(")") >> - (args, tokens::Paren::default()) - )); - - named!(and_binary -> (BinOp, ConstExpr), tuple!(binop, const_expr)); - - named!(expr_unary -> ConstExpr, do_parse!( - operator: unop >> - operand: const_expr >> - (ConstUnary { op: operator, expr: Box::new(operand) }.into()) - )); + use proc_macro2::TokenTree; + use synom::{IResult, Synom}; + use synom::tokens::*; + + impl Synom for ConstExpr { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + mut e: alt!( + map!(syn!(ConstUnary), |e: ConstUnary| e.into()) + | + map!(syn!(Lit), |e: Lit| e.into()) + | + map!(syn!(Path), |e: Path| e.into()) + | + map!(syn!(ConstParen), |e: ConstParen| e.into()) + // Cannot handle ConstExpr::Other here because for example + // `[u32; n!()]` would end up successfully parsing `n` as + // ConstExpr::Path and then fail to parse `!()`. Instead, callers + // are required to handle Other. See ty::parsing::array_len and + // data::parsing::discriminant. + ) >> + many0!(alt!( + tap!(args: and_call => { + let (args, paren) = args; + e = ConstCall { + func: Box::new(e), + args: args, + paren_token: paren, + }.into(); + }) + | + tap!(more: and_binary => { + let (op, other) = more; + e = ConstBinary { op: op, left: Box::new(e), right: Box::new(other) }.into(); + }) + | + tap!(ty: and_cast => { + let (ty, token) = ty; + e = ConstCast { + expr: Box::new(e), + ty: Box::new(ty), + as_token: token, + }.into(); + }) + | + tap!(i: and_index => { + let (i, bracket) = i; + e = ConstIndex { + expr: Box::new(e), + index: Box::new(i), + bracket_token: bracket, + }.into(); + }) + )) >> + (e) + } + } + } - named!(expr_lit -> ConstExpr, map!(lit, ConstExpr::Lit)); + named!(and_call -> (Delimited, tokens::Paren), + parens!(call!(Delimited::parse_terminated))); - named!(expr_path -> ConstExpr, map!(path, ConstExpr::Path)); + named!(and_binary -> (BinOp, ConstExpr), + tuple!(call!(BinOp::parse_binop), syn!(ConstExpr))); - named!(and_index -> (ConstExpr, tokens::Bracket), do_parse!( - punct!("[") >> - expr: const_expr >> - punct!("]") >> - (expr, tokens::Bracket::default()) - )); + impl Synom for ConstUnary { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + operator: syn!(UnOp) >> + operand: syn!(ConstExpr) >> + (ConstUnary { op: operator, expr: Box::new(operand) }) + } + } + } - named!(expr_paren -> ConstExpr, do_parse!( - punct!("(") >> - e: const_expr >> - punct!(")") >> - (ConstParen { - expr: Box::new(e), - paren_token: tokens::Paren::default(), - }.into()) - )); + named!(and_index -> (ConstExpr, tokens::Bracket), + brackets!(syn!(ConstExpr))); + + impl Synom for ConstParen { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + parens: parens!(syn!(ConstExpr)) >> + (ConstParen { + expr: Box::new(parens.0), + paren_token: parens.1, + }) + } + } + } named!(and_cast -> (Ty, tokens::As), do_parse!( - keyword!("as") >> - ty: ty >> - (ty, tokens::As::default()) + as_tok: syn!(As) >> + ty: syn!(Ty) >> + (ty, as_tok) )); } diff --git a/src/data.rs b/src/data.rs index a1cf522dd0..290c8ff6a3 100644 --- a/src/data.rs +++ b/src/data.rs @@ -107,196 +107,109 @@ ast_enum_of_structs! { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use WhereClause; - #[cfg(feature = "full")] - use ConstExpr; - use attr::parsing::outer_attr; - #[cfg(feature = "full")] - use constant::parsing::const_expr; - #[cfg(feature = "full")] - use expr::parsing::expr; - use generics::parsing::where_clause; - use ident::parsing::ident; - use ty::parsing::{mod_style_path, ty}; - named!(pub struct_body -> (WhereClause, VariantData, Option), alt!( - do_parse!( - wh: where_clause >> - body: struct_like_body >> - (wh, VariantData::Struct(body.0, body.1), None) - ) - | - do_parse!( - body: tuple_like_body >> - wh: where_clause >> - punct!(";") >> - (wh, VariantData::Tuple(body.0, body.1), Some(tokens::Semi::default())) - ) - | - do_parse!( - wh: where_clause >> - punct!(";") >> - (wh, VariantData::Unit, Some(tokens::Semi::default())) - ) - )); - - named!(pub enum_body -> (WhereClause, Delimited, tokens::Brace), do_parse!( - wh: where_clause >> - punct!("{") >> - variants: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - variant) >> - punct!("}") >> - (wh, variants, tokens::Brace::default()) - )); - - named!(variant -> Variant, do_parse!( - attrs: many0!(outer_attr) >> - id: ident >> - data: alt!( - struct_like_body => { |(d, b)| VariantData::Struct(d, b) } - | - tuple_like_body => { |(d, b)| VariantData::Tuple(d, b) } - | - epsilon!() => { |_| VariantData::Unit } - ) >> - disr: option!(preceded!(punct!("="), discriminant)) >> - (Variant { - ident: id, - attrs: attrs, - data: data, - eq_token: disr.as_ref().map(|_| tokens::Eq::default()), - discriminant: disr, - }) - )); - - #[cfg(not(feature = "full"))] - use constant::parsing::const_expr as discriminant; - - #[cfg(feature = "full")] - named!(discriminant -> ConstExpr, alt!( - terminated!(const_expr, after_discriminant) - | - terminated!(expr, after_discriminant) => { ConstExpr::Other } - )); + use synom::{IResult, Synom}; + use synom::tokens; + use synom::tokens::*; + use proc_macro2::TokenTree; + + impl Field { + pub fn parse_struct(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + id: syn!(Ident) >> + colon: syn!(Colon) >> + ty: syn!(Ty) >> + (Field { + ident: Some(id), + vis: vis, + attrs: attrs, + ty: ty, + colon_token: Some(colon), + }) + } + } - // XXX: HACKY - #[cfg(feature = "full")] - pub fn eof(input: &[synom::TokenTree]) -> synom::IResult<&[synom::TokenTree], &'static str> { - if input.is_empty() { - synom::IResult::Done(&[], "") - } else { - synom::IResult::Error + pub fn parse_tuple(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + ty: syn!(Ty) >> + (Field { + ident: None, + colon_token: None, + vis: vis, + attrs: attrs, + ty: ty, + }) + } } } - #[cfg(feature = "full")] - named!(after_discriminant -> &str, peek!(alt!(punct!(",") | input_end!()))); - - named!(pub struct_like_body -> (Delimited, tokens::Brace), do_parse!( - punct!("{") >> - fields: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - struct_field) >> - punct!("}") >> - (fields, tokens::Brace::default()) - )); - - named!(tuple_like_body -> (Delimited, tokens::Paren), do_parse!( - punct!("(") >> - fields: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - tuple_field) >> - punct!(")") >> - (fields, tokens::Paren::default()) - )); - - named!(struct_field -> Field, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - id: ident >> - punct!(":") >> - ty: ty >> - (Field { - ident: Some(id), - vis: vis, - attrs: attrs, - ty: ty, - colon_token: Some(tokens::Colon::default()), - }) - )); - - named!(tuple_field -> Field, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - ty: ty >> - (Field { - ident: None, - colon_token: None, - vis: vis, - attrs: attrs, - ty: ty, - }) - )); - - named!(pub visibility -> Visibility, alt!( - do_parse!( - keyword!("pub") >> - punct!("(") >> - keyword!("crate") >> - punct!(")") >> - (Visibility::Crate(VisCrate { - crate_token: tokens::Crate::default(), - paren_token: tokens::Paren::default(), - pub_token: tokens::Pub::default(), - })) - ) - | - do_parse!( - keyword!("pub") >> - punct!("(") >> - keyword!("self") >> - punct!(")") >> - (Visibility::Restricted(VisRestricted { - path: Box::new("self".into()), - in_token: None, - paren_token: tokens::Paren::default(), - pub_token: tokens::Pub::default(), - })) - ) - | - do_parse!( - keyword!("pub") >> - punct!("(") >> - keyword!("super") >> - punct!(")") >> - (Visibility::Restricted(VisRestricted { - path: Box::new("super".into()), - in_token: None, - paren_token: tokens::Paren::default(), - pub_token: tokens::Pub::default(), - })) - ) - | - do_parse!( - keyword!("pub") >> - punct!("(") >> - keyword!("in") >> - restricted: mod_style_path >> - punct!(")") >> - (Visibility::Restricted(VisRestricted { - path: Box::new(restricted), - in_token: Some(tokens::In::default()), - paren_token: tokens::Paren::default(), - pub_token: tokens::Pub::default(), - })) - ) - | - keyword!("pub") => { |_| { - Visibility::Public(VisPublic { - pub_token: tokens::Pub::default(), - }) - } } - | - epsilon!() => { |_| Visibility::Inherited(VisInherited {}) } - )); + impl Synom for Visibility { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + pub_token: syn!(Pub) >> + other: parens!(syn!(tokens::Crate)) >> + (Visibility::Crate(VisCrate { + crate_token: other.0, + paren_token: other.1, + pub_token: pub_token, + })) + ) + | + do_parse!( + pub_token: syn!(Pub) >> + other: parens!(syn!(Self_)) >> + (Visibility::Restricted(VisRestricted { + path: Box::new(other.0.into()), + in_token: None, + paren_token: other.1, + pub_token: pub_token, + })) + ) + | + do_parse!( + pub_token: syn!(Pub) >> + other: parens!(syn!(Super)) >> + (Visibility::Restricted(VisRestricted { + path: Box::new(other.0.into()), + in_token: None, + paren_token: other.1, + pub_token: pub_token, + })) + ) + | + do_parse!( + pub_token: syn!(Pub) >> + other: parens!(do_parse!( + in_tok: syn!(In) >> + restricted: call!(Path::parse_mod_style) >> + (in_tok, restricted) + )) >> + (Visibility::Restricted(VisRestricted { + path: Box::new((other.0).1), + in_token: Some((other.0).0), + paren_token: other.1, + pub_token: pub_token, + })) + ) + | + syn!(Pub) => { |tok| { + Visibility::Public(VisPublic { + pub_token: tok, + }) + } } + | + epsilon!() => { |_| Visibility::Inherited(VisInherited {}) } + } + } + } } #[cfg(feature = "printing")] diff --git a/src/derive.rs b/src/derive.rs index cf44b4d749..7d6aabd0e2 100644 --- a/src/derive.rs +++ b/src/derive.rs @@ -46,51 +46,144 @@ ast_enum_of_structs! { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use Generics; - use attr::parsing::outer_attr; - use data::parsing::{visibility, struct_body, enum_body}; - use generics::parsing::generics; - use ident::parsing::ident; - - named!(pub derive_input -> DeriveInput, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - which: alt!(keyword!("struct") | keyword!("enum")) >> - id: ident >> - generics: generics >> - item: switch!(value!(which), - "struct" => map!(struct_body, move |(wh, body, semi)| DeriveInput { - ident: id, - vis: vis, - attrs: attrs, - generics: Generics { - where_clause: wh, - .. generics - }, - body: Body::Struct(BodyStruct { - struct_token: tokens::Struct::default(), - data: body, - semi_token: semi, - }), - }) - | - "enum" => map!(enum_body, move |(wh, body, brace)| DeriveInput { - ident: id, - vis: vis, - attrs: attrs, - generics: Generics { - where_clause: wh, - .. generics - }, - body: Body::Enum(BodyEnum { - variants: body, - brace_token: brace, - enum_token: tokens::Enum::default(), - }), - }) - ) >> - (item) + + use proc_macro2::TokenTree; + use synom::{IResult, Synom}; + use synom::tokens::*; + + impl Synom for DeriveInput { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + which: alt!( + syn!(Struct) => { Ok } + | + // weird hack to get around exhaustiveness check below + syn!(Enum) => { |e| Err((e, 1)) } + ) >> + id: syn!(Ident) >> + generics: syn!(Generics) >> + item: switch!(value!(which), + Ok(s) => map!(struct_body, move |(wh, body, semi)| DeriveInput { + ident: id, + vis: vis, + attrs: attrs, + generics: Generics { + where_clause: wh, + .. generics + }, + body: Body::Struct(BodyStruct { + struct_token: s, + data: body, + semi_token: semi, + }), + }) + | + Err((e, 1)) => map!(enum_body, move |(wh, body, brace)| DeriveInput { + ident: id, + vis: vis, + attrs: attrs, + generics: Generics { + where_clause: wh, + .. generics + }, + body: Body::Enum(BodyEnum { + variants: body, + brace_token: brace, + enum_token: e, + }), + }) + ) >> + (item) + } + } + + fn description() -> Option<&'static str> { + Some("derive input") + } + } + + + named!(struct_body -> (WhereClause, VariantData, Option), alt!( + do_parse!( + wh: syn!(WhereClause) >> + body: struct_like_body >> + (wh, VariantData::Struct(body.0, body.1), None) + ) + | + do_parse!( + body: tuple_like_body >> + wh: syn!(WhereClause) >> + semi: syn!(Semi) >> + (wh, VariantData::Tuple(body.0, body.1), Some(semi)) + ) + | + do_parse!( + wh: syn!(WhereClause) >> + semi: syn!(Semi) >> + (wh, VariantData::Unit, Some(semi)) + ) + )); + + named!(enum_body -> (WhereClause, Delimited, tokens::Brace), do_parse!( + wh: syn!(WhereClause) >> + data: braces!(Delimited::parse_terminated) >> + (wh, data.0, data.1) )); + + impl Synom for Variant { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + attrs: many0!(call!(Attribute::parse_outer)) >> + id: syn!(Ident) >> + data: alt!( + struct_like_body => { |(d, b)| VariantData::Struct(d, b) } + | + tuple_like_body => { |(d, b)| VariantData::Tuple(d, b) } + | + epsilon!() => { |_| VariantData::Unit } + ) >> + disr: option!(do_parse!( + eq: syn!(Eq) >> + disr: discriminant >> + (eq, disr) + )) >> + (Variant { + ident: id, + attrs: attrs, + data: data, + eq_token: disr.as_ref().map(|p| tokens::Eq((p.0).0)), + discriminant: disr.map(|p| p.1), + }) + } + } + } + + #[cfg(not(feature = "full"))] + named!(discriminant -> ConstExpr, syn!(ConstExpr)); + + #[cfg(feature = "full")] + named!(discriminant -> ConstExpr, alt!( + terminated!(syn!(ConstExpr), after_discriminant) + | + terminated!(syn!(Expr), after_discriminant) => { ConstExpr::Other } + )); + + #[cfg(feature = "full")] + named!(after_discriminant -> (), peek!(alt!( + syn!(Comma) => { |_| () } + | + input_end!() => { |_| () } + ))); + + named!(struct_like_body -> (Delimited, tokens::Brace), + braces!(call!(Delimited::parse_terminated_with, Field::parse_struct))); + + named!(tuple_like_body -> (Delimited, tokens::Paren), + parens!(call!(Delimited::parse_terminated_with, Field::parse_tuple))); } #[cfg(feature = "printing")] diff --git a/src/expr.rs b/src/expr.rs index 6ec7253eb4..277a2ab26a 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -26,7 +26,7 @@ ast_enum_of_structs! { /// A `box x` expression. pub Box(ExprBox { pub expr: Box, - pub box_token: tokens::Box, + pub box_token: tokens::Box_, }), /// E.g. 'place <- val'. @@ -485,7 +485,7 @@ ast_enum_of_structs! { /// A `box` pattern pub Box(PatBox { pub pat: Box, - pub box_token: tokens::Box, + pub box_token: tokens::Box_, }), /// A reference pattern, e.g. `&mut (a, b)` pub Ref(PatRef { @@ -587,20 +587,11 @@ ast_enum! { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use {BinOp, FnArg, FnDecl, FunctionRetTy, Ident, Lifetime, Mac, - TokenTree, Ty, UnOp, Unsafety, ArgCaptured, TyInfer}; - use attr::parsing::outer_attr; - use generics::parsing::lifetime; - use ident::parsing::{ident, wordlike}; - use item::parsing::item; - use lit::parsing::lit; - use mac::parsing::{mac, token_stream}; - use synom::IResult::{self, Error}; - use op::parsing::{assign_op, binop, unop}; - use ty::parsing::{mutability, path, qpath, ty, unsafety}; - use synom::{self, IResult}; - - use proc_macro2::{self, TokenKind, Delimiter}; + use ty::parsing::qpath; + + use proc_macro2::{TokenTree, TokenStream, TokenKind, Delimiter}; + use synom::{IResult, Synom}; + use synom::tokens::*; // Struct literals are ambiguous in certain positions // https://github.com/rust-lang/rfcs/pull/92 @@ -619,7 +610,16 @@ pub mod parsing { }; } - named!(pub expr -> Expr, ambiguous_expr!(true)); + impl Synom for Expr { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + ambiguous_expr!(input, true) + } + + fn description() -> Option<&'static str> { + Some("expression") + } + } + named!(expr_no_struct -> Expr, ambiguous_expr!(false)); @@ -628,56 +628,61 @@ pub mod parsing { allow_struct: bool, allow_block: bool) -> IResult<&[synom::TokenTree], Expr> { - do_parse!( + do_parse! { i, mut e: alt!( - expr_lit // must be before expr_struct + syn!(Lit) => { ExprKind::Lit } // must be before expr_struct | - cond_reduce!(allow_struct, expr_struct) // must be before expr_path + // must be before expr_path + cond_reduce!(allow_struct, map!(syn!(ExprStruct), ExprKind::Struct)) | - expr_paren // must be before expr_tup + syn!(ExprParen) => { ExprKind::Paren } // must be before expr_tup | - expr_mac // must be before expr_path + syn!(Mac) => { ExprKind::Mac } // must be before expr_path | call!(expr_break, allow_struct) // must be before expr_path | - expr_continue // must be before expr_path + syn!(ExprContinue) => { ExprKind::Continue } // must be before expr_path | call!(expr_ret, allow_struct) // must be before expr_path | call!(expr_box, allow_struct) | - expr_in_place + syn!(ExprInPlace) => { ExprKind::InPlace } | - expr_array + syn!(ExprArray) => { ExprKind::Array } | - expr_tup + syn!(ExprTup) => { ExprKind::Tup } | call!(expr_unary, allow_struct) | - expr_if + syn!(ExprIf) => { ExprKind::If } + | + syn!(ExprIfLet) => { ExprKind::IfLet } + | + syn!(ExprWhile) => { ExprKind::While } | - expr_while + syn!(ExprWhileLet) => { ExprKind::WhileLet } | - expr_for_loop + syn!(ExprForLoop) => { ExprKind::ForLoop } | - expr_loop + syn!(ExprLoop) => { ExprKind::Loop } | - expr_match + syn!(ExprMatch) => { ExprKind::Match } | - expr_catch + syn!(ExprCatch) => { ExprKind::Catch } | call!(expr_closure, allow_struct) | - cond_reduce!(allow_block, expr_block) + cond_reduce!(allow_block, map!(syn!(ExprBlock), ExprKind::Block)) | call!(expr_range, allow_struct) | - expr_path + syn!(ExprPath) => { ExprKind::Path } | call!(expr_addr_of, allow_struct) | - expr_repeat + syn!(ExprRepeat) => { ExprKind::Repeat } ) >> many0!(alt!( tap!(args: and_call => { @@ -776,287 +781,315 @@ pub mod parsing { }.into(); }) | - tap!(_try: punct!("?") => { + tap!(question: syn!(Question) => { e = ExprTry { expr: Box::new(e.into()), - question_token: tokens::Question::default(), + question_token: question, }.into(); }) )) >> (e.into()) - ) + } } - named!(expr_mac -> ExprKind, map!(mac, ExprKind::Mac)); - - named!(expr_paren -> ExprKind, do_parse!( - punct!("(") >> - e: expr >> - punct!(")") >> - (ExprParen { - expr: Box::new(e), - paren_token: tokens::Paren::default(), - }.into()) - )); + impl Synom for ExprParen { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + e: parens!(syn!(Expr)) >> + (ExprParen { + expr: Box::new(e.0), + paren_token: e.1, + }.into()) + } + } + } named_ambiguous_expr!(expr_box -> ExprKind, allow_struct, do_parse!( - keyword!("box") >> + box_: syn!(Box_) >> inner: ambiguous_expr!(allow_struct) >> (ExprBox { expr: Box::new(inner), - box_token: tokens::Box::default(), + box_token: box_, }.into()) )); - named!(expr_in_place -> ExprKind, do_parse!( - keyword!("in") >> - place: expr_no_struct >> - punct!("{") >> - value: within_block >> - punct!("}") >> - (ExprInPlace { - in_token: tokens::In::default(), - place: Box::new(place), - value: Box::new(Expr { - node: ExprBlock { - unsafety: Unsafety::Normal, - block: Block { - stmts: value, - brace_token: tokens::Brace::default(), - }, - }.into(), - attrs: Vec::new(), - }), - }.into()) - )); + impl Synom for ExprInPlace { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + in_: syn!(In) >> + place: expr_no_struct >> + value: braces!(call!(Block::parse_within)) >> + (ExprInPlace { + in_token: in_, + place: Box::new(place), + value: Box::new(Expr { + node: ExprBlock { + unsafety: Unsafety::Normal, + block: Block { + stmts: value.0, + brace_token: value.1, + }, + }.into(), + attrs: Vec::new(), + }), + }) + } + } + } - named!(expr_array -> ExprKind, do_parse!( - punct!("[") >> - elems: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - expr) >> - punct!("]") >> - (ExprArray { - exprs: elems, - bracket_token: tokens::Bracket::default(), - }.into()) - )); + impl Synom for ExprArray { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + elems: brackets!(call!(Delimited::parse_terminated)) >> + (ExprArray { + exprs: elems.0, + bracket_token: elems.1, + }) + } + } + } - named!(and_call -> (Delimited, tokens::Paren), do_parse!( - punct!("(") >> - args: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - expr) >> - punct!(")") >> - (args, tokens::Paren::default()) - )); + named!(and_call -> (Delimited, tokens::Paren), + parens!(call!(Delimited::parse_terminated))); named!(and_method_call -> ExprMethodCall, do_parse!( - punct!(".") >> - method: ident >> + dot: syn!(Dot) >> + method: syn!(Ident) >> typarams: option!(do_parse!( - punct!("::") >> - punct!("<") >> - tys: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - ty) >> - punct!(">") >> - (tys) + colon2: syn!(Colon2) >> + lt: syn!(Lt) >> + tys: call!(Delimited::parse_terminated) >> + gt: syn!(Gt) >> + (colon2, lt, tys, gt) )) >> - punct!("(") >> - args: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - expr) >> - punct!(")") >> - (ExprMethodCall { - // this expr will get overwritten after being returned - expr: Box::new(ExprKind::Lit(Lit { - span: Span::default(), - value: LitKind::Bool(false), - }).into()), - - method: method, - args: args, - paren_token: tokens::Paren::default(), - dot_token: tokens::Dot::default(), - lt_token: typarams.as_ref().map(|_| tokens::Lt::default()), - gt_token: typarams.as_ref().map(|_| tokens::Gt::default()), - colon2_token: typarams.as_ref().map(|_| tokens::Colon2::default()), - typarams: typarams.unwrap_or_default(), + args: parens!(call!(Delimited::parse_terminated)) >> + ({ + let (colon2, lt, tys, gt) = match typarams { + Some((a, b, c, d)) => (Some(a), Some(b), Some(c), Some(d)), + None => (None, None, None, None), + }; + ExprMethodCall { + // this expr will get overwritten after being returned + expr: Box::new(ExprKind::Lit(Lit { + span: Span::default(), + value: LitKind::Bool(false), + }).into()), + + method: method, + args: args.0, + paren_token: args.1, + dot_token: dot, + lt_token: lt, + gt_token: gt, + colon2_token: colon2, + typarams: tys.unwrap_or_default(), + } }) )); - named!(expr_tup -> ExprKind, do_parse!( - punct!("(") >> - elems: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - expr) >> - punct!(")") >> - (ExprTup { - args: elems, - paren_token: tokens::Paren::default(), - lone_comma: None, // TODO: parse this - }.into()) - )); + impl Synom for ExprTup { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + elems: parens!(call!(Delimited::parse_terminated)) >> + (ExprTup { + args: elems.0, + paren_token: elems.1, + lone_comma: None, // TODO: parse this + }) + } + } + } named_ambiguous_expr!(and_binary -> (BinOp, Expr), allow_struct, tuple!( - binop, + call!(BinOp::parse_binop), ambiguous_expr!(allow_struct) )); named_ambiguous_expr!(expr_unary -> ExprKind, allow_struct, do_parse!( - operator: unop >> + operator: syn!(UnOp) >> operand: ambiguous_expr!(allow_struct) >> (ExprUnary { op: operator, expr: Box::new(operand) }.into()) )); - named!(expr_lit -> ExprKind, map!(lit, ExprKind::Lit)); - - named!(and_cast -> (Ty, tokens::As), do_parse!( - keyword!("as") >> - ty: ty >> - (ty, tokens::As::default()) + named!(and_cast -> (Ty, As), do_parse!( + as_: syn!(As) >> + ty: syn!(Ty) >> + (ty, as_) )); - named!(and_ascription -> (Ty, tokens::Colon), - preceded!(punct!(":"), map!(ty, |t| (t, tokens::Colon::default())))); - - enum Cond { - Let(Pat, Expr, tokens::Eq, tokens::Let), - Expr(Expr), + named!(and_ascription -> (Ty, Colon), + map!(tuple!(syn!(Colon), syn!(Ty)), |(a, b)| (b, a))); + + impl Synom for ExprIfLet { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + if_: syn!(If) >> + let_: syn!(Let) >> + pat: syn!(Pat) >> + eq: syn!(Eq) >> + cond: expr_no_struct >> + then_block: braces!(call!(Block::parse_within)) >> + else_block: option!(else_block) >> + (ExprIfLet { + pat: Box::new(pat), + let_token: let_, + eq_token: eq, + expr: Box::new(cond), + if_true: Block { + stmts: then_block.0, + brace_token: then_block.1, + }, + if_token: if_, + else_token: else_block.as_ref().map(|p| Else((p.0).0)), + if_false: else_block.map(|p| Box::new(p.1.into())), + }) + } + } } - named!(cond -> Cond, alt!( - do_parse!( - keyword!("let") >> - pat: pat >> - punct!("=") >> - value: expr_no_struct >> - (Cond::Let(pat, value, tokens::Eq::default(), tokens::Let::default())) - ) - | - map!(expr_no_struct, Cond::Expr) - )); + impl Synom for ExprIf { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + if_: syn!(If) >> + cond: expr_no_struct >> + then_block: braces!(call!(Block::parse_within)) >> + else_block: option!(else_block) >> + (ExprIf { + cond: Box::new(cond), + if_true: Block { + stmts: then_block.0, + brace_token: then_block.1, + }, + if_token: if_, + else_token: else_block.as_ref().map(|p| Else((p.0).0)), + if_false: else_block.map(|p| Box::new(p.1.into())), + }) + } + } + } - named!(expr_if -> ExprKind, do_parse!( - keyword!("if") >> - cond: cond >> - then_block: delim!(Brace, within_block) >> - else_block: option!(preceded!( - keyword!("else"), - alt!( - expr_if - | - do_parse!( - punct!("{") >> - else_block: within_block >> - punct!("}") >> - (ExprKind::Block(ExprBlock { - unsafety: Unsafety::Normal, - block: Block { - stmts: else_block, - brace_token: tokens::Brace::default(), - }, - }).into()) - ) + named!(else_block -> (Else, ExprKind), do_parse!( + else_: syn!(Else) >> + expr: alt!( + syn!(ExprIf) => { ExprKind::If } + | + syn!(ExprIfLet) => { ExprKind::IfLet } + | + do_parse!( + else_block: braces!(call!(Block::parse_within)) >> + (ExprKind::Block(ExprBlock { + unsafety: Unsafety::Normal, + block: Block { + stmts: else_block.0, + brace_token: else_block.1, + }, + })) ) - )) >> - (match cond { - Cond::Let(pat, expr, eq_token, let_token) => ExprIfLet { - pat: Box::new(pat), - expr: Box::new(expr), - eq_token: eq_token, - let_token: let_token, - if_true: Block { - stmts: then_block, - brace_token: tokens::Brace::default(), - }, - if_token: tokens::If::default(), - else_token: else_block.as_ref().map(|_| tokens::Else::default()), - if_false: else_block.map(|els| Box::new(els.into())), - }.into(), - Cond::Expr(cond) => ExprIf { - cond: Box::new(cond), - if_true: Block { - stmts: then_block, - brace_token: tokens::Brace::default(), - }, - if_token: tokens::If::default(), - else_token: else_block.as_ref().map(|_| tokens::Else::default()), - if_false: else_block.map(|els| Box::new(els.into())), - }.into(), - }) + ) >> + (else_, expr) )); - named!(expr_for_loop -> ExprKind, do_parse!( - lbl: option!(terminated!(label, punct!(":"))) >> - keyword!("for") >> - pat: pat >> - keyword!("in") >> - expr: expr_no_struct >> - loop_block: block >> - (ExprForLoop { - for_token: tokens::For::default(), - in_token: tokens::In::default(), - colon_token: lbl.as_ref().map(|_| tokens::Colon::default()), - pat: Box::new(pat), - expr: Box::new(expr), - body: loop_block, - label: lbl, - }.into()) - )); - named!(expr_loop -> ExprKind, do_parse!( - lbl: option!(terminated!(label, punct!(":"))) >> - keyword!("loop") >> - loop_block: block >> - (ExprLoop { - loop_token: tokens::Loop::default(), - colon_token: lbl.as_ref().map(|_| tokens::Colon::default()), - body: loop_block, - label: lbl, - }.into()) - )); + impl Synom for ExprForLoop { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + lbl: option!(tuple!(label, syn!(Colon))) >> + for_: syn!(For) >> + pat: syn!(Pat) >> + in_: syn!(In) >> + expr: expr_no_struct >> + loop_block: syn!(Block) >> + (ExprForLoop { + for_token: for_, + in_token: in_, + pat: Box::new(pat), + expr: Box::new(expr), + body: loop_block, + colon_token: lbl.as_ref().map(|p| Colon((p.1).0)), + label: lbl.map(|p| p.0), + }) + } + } + } - named!(expr_match -> ExprKind, do_parse!( - keyword!("match") >> - obj: expr_no_struct >> - res: delim!(Brace, do_parse!( - mut arms: many0!(do_parse!( - arm: match_arm >> - cond!(arm_requires_comma(&arm), punct!(",")) >> - cond!(!arm_requires_comma(&arm), option!(punct!(","))) >> - (arm) - )) >> - last_arm: option!(match_arm) >> - (ExprKind::Match(Box::new(obj), { - arms.extend(last_arm); - arms - })) - )) >> - last_arm: option!(match_arm) >> - punct!("}") >> - (ExprMatch { - expr: Box::new(obj), - match_token: tokens::Match::default(), - brace_token: tokens::Brace::default(), - arms: { - for arm in &mut arms { - if arm_requires_comma(arm) { - arm.comma = Some(tokens::Comma::default()); + impl Synom for ExprLoop { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + lbl: option!(tuple!(label, syn!(Colon))) >> + loop_: syn!(Loop) >> + loop_block: syn!(Block) >> + (ExprLoop { + loop_token: loop_, + body: loop_block, + colon_token: lbl.as_ref().map(|p| Colon((p.1).0)), + label: lbl.map(|p| p.0), + }) + } + } + } + + impl Synom for ExprMatch { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + match_: syn!(Match) >> + obj: expr_no_struct >> + res: braces!(do_parse!( + mut arms: many0!(do_parse!( + arm: syn!(Arm) >> + cond!(arm_requires_comma(&arm), syn!(Comma)) >> + cond!(!arm_requires_comma(&arm), option!(syn!(Comma))) >> + (arm) + )) >> + last_arm: option!(syn!(Arm)) >> + ({ + arms.extend(last_arm); + arms + }) + )) >> + ({ + let (mut arms, brace) = res; + ExprMatch { + expr: Box::new(obj), + match_token: match_, + brace_token: brace, + arms: { + for arm in &mut arms { + if arm_requires_comma(arm) { + arm.comma = Some(tokens::Comma::default()); + } + } + arms + }, } - } - arms.extend(last_arm); - arms - }, - }.into()) - )); + }) + } + } + } - named!(expr_catch -> ExprKind, do_parse!( - keyword!("do") >> - keyword!("catch") >> - catch_block: block >> - (ExprCatch { - block: catch_block, - do_token: tokens::Do::default(), - catch_token: tokens::Catch::default(), - }.into()) - )); + impl Synom for ExprCatch { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + do_: syn!(Do) >> + catch_: syn!(Catch) >> + catch_block: syn!(Block) >> + (ExprCatch { + block: catch_block, + do_token: do_, + catch_token: catch_, + }.into()) + } + } + } fn arm_requires_comma(arm: &Arm) -> bool { if let ExprKind::Block(ExprBlock { unsafety: Unsafety::Normal, .. }) = arm.body.node { @@ -1066,45 +1099,48 @@ pub mod parsing { } } - named!(match_arm -> Arm, do_parse!( - attrs: many0!(outer_attr) >> - pats: separated_nonempty_list!(map!(punct!("|"), |_| tokens::Or::default()), - pat) >> - guard: option!(preceded!(keyword!("if"), expr)) >> - punct!("=>") >> - body: alt!( - map!(block, |blk| { - ExprKind::Block(ExprBlock { - unsafety: Unsafety::Normal, - block: blk, - }).into() - }) - | - expr - ) >> - (Arm { - rocket_token: tokens::Rocket::default(), - if_token: guard.as_ref().map(|_| tokens::If::default()), - attrs: attrs, - pats: pats, - guard: guard.map(Box::new), - body: Box::new(body), - comma: None, - }) - )); + impl Synom for Arm { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + attrs: many0!(call!(Attribute::parse_outer)) >> + pats: call!(Delimited::parse_separated_nonempty) >> + guard: option!(tuple!(syn!(If), syn!(Expr))) >> + rocket: syn!(Rocket) >> + body: alt!( + map!(syn!(Block), |blk| { + ExprKind::Block(ExprBlock { + unsafety: Unsafety::Normal, + block: blk, + }).into() + }) + | + syn!(Expr) + ) >> + (Arm { + rocket_token: rocket, + if_token: guard.as_ref().map(|p| If((p.0).0)), + attrs: attrs, + pats: pats, + guard: guard.map(|p| Box::new(p.1)), + body: Box::new(body), + comma: None, + }) + } + } + } named_ambiguous_expr!(expr_closure -> ExprKind, allow_struct, do_parse!( - capture: capture_by >> - punct!("|") >> - inputs: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - closure_arg) >> - punct!("|") >> + capture: syn!(CaptureBy) >> + or1: syn!(Or) >> + inputs: call!(Delimited::parse_terminated_with, fn_arg) >> + or2: syn!(Or) >> ret_and_body: alt!( do_parse!( - punct!("->") >> - ty: ty >> - body: block >> - (FunctionRetTy::Ty(ty, tokens::RArrow::default()), + arrow: syn!(RArrow) >> + ty: syn!(Ty) >> + body: syn!(Block) >> + (FunctionRetTy::Ty(ty, arrow), ExprKind::Block(ExprBlock { unsafety: Unsafety::Normal, block: body, @@ -1115,14 +1151,14 @@ pub mod parsing { ) >> (ExprClosure { capture: capture, - or1_token: tokens::Or::default(), - or2_token: tokens::Or::default(), + or1_token: or1, + or2_token: or2, decl: Box::new(FnDecl { inputs: inputs, output: ret_and_body.0, variadic: false, dot_tokens: None, - fn_token: tokens::Fn::default(), + fn_token: tokens::Fn_::default(), generics: Generics::default(), paren_token: tokens::Paren::default(), }), @@ -1130,274 +1166,356 @@ pub mod parsing { }.into()) )); - named!(closure_arg -> FnArg, do_parse!( - pat: pat >> - ty: option!(preceded!(punct!(":"), ty)) >> - (ArgCaptured { - pat: pat, - colon_token: tokens::Colon::default(), - ty: ty.unwrap_or_else(|| TyInfer { - underscore_token: tokens::Underscore::default(), - }.into()), - }.into()) - )); - - named!(expr_while -> ExprKind, do_parse!( - lbl: option!(terminated!(label, punct!(":"))) >> - keyword!("while") >> - cond: cond >> - while_block: block >> - (match cond { - Cond::Let(pat, expr, eq_token, let_token) => ExprWhileLet { - eq_token: eq_token, - let_token: let_token, - while_token: tokens::While::default(), - colon_token: lbl.as_ref().map(|_| tokens::Colon::default()), - pat: Box::new(pat), - expr: Box::new(expr), - body: while_block, - label: lbl, - }.into(), - Cond::Expr(cond) => ExprWhile { - while_token: tokens::While::default(), - colon_token: lbl.as_ref().map(|_| tokens::Colon::default()), - cond: Box::new(cond), - body: while_block, - label: lbl, - }.into(), + named!(fn_arg -> FnArg, do_parse!( + pat: syn!(Pat) >> + ty: option!(tuple!(syn!(Colon), syn!(Ty))) >> + ({ + let (colon, ty) = ty.unwrap_or_else(|| { + (Colon::default(), TyInfer { + underscore_token: Underscore::default(), + }.into()) + }); + ArgCaptured { + pat: pat, + colon_token: colon, + ty: ty, + }.into() }) )); - named!(expr_continue -> ExprKind, do_parse!( - keyword!("continue") >> - lbl: option!(label) >> - (ExprContinue { - continue_token: tokens::Continue::default(), - label: lbl, - }.into()) - )); + impl Synom for ExprWhile { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + lbl: option!(tuple!(label, syn!(Colon))) >> + while_: syn!(While) >> + cond: expr_no_struct >> + while_block: syn!(Block) >> + (ExprWhile { + while_token: while_, + colon_token: lbl.as_ref().map(|p| Colon((p.1).0)), + cond: Box::new(cond), + body: while_block, + label: lbl.map(|p| p.0), + }) + } + } + } + + impl Synom for ExprWhileLet { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + lbl: option!(tuple!(label, syn!(Colon))) >> + while_: syn!(While) >> + let_: syn!(Let) >> + pat: syn!(Pat) >> + eq: syn!(Eq) >> + value: expr_no_struct >> + while_block: syn!(Block) >> + (ExprWhileLet { + eq_token: eq, + let_token: let_, + while_token: while_, + colon_token: lbl.as_ref().map(|p| Colon((p.1).0)), + pat: Box::new(pat), + expr: Box::new(value), + body: while_block, + label: lbl.map(|p| p.0), + }) + } + } + } + + impl Synom for ExprContinue { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + cont: syn!(Continue) >> + lbl: option!(label) >> + (ExprContinue { + continue_token: cont, + label: lbl, + }) + } + } + } named_ambiguous_expr!(expr_break -> ExprKind, allow_struct, do_parse!( - keyword!("break") >> + break_: syn!(Break) >> lbl: option!(label) >> val: option!(call!(ambiguous_expr, allow_struct, false)) >> (ExprBreak { label: lbl, expr: val.map(Box::new), - break_token: tokens::Break::default(), + break_token: break_, }.into()) )); named_ambiguous_expr!(expr_ret -> ExprKind, allow_struct, do_parse!( - keyword!("return") >> + return_: syn!(Return) >> ret_value: option!(ambiguous_expr!(allow_struct)) >> (ExprRet { expr: ret_value.map(Box::new), - return_token: tokens::Return::default(), + return_token: return_, }.into()) )); - named!(expr_struct -> ExprKind, do_parse!( - path: path >> - punct!("{") >> - fields: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - field_value) >> - base: option!( - cond!(fields.is_empty() || fields.trailing_delim(), + impl Synom for ExprStruct { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + path: syn!(Path) >> + data: braces!(do_parse!( + fields: call!(Delimited::parse_terminated) >> + base: option!( + cond!(fields.is_empty() || fields.trailing_delim(), + do_parse!( + dots: syn!(Dot2) >> + base: syn!(Expr) >> + (dots, base) + ) + ) + ) >> + (fields, base) + )) >> + ({ + let ((fields, base), brace) = data; + let (dots, rest) = match base.and_then(|b| b) { + Some((dots, base)) => (Some(dots), Some(base)), + None => (None, None), + }; + ExprStruct { + brace_token: brace, + path: path, + fields: fields, + dot2_token: dots, + rest: rest.map(Box::new), + } + }) + } + } + } + + impl Synom for FieldValue { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, do_parse!( - punct!("..") >> - base: expr >> - (base) + name: wordlike >> + colon: syn!(Colon) >> + value: syn!(Expr) >> + (FieldValue { + ident: name, + expr: value, + is_shorthand: false, + attrs: Vec::new(), + colon_token: Some(colon), + }) ) - ) - ) >> - punct!("}") >> - (ExprStruct { - brace_token: tokens::Brace::default(), - path: path, - fields: fields, - dot2_token: base.as_ref().and_then(|b| b.as_ref()) - .map(|_| tokens::Dot2::default()), - rest: base.and_then(|b| b.map(Box::new)), - }.into()) - )); - - named!(field_value -> FieldValue, alt!( - do_parse!( - name: wordlike >> - punct!(":") >> - value: expr >> - (FieldValue { - ident: name, - expr: value, - is_shorthand: false, - attrs: Vec::new(), - colon_token: Some(tokens::Colon::default()), - }) - ) - | - map!(ident, |name: Ident| FieldValue { - ident: name.clone(), - expr: ExprKind::Path(ExprPath { qself: None, path: name.into() }).into(), - is_shorthand: true, - attrs: Vec::new(), - colon_token: None, - }) - )); + | + map!(syn!(Ident), |name: Ident| FieldValue { + ident: name.clone(), + expr: ExprKind::Path(ExprPath { qself: None, path: name.into() }).into(), + is_shorthand: true, + attrs: Vec::new(), + colon_token: None, + }) + } + } + } - named!(expr_repeat -> ExprKind, delim!(Bracket, do_parse!( - value: expr >> - punct!(";") >> - times: expr >> - punct!("]") >> - (ExprRepeat { - expr: Box::new(value), - amt: Box::new(times), - bracket_token: tokens::Bracket::default(), - semi_token: tokens::Semi::default(), - }.into()) - )); + impl Synom for ExprRepeat { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + data: brackets!(do_parse!( + value: syn!(Expr) >> + semi: syn!(Semi) >> + times: syn!(Expr) >> + (value, semi, times) + )) >> + (ExprRepeat { + expr: Box::new((data.0).0), + amt: Box::new((data.0).2), + bracket_token: data.1, + semi_token: (data.0).1, + }) + } + } + } - named!(expr_block -> ExprKind, do_parse!( - rules: unsafety >> - b: block >> - (ExprBlock { - unsafety: rules, - block: b, - }.into()) - )); + impl Synom for ExprBlock { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + rules: syn!(Unsafety) >> + b: syn!(Block) >> + (ExprBlock { + unsafety: rules, + block: b, + }) + } + } + } named_ambiguous_expr!(expr_range -> ExprKind, allow_struct, do_parse!( - limits: range_limits >> + limits: syn!(RangeLimits) >> hi: option!(ambiguous_expr!(allow_struct)) >> (ExprRange { from: None, to: hi.map(Box::new), limits: limits }.into()) )); - named!(range_limits -> RangeLimits, alt!( - punct!("...") => { |_| RangeLimits::Closed(tokens::Dot3::default()) } - | - punct!("..") => { |_| RangeLimits::HalfOpen(tokens::Dot2::default()) } - )); + impl Synom for RangeLimits { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(Dot3) => { RangeLimits::Closed } + | + syn!(Dot2) => { RangeLimits::HalfOpen } + } + } + } - named!(expr_path -> ExprKind, map!(qpath, |(qself, path)| { - ExprPath { qself: qself, path: path }.into() - })); + impl Synom for ExprPath { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + pair: qpath >> + (ExprPath { + qself: pair.0, + path: pair.1, + }) + } + } + } named_ambiguous_expr!(expr_addr_of -> ExprKind, allow_struct, do_parse!( - punct!("&") >> - mutability: mutability >> + and: syn!(And) >> + mutability: syn!(Mutability) >> expr: ambiguous_expr!(allow_struct) >> (ExprAddrOf { mutbl: mutability, expr: Box::new(expr), - and_token: tokens::And::default(), + and_token: and, }.into()) )); - named_ambiguous_expr!(and_assign -> (Expr, tokens::Eq), allow_struct, preceded!( - punct!("="), - map!(ambiguous_expr!(allow_struct), |t| (t, tokens::Eq::default())) - )); + named_ambiguous_expr!(and_assign -> (Expr, Eq), allow_struct, + map!( + tuple!(syn!(Eq), ambiguous_expr!(allow_struct)), + |(a, b)| (b, a) + ) + ); named_ambiguous_expr!(and_assign_op -> (BinOp, Expr), allow_struct, tuple!( - assign_op, + call!(BinOp::parse_assign_op), ambiguous_expr!(allow_struct) )); - named!(and_field -> (Ident, tokens::Dot), - preceded!(punct!("."), map!(ident, |t| (t, tokens::Dot::default())))); + named!(and_field -> (Ident, Dot), + map!(tuple!(syn!(Dot), syn!(Ident)), |(a, b)| (b, a))); - named!(and_tup_field -> (Lit, tokens::Dot), - preceded!(punct!("."), map!(lit, |l| (l, tokens::Dot::default())))); + named!(and_tup_field -> (Lit, Dot), + map!(tuple!(syn!(Dot), syn!(Lit)), |(a, b)| (b, a))); - named!(and_index -> (Expr, tokens::Bracket), - map!(delimited!(punct!("["), expr, punct!("]")), - |t| (t, tokens::Bracket::default()))); + named!(and_index -> (Expr, tokens::Bracket), brackets!(syn!(Expr))); named_ambiguous_expr!(and_range -> (RangeLimits, Option), allow_struct, tuple!( - range_limits, + syn!(RangeLimits), option!(call!(ambiguous_expr, allow_struct, false)) )); - named!(pub block -> Block, do_parse!( - stmts: delim!(Brace, within_block) >> - (Block { - stmts: stmts, - brace_token: tokens::Brace::default(), - }) - )); + impl Synom for Block { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + stmts: braces!(call!(Block::parse_within)) >> + (Block { + stmts: stmts.0, + brace_token: stmts.1, + }) + } + } + } - named!(pub within_block -> Vec, do_parse!( - many0!(punct!(";")) >> - mut standalone: many0!(terminated!(stmt, many0!(punct!(";")))) >> - last: option!(expr) >> - (match last { - None => standalone, - Some(last) => { - standalone.push(Stmt::Expr(Box::new(last))); - standalone + impl Block { + pub fn parse_within(input: &[TokenTree]) -> IResult<&[TokenTree], Vec> { + do_parse! { + input, + many0!(syn!(Semi)) >> + mut standalone: many0!(terminated!(syn!(Stmt), many0!(syn!(Semi)))) >> + last: option!(syn!(Expr)) >> + (match last { + None => standalone, + Some(last) => { + standalone.push(Stmt::Expr(Box::new(last))); + standalone + } + }) } - }) - )); + } + } - named!(pub stmt -> Stmt, alt!( - stmt_mac - | - stmt_local - | - stmt_item - | - stmt_expr - )); + impl Synom for Stmt { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + stmt_mac + | + stmt_local + | + stmt_item + | + stmt_expr + } + } + } named!(stmt_mac -> Stmt, do_parse!( - attrs: many0!(outer_attr) >> - what: path >> - punct!("!") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + what: syn!(Path) >> + bang: syn!(Bang) >> // Only parse braces here; paren and bracket will get parsed as // expression statements - punct!("{") >> - ts: token_stream >> - punct!("}") >> - semi: option!(punct!(";")) >> + data: braces!(syn!(TokenStream)) >> + semi: option!(syn!(Semi)) >> (Stmt::Mac(Box::new(( Mac { path: what, - bang_token: tokens::Bang::default(), + bang_token: bang, tokens: vec![TokenTree(proc_macro2::TokenTree { - span: Default::default(), - kind: TokenKind::Sequence(Delimiter::Brace, ts), + span: ((data.1).0).0, + kind: TokenKind::Sequence(Delimiter::Brace, data.0), })], }, - if semi.is_some() { - MacStmtStyle::Semicolon(tokens::Semi::default()) - } else { - MacStmtStyle::Braces + match semi { + Some(semi) => MacStmtStyle::Semicolon(semi), + None => MacStmtStyle::Braces, }, attrs, )))) )); named!(stmt_local -> Stmt, do_parse!( - attrs: many0!(outer_attr) >> - keyword!("let") >> - pat: pat >> - ty: option!(preceded!(punct!(":"), ty)) >> - init: option!(preceded!(punct!("="), expr)) >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + let_: syn!(Let) >> + pat: syn!(Pat) >> + ty: option!(tuple!(syn!(Colon), syn!(Ty))) >> + init: option!(tuple!(syn!(Eq), syn!(Expr))) >> + semi: syn!(Semi) >> (Stmt::Local(Box::new(Local { - let_token: tokens::Let::default(), - semi_token: tokens::Semi::default(), - colon_token: ty.as_ref().map(|_| tokens::Colon::default()), - eq_token: init.as_ref().map(|_| tokens::Eq::default()), + let_token: let_, + semi_token: semi, + colon_token: ty.as_ref().map(|p| Colon((p.0).0)), + eq_token: init.as_ref().map(|p| Eq((p.0).0)), pat: Box::new(pat), - ty: ty.map(Box::new), - init: init.map(Box::new), + ty: ty.map(|p| Box::new(p.1)), + init: init.map(|p| Box::new(p.1)), attrs: attrs, }))) )); - named!(stmt_item -> Stmt, map!(item, |i| Stmt::Item(Box::new(i)))); + named!(stmt_item -> Stmt, map!(syn!(Item), |i| Stmt::Item(Box::new(i)))); fn requires_semi(e: &Expr) -> bool { match e.node { @@ -1415,13 +1533,13 @@ pub mod parsing { } named!(stmt_expr -> Stmt, do_parse!( - attrs: many0!(outer_attr) >> - mut e: expr >> - semi: option!(punct!(";")) >> + attrs: many0!(call!(Attribute::parse_outer)) >> + mut e: syn!(Expr) >> + semi: option!(syn!(Semi)) >> ({ e.attrs = attrs; - if semi.is_some() { - Stmt::Semi(Box::new(e), tokens::Semi::default()) + if let Some(s) = semi { + Stmt::Semi(Box::new(e), s) } else if requires_semi(&e) { return IResult::Error; } else { @@ -1430,225 +1548,303 @@ pub mod parsing { }) )); - named!(pub pat -> Pat, alt!( - pat_wild // must be before pat_ident - | - pat_box // must be before pat_ident - | - pat_range // must be before pat_lit - | - pat_tuple_struct // must be before pat_ident - | - pat_struct // must be before pat_ident - | - pat_mac // must be before pat_ident - | - pat_lit // must be before pat_ident - | - pat_ident // must be before pat_path - | - pat_path - | - map!(pat_tuple, |t: PatTuple| t.into()) - | - pat_ref - | - pat_slice - )); + impl Synom for Pat { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(PatWild) => { Pat::Wild } // must be before pat_ident + | + syn!(PatBox) => { Pat::Box } // must be before pat_ident + | + syn!(PatRange) => { Pat::Range } // must be before pat_lit + | + syn!(PatTupleStruct) => { Pat::TupleStruct } // must be before pat_ident + | + syn!(PatStruct) => { Pat::Struct } // must be before pat_ident + | + syn!(Mac) => { Pat::Mac } // must be before pat_ident + | + syn!(PatLit) => { Pat::Lit } // must be before pat_ident + | + syn!(PatIdent) => { Pat::Ident } // must be before pat_path + | + syn!(PatPath) => { Pat::Path } + | + syn!(PatTuple) => { Pat::Tuple } + | + syn!(PatRef) => { Pat::Ref } + | + syn!(PatSlice) => { Pat::Slice } + } + } + } - named!(pat_mac -> Pat, map!(mac, Pat::Mac)); + impl Synom for PatWild { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + map! { + input, + syn!(Underscore), + |u| PatWild { underscore_token: u } + } + } + } - named!(pat_wild -> Pat, map!(keyword!("_"), |_| { - PatWild { underscore_token: tokens::Underscore::default() }.into() - })); + impl Synom for PatBox { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + boxed: syn!(Box_) >> + pat: syn!(Pat) >> + (PatBox { + pat: Box::new(pat), + box_token: boxed, + }) + } + } + } - named!(pat_box -> Pat, do_parse!( - keyword!("box") >> - pat: pat >> - (PatBox { - pat: Box::new(pat), - box_token: tokens::Box::default(), - }.into()) - )); + impl Synom for PatIdent { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + mode: option!(syn!(Ref)) >> + mutability: syn!(Mutability) >> + name: alt!( + syn!(Ident) + | + syn!(Self_) => { Into::into } + ) >> + not!(syn!(Lt)) >> + not!(syn!(Colon2)) >> + subpat: option!(tuple!(syn!(At), syn!(Pat))) >> + (PatIdent { + mode: match mode { + Some(mode) => BindingMode::ByRef(mode, mutability), + None => BindingMode::ByValue(mutability), + }, + ident: name, + at_token: subpat.as_ref().map(|p| At((p.0).0)), + subpat: subpat.map(|p| Box::new(p.1)), + }) + } + } + } - named!(pat_ident -> Pat, do_parse!( - mode: option!(keyword!("ref")) >> - mutability: mutability >> - name: alt!( - ident - | - keyword!("self") => { Into::into } - ) >> - not!(punct!("<")) >> - not!(punct!("::")) >> - subpat: option!(preceded!(punct!("@"), pat)) >> - (PatIdent { - mode: if mode.is_some() { - BindingMode::ByRef(tokens::Ref::default(), mutability) - } else { - BindingMode::ByValue(mutability) - }, - ident: name, - at_token: subpat.as_ref().map(|_| tokens::At::default()), - subpat: subpat.map(Box::new), - }.into()) - )); + impl Synom for PatTupleStruct { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + path: syn!(Path) >> + tuple: syn!(PatTuple) >> + (PatTupleStruct { + path: path, + pat: tuple, + }) + } + } + } - named!(pat_tuple_struct -> Pat, do_parse!( - path: path >> - tuple: pat_tuple >> - (PatTupleStruct { - path: path, - pat: tuple, - }.into()) - )); + impl Synom for PatStruct { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + path: syn!(Path) >> + data: braces!(do_parse!( + fields: call!(Delimited::parse_terminated) >> + base: option!( + cond!(fields.is_empty() || fields.trailing_delim(), + syn!(Dot2)) + ) >> + (fields, base) + )) >> + (PatStruct { + path: path, + fields: (data.0).0, + brace_token: data.1, + dot2_token: (data.0).1.and_then(|m| m), + }) + } + } + } - named!(pat_struct -> Pat, do_parse!( - path: path >> - punct!("{") >> - fields: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - field_pat) >> - base: option!( - cond!(fields.is_empty() || fields.trailing_delim(), - punct!("..")) - ) >> - punct!("}") >> - (PatStruct { - path: path, - fields: fields, - brace_token: tokens::Brace::default(), - dot2_token: base.and_then(|m| m).map(|_| tokens::Dot2::default()), - }.into()) - )); + impl Synom for FieldPat { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + ident: wordlike >> + colon: syn!(Colon) >> + pat: syn!(Pat) >> + (FieldPat { + ident: ident, + pat: Box::new(pat), + is_shorthand: false, + attrs: Vec::new(), + colon_token: Some(colon), + }) + ) + | + do_parse!( + boxed: option!(syn!(Box_)) >> + mode: option!(syn!(Ref)) >> + mutability: syn!(Mutability) >> + ident: syn!(Ident) >> + ({ + let mut pat: Pat = PatIdent { + mode: if let Some(mode) = mode { + BindingMode::ByRef(mode, mutability) + } else { + BindingMode::ByValue(mutability) + }, + ident: ident.clone(), + subpat: None, + at_token: None, + }.into(); + if let Some(boxed) = boxed { + pat = PatBox { + pat: Box::new(pat), + box_token: boxed, + }.into(); + } + FieldPat { + ident: ident, + pat: Box::new(pat), + is_shorthand: true, + attrs: Vec::new(), + colon_token: None, + } + }) + ) + } + } + } - named!(field_pat -> FieldPat, alt!( - do_parse!( - ident: wordlike >> - punct!(":") >> - pat: pat >> - (FieldPat { - ident: ident, - pat: Box::new(pat), - is_shorthand: false, - attrs: Vec::new(), - colon_token: Some(tokens::Colon::default()), - }) - ) + named!(wordlike -> Ident, alt!( + syn!(Ident) | do_parse!( - boxed: option!(keyword!("box")) >> - mode: option!(keyword!("ref")) >> - mutability: mutability >> - ident: ident >> + lit: syn!(Lit) >> ({ - let mut pat: Pat = PatIdent { - mode: if mode.is_some() { - BindingMode::ByRef(tokens::Ref::default(), mutability) - } else { - BindingMode::ByValue(mutability) - }, - ident: ident.clone(), - subpat: None, - at_token: None, - }.into(); - if boxed.is_some() { - pat = PatBox { - pat: Box::new(pat), - box_token: tokens::Box::default(), - }.into(); - } - FieldPat { - ident: ident, - pat: Box::new(pat), - is_shorthand: true, - attrs: Vec::new(), - colon_token: None, + let s = lit.value.to_string(); + if s.parse::().is_ok() { + Ident::new(s.into(), lit.span) + } else { + return IResult::Error } }) ) )); - named!(pat_path -> Pat, map!(qpath, |(qself, path)| { - PatPath { qself: qself, path: path }.into() - })); - - named!(pat_tuple -> PatTuple, do_parse!( - punct!("(") >> - mut elems: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - pat) >> - dotdot: map!(cond!( - elems.is_empty() || elems.trailing_delim(), - option!(do_parse!( - punct!("..") >> - trailing: option!(punct!(",")) >> - (trailing.is_some()) - )) - ), |x: Option<_>| x.and_then(|x| x)) >> - rest: cond!(dotdot == Some(true), - terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - pat)) >> - punct!(")") >> - (PatTuple { - paren_token: tokens::Paren::default(), - dots_pos: dotdot.map(|_| elems.len()), - dot2_token: dotdot.map(|_| tokens::Dot2::default()), - comma_token: dotdot.and_then(|b| { - if b { - Some(tokens::Comma::default()) - } else { - None - } - }), - pats: { - if let Some(rest) = rest { - for elem in rest.into_iter() { - elems.push(elem); + impl Synom for PatPath { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + map! { + input, + syn!(ExprPath), + |p: ExprPath| PatPath { qself: p.qself, path: p.path } + } + } + } + + impl Synom for PatTuple { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + data: parens!(do_parse!( + elems: call!(Delimited::parse_terminated) >> + dotdot: map!(cond!( + elems.is_empty() || elems.trailing_delim(), + option!(do_parse!( + dots: syn!(Dot2) >> + trailing: option!(syn!(Comma)) >> + (dots, trailing) + )) + ), |x: Option<_>| x.and_then(|x| x)) >> + rest: cond!(match dotdot { + Some((_, Some(_))) => true, + _ => false, + }, + call!(Delimited::parse_terminated)) >> + (elems, dotdot, rest) + )) >> + ({ + let ((mut elems, dotdot, rest), parens) = data; + let (dotdot, trailing) = match dotdot { + Some((a, b)) => (Some(a), Some(b)), + None => (None, None), + }; + PatTuple { + paren_token: parens, + dots_pos: dotdot.as_ref().map(|_| elems.len()), + dot2_token: dotdot, + comma_token: trailing.and_then(|b| b), + pats: { + if let Some(rest) = rest { + for elem in rest.into_iter() { + elems.push(elem); + } + } + elems + }, } - } - elems - }, - }) - ))); + }) + } + } + } - named!(pat_ref -> Pat, do_parse!( - punct!("&") >> - mutability: mutability >> - pat: pat >> - (PatRef { - pat: Box::new(pat), - mutbl: mutability, - and_token: tokens::And::default(), - }.into()) - )); + impl Synom for PatRef { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + and: syn!(And) >> + mutability: syn!(Mutability) >> + pat: syn!(Pat) >> + (PatRef { + pat: Box::new(pat), + mutbl: mutability, + and_token: and, + }) + } + } + } - named!(pat_lit -> Pat, do_parse!( - lit: pat_lit_expr >> - (if let ExprKind::Path(_) = lit.node { - return IResult::Error; // these need to be parsed by pat_path - } else { - PatLit { - expr: Box::new(lit), - }.into() - }) - )); + impl Synom for PatLit { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + lit: pat_lit_expr >> + (if let ExprKind::Path(_) = lit.node { + return IResult::Error; // these need to be parsed by pat_path + } else { + PatLit { + expr: Box::new(lit), + } + }) + } + } + } - named!(pat_range -> Pat, do_parse!( - lo: pat_lit_expr >> - limits: range_limits >> - hi: pat_lit_expr >> - (PatRange { - lo: Box::new(lo), - hi: Box::new(hi), - limits: limits, - }.into()) - )); + impl Synom for PatRange { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + lo: pat_lit_expr >> + limits: syn!(RangeLimits) >> + hi: pat_lit_expr >> + (PatRange { + lo: Box::new(lo), + hi: Box::new(hi), + limits: limits, + }) + } + } + } named!(pat_lit_expr -> Expr, do_parse!( - neg: option!(punct!("-")) >> + neg: option!(syn!(Sub)) >> v: alt!( - lit => { ExprKind::Lit } + syn!(Lit) => { ExprKind::Lit } | - path => { |p| ExprPath { qself: None, path: p }.into() } + syn!(ExprPath) => { ExprKind::Path } ) >> (if neg.is_some() { ExprKind::Unary(ExprUnary { @@ -1660,50 +1856,63 @@ pub mod parsing { }) )); - named!(pat_slice -> Pat, do_parse!( - punct!("[") >> - mut before: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - pat) >> - middle: option!(do_parse!( - punct!("..") >> - trailing: option!(punct!(",")) >> - (trailing.is_some()) - )) >> - after: cond!( - match middle { - Some(trailing) => trailing, - _ => false, - }, - terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - pat) - ) >> - punct!("]") >> - (PatSlice { - dot2_token: middle.as_ref().map(|_| tokens::Dot2::default()), - comma_token: { - let trailing = middle.unwrap_or(false); - if trailing {Some(tokens::Comma::default())} else {None} - }, - bracket_token: tokens::Bracket::default(), - middle: middle.and_then(|_| { - if !before.is_empty() && !before.trailing_delim() { - Some(Box::new(before.pop().unwrap().into_item())) - } else { - None + impl Synom for PatSlice { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + map! { + input, + brackets!(do_parse!( + before: call!(Delimited::parse_terminated) >> + middle: option!(do_parse!( + dots: syn!(Dot2) >> + trailing: option!(syn!(Comma)) >> + (dots, trailing) + )) >> + after: cond!( + match middle { + Some((_, ref trailing)) => trailing.is_some(), + _ => false, + }, + call!(Delimited::parse_terminated) + ) >> + (before, middle, after) + )), + |((before, middle, after), brackets)| { + let mut before: Delimited = before; + let after: Option> = after; + let middle: Option<(Dot2, Option)> = middle; + PatSlice { + dot2_token: middle.as_ref().map(|m| Dot2((m.0).0)), + comma_token: middle.as_ref().and_then(|m| { + m.1.as_ref().map(|m| Comma(m.0)) + }), + bracket_token: brackets, + middle: middle.and_then(|_| { + if !before.is_empty() && !before.trailing_delim() { + Some(Box::new(before.pop().unwrap().into_item())) + } else { + None + } + }), + front: before, + back: after.unwrap_or_default(), + } } - }), - front: before, - back: after.unwrap_or_default(), - }.into()) - )); + } + } + } - named!(capture_by -> CaptureBy, alt!( - keyword!("move") => { |_| CaptureBy::Value(tokens::Move::default()) } - | - epsilon!() => { |_| CaptureBy::Ref } - )); + impl Synom for CaptureBy { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(Move) => { CaptureBy::Value } + | + epsilon!() => { |_| CaptureBy::Ref } + } + } + } - named!(label -> Ident, map!(lifetime, |lt: Lifetime| lt.ident)); + named!(label -> Ident, map!(syn!(Lifetime), |lt: Lifetime| lt.ident)); } #[cfg(feature = "printing")] diff --git a/src/generics.rs b/src/generics.rs index fa29e6a97d..71cd8f6387 100644 --- a/src/generics.rs +++ b/src/generics.rs @@ -210,169 +210,206 @@ ast_enum_of_structs! { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use attr::parsing::outer_attr; - use ident::parsing::ident; - use ty::parsing::{ty, poly_trait_ref}; - use synom::{TokenTree, IResult}; - - named!(pub generics -> Generics, map!( - alt!( - do_parse!( - punct!("<") >> - lifetimes: terminated_list!( - map!(punct!(","), |_| tokens::Comma::default()), - lifetime_def - ) >> - ty_params: cond!( - lifetimes.is_empty() || lifetimes.trailing_delim(), - terminated_list!( - map!(punct!(","), |_| tokens::Comma::default()), - ty_param + + use synom::{IResult, Synom}; + use synom::tokens::*; + use proc_macro2::{TokenTree, TokenKind}; + + impl Synom for Generics { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + map! { + input, + alt!( + do_parse!( + lt: syn!(Lt) >> + lifetimes: call!(Delimited::parse_terminated) >> + ty_params: cond!( + lifetimes.is_empty() || lifetimes.trailing_delim(), + call!(Delimited::parse_terminated) + ) >> + gt: syn!(Gt) >> + (lifetimes, ty_params, Some(lt), Some(gt)) ) + | + epsilon!() => { |_| (Delimited::new(), None, None, None) } + ), + |(lifetimes, ty_params, lt, gt): (_, Option<_>, _, _)| Generics { + lifetimes: lifetimes, + ty_params: ty_params.unwrap_or_default(), + where_clause: WhereClause::default(), + gt_token: gt, + lt_token: lt, + } + } + } + } + + impl Synom for Lifetime { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + let mut tokens = input.iter(); + let token = match tokens.next() { + Some(token) => token, + None => return IResult::Error, + }; + if let TokenKind::Word(s) = token.kind { + if s.as_str().starts_with('\'') { + return IResult::Done(tokens.as_slice(), Lifetime { + ident: Ident { + span: Span(token.span), + sym: s, + }, + }) + } + } + IResult::Error + } + } + + impl Synom for LifetimeDef { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + attrs: many0!(call!(Attribute::parse_outer)) >> + life: syn!(Lifetime) >> + colon: option!(syn!(Colon)) >> + bounds: cond!( + colon.is_some(), + call!(Delimited::parse_separated_nonempty) ) >> - punct!(">") >> - (lifetimes, ty_params, true) - ) - | - epsilon!() => { |_| (Delimited::new(), None, false) } - ), - |(lifetimes, ty_params, any): (_, Option<_>, _)| Generics { - lifetimes: lifetimes, - ty_params: ty_params.unwrap_or_default(), - where_clause: WhereClause::default(), - gt_token: if any {Some(tokens::Gt::default())} else {None}, - lt_token: if any {Some(tokens::Lt::default())} else {None}, + (LifetimeDef { + attrs: attrs, + lifetime: life, + bounds: bounds.unwrap_or_default(), + colon_token: colon.map(|_| tokens::Colon::default()), + }) + } } - )); - - pub fn lifetime(input: &[TokenTree]) -> IResult<&[TokenTree], Lifetime> { - use synom::*; - if let Some(&TokenTree { kind: TokenKind::Word(ref id), .. }) = input.first() { - // Check if this word is _actually_ a lifetime, and treat that differently - if id.chars().next().unwrap() == '\'' { - IResult::Done(&input[1..], Lifetime { - ident: id.to_string().into() + } + + impl Synom for BoundLifetimes { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + for_: syn!(For) >> + lt: syn!(Lt) >> + lifetimes: call!(Delimited::parse_terminated) >> + gt: syn!(Gt) >> + (BoundLifetimes { + for_token: for_, + lt_token: lt, + gt_token: gt, + lifetimes: lifetimes, + }) + } + } + } + + impl Synom for TyParam { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + attrs: many0!(call!(Attribute::parse_outer)) >> + id: syn!(Ident) >> + colon: option!(syn!(Colon)) >> + bounds: cond!( + colon.is_some(), + call!(Delimited::parse_separated_nonempty) + ) >> + default: option!(do_parse!( + eq: syn!(Eq) >> + ty: syn!(Ty) >> + (eq, ty) + )) >> + (TyParam { + attrs: attrs, + ident: id, + bounds: bounds.unwrap_or_default(), + colon_token: colon, + eq_token: default.as_ref().map(|d| tokens::Eq((d.0).0)), + default: default.map(|d| d.1), }) - } else { - IResult::Error } - } else { - IResult::Error } } - named!(pub lifetime_def -> LifetimeDef, do_parse!( - attrs: many0!(outer_attr) >> - life: lifetime >> - colon: option!(punct!(":")) >> - bounds: cond!( - colon.is_some(), - separated_nonempty_list!(map!(punct!("+"), |_| tokens::Add::default()), - lifetime) - ) >> - (LifetimeDef { - attrs: attrs, - lifetime: life, - bounds: bounds.unwrap_or_default(), - colon_token: colon.map(|_| tokens::Colon::default()), - }) - )); - - named!(pub bound_lifetimes -> Option, option!(do_parse!( - keyword!("for") >> - punct!("<") >> - lifetimes: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - lifetime_def) >> - punct!(">") >> - (BoundLifetimes { - for_token: tokens::For::default(), - lt_token: tokens::Lt::default(), - gt_token: tokens::Gt::default(), - lifetimes: lifetimes, - }) - ))); - - named!(ty_param -> TyParam, do_parse!( - attrs: many0!(outer_attr) >> - id: ident >> - colon: option!(punct!(":")) >> - bounds: cond!( - colon.is_some(), - separated_nonempty_list!(map!(punct!("+"), |_| tokens::Add::default()), - ty_param_bound) - ) >> - default: option!(preceded!( - punct!("="), - ty - )) >> - (TyParam { - attrs: attrs, - ident: id, - bounds: bounds.unwrap_or_default(), - colon_token: colon.map(|_| tokens::Colon::default()), - eq_token: default.as_ref().map(|_| tokens::Eq::default()), - default: default, - }) - )); - - named!(pub ty_param_bound -> TyParamBound, alt!( - preceded!(punct!("?"), poly_trait_ref) => { - |poly| TyParamBound::Trait(poly, TraitBoundModifier::Maybe(tokens::Question::default())) + impl Synom for TyParamBound { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + question: syn!(Question) >> + poly: syn!(PolyTraitRef) >> + (TyParamBound::Trait(poly, TraitBoundModifier::Maybe(question))) + ) + | + syn!(Lifetime) => { TyParamBound::Region } + | + syn!(PolyTraitRef) => { + |poly| TyParamBound::Trait(poly, TraitBoundModifier::None) + } + } } - | - lifetime => { TyParamBound::Region } - | - poly_trait_ref => { - |poly| TyParamBound::Trait(poly, TraitBoundModifier::None) + + fn description() -> Option<&'static str> { + Some("type parameter buond") } - )); - - named!(pub where_clause -> WhereClause, alt!( - do_parse!( - keyword!("where") >> - predicates: terminated_list!( - map!(punct!(","), |_| tokens::Comma::default()), - where_predicate - ) >> - (WhereClause { - predicates: predicates, - where_token: Some(tokens::Where::default()), - }) - ) - | - epsilon!() => { |_| WhereClause::default() } - )); - - named!(where_predicate -> WherePredicate, alt!( - do_parse!( - ident: lifetime >> - colon: option!(punct!(":")) >> - bounds: cond!( - colon.is_some(), - separated_list!(map!(punct!("+"), |_| tokens::Add::default()), - lifetime) - ) >> - (WherePredicate::RegionPredicate(WhereRegionPredicate { - lifetime: ident, - bounds: bounds.unwrap_or_default(), - colon_token: colon.map(|_| tokens::Colon::default()), - })) - ) - | - do_parse!( - bound_lifetimes: bound_lifetimes >> - bounded_ty: ty >> - punct!(":") >> - bounds: separated_nonempty_list!(map!(punct!("+"), |_| tokens::Add::default()), - ty_param_bound) >> - (WherePredicate::BoundPredicate(WhereBoundPredicate { - bound_lifetimes: bound_lifetimes, - bounded_ty: bounded_ty, - bounds: bounds, - colon_token: tokens::Colon::default(), - })) - ) - )); + } + + impl Synom for WhereClause { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + where_: syn!(Where) >> + predicates: call!(Delimited::parse_terminated) >> + (WhereClause { + predicates: predicates, + where_token: Some(where_), + }) + ) + | + epsilon!() => { |_| WhereClause::default() } + } + } + + fn description() -> Option<&'static str> { + Some("where clause") + } + } + + impl Synom for WherePredicate { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + ident: syn!(Lifetime) >> + colon: option!(syn!(Colon)) >> + bounds: cond!( + colon.is_some(), + call!(Delimited::parse_separated) + ) >> + (WherePredicate::RegionPredicate(WhereRegionPredicate { + lifetime: ident, + bounds: bounds.unwrap_or_default(), + colon_token: colon, + })) + ) + | + do_parse!( + bound_lifetimes: option!(syn!(BoundLifetimes)) >> + bounded_ty: syn!(Ty) >> + colon: syn!(Colon) >> + bounds: call!(Delimited::parse_separated_nonempty) >> + (WherePredicate::BoundPredicate(WhereBoundPredicate { + bound_lifetimes: bound_lifetimes, + bounded_ty: bounded_ty, + bounds: bounds, + colon_token: colon, + })) + ) + } + } + } } #[cfg(feature = "printing")] diff --git a/src/ident.rs b/src/ident.rs index 1ed2c3e956..3ffacc71ca 100644 --- a/src/ident.rs +++ b/src/ident.rs @@ -6,6 +6,7 @@ use std::hash::{Hash, Hasher}; use proc_macro2::Symbol; use Span; +use tokens; #[derive(Clone)] pub struct Ident { @@ -28,6 +29,24 @@ impl<'a> From<&'a str> for Ident { } } +impl From for Ident { + fn from(tok: tokens::Self_) -> Self { + Ident::new("self".into(), tok.0) + } +} + +impl From for Ident { + fn from(tok: tokens::CapSelf) -> Self { + Ident::new("Self".into(), tok.0) + } +} + +impl From for Ident { + fn from(tok: tokens::Super) -> Self { + Ident::new("super".into(), tok.0) + } +} + impl<'a> From> for Ident { fn from(s: Cow<'a, str>) -> Self { Ident::new(s[..].into(), Span::default()) @@ -95,13 +114,24 @@ impl Hash for Ident { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use synom::{TokenTree, TokenKind, IResult}; - #[cfg(feature = "full")] - use lit::parsing::int; - - pub fn ident(input: &[TokenTree]) -> IResult<&[TokenTree], Ident> { - if let IResult::Done(rest, id) = word(input) { - match id.as_ref() { + use proc_macro2::{TokenTree, TokenKind}; + use synom::{Synom, IResult}; + + impl Synom for Ident { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + let mut tokens = input.iter(); + let token = match tokens.next() { + Some(token) => token, + None => return IResult::Error, + }; + let word = match token.kind { + TokenKind::Word(s) => s, + _ => return IResult::Error, + }; + if word.as_str().starts_with('\'') { + return IResult::Error + } + match word.as_str() { // From https://doc.rust-lang.org/grammar.html#keywords "abstract" | "alignof" | "as" | "become" | "box" | "break" | "const" | "continue" | "crate" | "do" | "else" | "enum" | "extern" | "false" | "final" | "fn" | "for" | @@ -109,33 +139,20 @@ pub mod parsing { "mut" | "offsetof" | "override" | "priv" | "proc" | "pub" | "pure" | "ref" | "return" | "Self" | "self" | "sizeof" | "static" | "struct" | "super" | "trait" | "true" | "type" | "typeof" | "unsafe" | "unsized" | "use" | "virtual" | "where" | - "while" | "yield" => IResult::Error, - _ => IResult::Done(rest, id), + "while" | "yield" => return IResult::Error, + _ => {} } - } else { - IResult::Error + + IResult::Done(tokens.as_slice(), Ident { + span: Span(token.span), + sym: word, + }) } - } - pub fn word(input: &[TokenTree]) -> IResult<&[TokenTree], Ident> { - if let Some(&TokenTree { kind: TokenKind::Word(ref id), .. }) = input.first() { - // Check if this word is _actually_ a lifetime, and treat that differently - if id.chars().next().unwrap() == '\'' { - IResult::Error - } else { - IResult::Done(&input[1..], Ident(id.to_string())) - } - } else { - IResult::Error + fn description() -> Option<&'static str> { + Some("identifier") } } - - #[cfg(feature = "full")] - named!(pub wordlike -> Ident, alt!( - word - | - int => { |d| format!("{}", d).into() } - )); } #[cfg(feature = "printing")] diff --git a/src/item.rs b/src/item.rs index 3f8446af7e..d6d1521142 100644 --- a/src/item.rs +++ b/src/item.rs @@ -243,7 +243,7 @@ ast_enum! { ast_enum! { #[cfg_attr(feature = "clone-impls", derive(Copy))] pub enum Defaultness { - Default(tokens::Default), + Default(tokens::Default_), Final, } } @@ -380,7 +380,7 @@ ast_struct! { /// /// E.g. `fn foo(bar: baz)` pub struct FnDecl { - pub fn_token: tokens::Fn, + pub fn_token: tokens::Fn_, pub paren_token: tokens::Paren, pub inputs: Delimited, pub output: FunctionRetTy, @@ -417,62 +417,64 @@ ast_enum_of_structs! { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use {Block, Generics, Ident, Mac, Path, VariantData}; - use attr::parsing::{inner_attr, outer_attr}; - use data::parsing::{struct_like_body, visibility}; - use expr::parsing::{expr, pat, within_block}; - use generics::parsing::{generics, lifetime, ty_param_bound, where_clause}; - use ident::parsing::ident; - use mac::parsing::delimited; - use derive::{Body, DeriveInput}; - use derive::parsing::derive_input; - use ty::parsing::{abi, mutability, path, ty, unsafety, fn_ret_ty}; - - named!(pub item -> Item, alt!( - item_extern_crate - | - item_use - | - item_static - | - item_const - | - item_fn - | - item_mod - | - item_foreign_mod - | - item_ty - | - item_struct_or_enum - | - item_union - | - item_trait - | - item_default_impl - | - item_impl - | - item_mac - )); - named!(pub items -> Vec, many0!(item)); + use proc_macro2::TokenTree; + use synom::{IResult, Synom}; + use synom::tokens::*; + use synom::tokens; + + impl Synom for Item { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + item_extern_crate + | + item_use + | + item_static + | + item_const + | + item_fn + | + item_mod + | + item_foreign_mod + | + item_ty + | + item_struct_or_enum + | + item_union + | + item_trait + | + item_default_impl + | + item_impl + | + item_mac + } + } + + fn description() -> Option<&'static str> { + Some("item") + } + } named!(item_mac -> Item, do_parse!( - attrs: many0!(outer_attr) >> - what: path >> - punct!("!") >> - name: option!(ident) >> - body: delimited >> - cond!(!body.is_braced(), punct!(";")) >> + attrs: many0!(call!(Attribute::parse_outer)) >> + what: syn!(Path) >> + bang: syn!(Bang) >> + name: option!(syn!(Ident)) >> + body: call!(::TokenTree::parse_delimited) >> + cond!(!body.is_braced(), syn!(Semi)) >> (Item { ident: name.unwrap_or_else(|| Ident::from("")), vis: VisInherited {}.into(), attrs: attrs, node: ItemKind::Mac(Mac { - bang_token: tokens::Bang::default(), + bang_token: bang, path: what, tokens: vec![body], }), @@ -480,144 +482,159 @@ pub mod parsing { )); named!(item_extern_crate -> Item, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("extern") >> - keyword!("crate") >> - id: ident >> - rename: option!(preceded!( - keyword!("as"), - ident - )) >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + extern_: syn!(Extern) >> + crate_: syn!(tokens::Crate) >> + id: syn!(Ident) >> + rename: option!(tuple!(syn!(As), syn!(Ident))) >> + semi: syn!(Semi) >> ({ - let (name, original_name) = match rename { - Some(rename) => (rename, Some(id)), - None => (id, None), + let (name, original_name, as_) = match rename { + Some((as_, rename)) => (rename, Some(id), Some(as_)), + None => (id, None, None), }; Item { ident: name, vis: vis, attrs: attrs, node: ItemExternCrate { - as_token: original_name.as_ref().map(|_| tokens::As::default()), + as_token: as_, original: original_name, - extern_token: tokens::Extern::default(), - crate_token: tokens::Crate::default(), - semi_token: tokens::Semi::default(), + extern_token: extern_, + crate_token: crate_, + semi_token: semi, }.into(), } }) )); named!(item_use -> Item, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("use") >> - what: view_path >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + use_: syn!(Use) >> + what: syn!(ViewPath) >> + semi: syn!(Semi) >> (Item { ident: "".into(), vis: vis, attrs: attrs, node: ItemUse { path: Box::new(what), - use_token: tokens::Use::default(), - semi_token: tokens::Semi::default(), + use_token: use_, + semi_token: semi, }.into(), }) )); - named!(view_path -> ViewPath, alt!( - view_path_glob - | - view_path_list - | - view_path_list_root - | - view_path_simple // must be last - )); - - - named!(view_path_simple -> ViewPath, do_parse!( - path: path >> - rename: option!(preceded!(keyword!("as"), ident)) >> - (PathSimple { - path: path, - as_token: rename.as_ref().map(|_| tokens::As::default()), - rename: rename, - }.into()) - )); + impl Synom for ViewPath { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(PathGlob) => { ViewPath::Glob } + | + syn!(PathList) => { ViewPath::List } + | + syn!(PathSimple) => { ViewPath::Simple } // must be last + } + } + } - named!(view_path_glob -> ViewPath, do_parse!( - path: path >> - punct!("::") >> - punct!("*") >> - (PathGlob { - path: path, - colon2_token: tokens::Colon2::default(), - star_token: tokens::Star::default(), - }.into()) - )); + impl Synom for PathSimple { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + path: syn!(Path) >> + rename: option!(tuple!(syn!(As), syn!(Ident))) >> + (PathSimple { + path: path, + as_token: rename.as_ref().map(|p| As((p.0).0)), + rename: rename.map(|p| p.1), + }) + } + } + } - named!(view_path_list -> ViewPath, do_parse!( - path: path >> - punct!("::") >> - punct!("{") >> - items: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - path_list_item) >> - punct!("}") >> - (PathList { - path: path, - items: items, - brace_token: tokens::Brace::default(), - colon2_token: tokens::Colon2::default(), - }.into()) - )); + impl Synom for PathGlob { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + path: syn!(Path) >> + colon2: syn!(Colon2) >> + star: syn!(Star) >> + (PathGlob { + path: path, + colon2_token: colon2, + star_token: star, + }) + } + } + } - named!(view_path_list_root -> ViewPath, do_parse!( - global: option!(punct!("::")) >> - punct!("{") >> - items: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - path_list_item) >> - punct!("}") >> - (PathList { - path: Path { - global: global.is_some(), - segments: Delimited::new(), - leading_colon: None, - }, - colon2_token: tokens::Colon2::default(), - brace_token: tokens::Brace::default(), - items: items, - }.into()) - )); + impl Synom for PathList { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + path: syn!(Path) >> + colon2: syn!(Colon2) >> + items: braces!(call!(Delimited::parse_terminated)) >> + (PathList { + path: path, + items: items.0, + brace_token: items.1, + colon2_token: colon2, + }) + ) + | + do_parse!( + global: option!(syn!(Colon2)) >> + items: braces!(call!(Delimited::parse_terminated)) >> + (PathList { + path: Path { + global: global.is_some(), + segments: Delimited::new(), + leading_colon: None, + }, + colon2_token: global.unwrap_or_default(), + brace_token: items.1, + items: items.0, + }) + ) + } + } + } - named!(path_list_item -> PathListItem, do_parse!( - name: alt!( - ident - | - map!(keyword!("self"), Into::into) - ) >> - rename: option!(preceded!(keyword!("as"), ident)) >> - (PathListItem { - name: name, - as_token: rename.as_ref().map(|_| tokens::As::default()), - rename: rename, - }) - )); + impl Synom for PathListItem { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + name: alt!( + syn!(Ident) + | + map!(syn!(Self_), Into::into) + ) >> + rename: option!(tuple!(syn!(As), syn!(Ident))) >> + (PathListItem { + name: name, + as_token: rename.as_ref().map(|p| As((p.0).0)), + rename: rename.map(|p| p.1), + }) + } + } + } named!(item_static -> Item, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("static") >> - mutability: mutability >> - id: ident >> - punct!(":") >> - ty: ty >> - punct!("=") >> - value: expr >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + static_: syn!(Static) >> + mutability: syn!(Mutability) >> + id: syn!(Ident) >> + colon: syn!(Colon) >> + ty: syn!(Ty) >> + eq: syn!(Eq) >> + value: syn!(Expr) >> + semi: syn!(Semi) >> (Item { ident: id, vis: vis, @@ -626,24 +643,24 @@ pub mod parsing { ty: Box::new(ty), mutbl: mutability, expr: Box::new(value), - static_token: tokens::Static::default(), - colon_token: tokens::Colon::default(), - eq_token: tokens::Eq::default(), - semi_token: tokens::Semi::default(), + static_token: static_, + colon_token: colon, + eq_token: eq, + semi_token: semi, }.into(), }) )); named!(item_const -> Item, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("const") >> - id: ident >> - punct!(":") >> - ty: ty >> - punct!("=") >> - value: expr >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + const_: syn!(Const) >> + id: syn!(Ident) >> + colon: syn!(Colon) >> + ty: syn!(Ty) >> + eq: syn!(Eq) >> + value: syn!(Expr) >> + semi: syn!(Semi) >> (Item { ident: id, vis: vis, @@ -651,46 +668,44 @@ pub mod parsing { node: ItemConst { ty: Box::new(ty), expr: Box::new(value), - const_token: tokens::Const::default(), - colon_token: tokens::Colon::default(), - eq_token: tokens::Eq::default(), - semi_token: tokens::Semi::default(), + const_token: const_, + colon_token: colon, + eq_token: eq, + semi_token: semi, }.into(), }) )); named!(item_fn -> Item, do_parse!( - outer_attrs: many0!(outer_attr) >> - vis: visibility >> - constness: constness >> - unsafety: unsafety >> - abi: option!(abi) >> - keyword!("fn") >> - name: ident >> - generics: generics >> - punct!("(") >> - inputs: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - fn_arg) >> - punct!(")") >> - ret: fn_ret_ty >> - where_clause: where_clause >> - inner_attrs_stmts: delim!(Brace, tuple!( - many0!(inner_attr), within_block + outer_attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + constness: syn!(Constness) >> + unsafety: syn!(Unsafety) >> + abi: option!(syn!(Abi)) >> + fn_: syn!(Fn_) >> + name: syn!(Ident) >> + generics: syn!(Generics) >> + inputs: parens!(Delimited::parse_terminated) >> + ret: syn!(FunctionRetTy) >> + where_clause: syn!(WhereClause) >> + inner_attrs_stmts: braces!(tuple!( + many0!(call!(Attribute::parse_inner)), + call!(Block::parse_within) )) >> (Item { ident: name, vis: vis, attrs: { let mut attrs = outer_attrs; - attrs.extend(inner_attrs_stmts.0); + attrs.extend((inner_attrs_stmts.0).0); attrs }, node: ItemFn { decl: Box::new(FnDecl { dot_tokens: None, - fn_token: tokens::Fn::default(), - paren_token: tokens::Paren::default(), - inputs: inputs, + fn_token: fn_, + paren_token: inputs.1, + inputs: inputs.0, output: ret, variadic: false, generics: Generics { @@ -702,70 +717,74 @@ pub mod parsing { constness: constness, abi: abi, block: Box::new(Block { - stmts: stmts, - brace_token: tokens::Brace::default(), + brace_token: inner_attrs_stmts.1, + stmts: (inner_attrs_stmts.0).1, }), }.into(), }) )); - named!(fn_arg -> FnArg, alt!( - do_parse!( - punct!("&") >> - lt: option!(lifetime) >> - mutability: mutability >> - keyword!("self") >> - not!(punct!(":")) >> - (ArgSelfRef { - lifetime: lt, - mutbl: mutability, - and_token: tokens::And::default(), - self_token: tokens::Self_::default(), - }.into()) - ) - | - do_parse!( - mutability: mutability >> - keyword!("self") >> - not!(punct!(":")) >> - (ArgSelf { - mutbl: mutability, - self_token: tokens::Self_::default(), - }.into()) - ) - | - do_parse!( - pat: pat >> - punct!(":") >> - ty: ty >> - (ArgCaptured { - pat: pat, - ty: ty, - colon_token: tokens::Colon::default(), - }.into()) - ) - | - ty => { FnArg::Ignored } - )); + impl Synom for FnArg { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + and: syn!(And) >> + lt: option!(syn!(Lifetime)) >> + mutability: syn!(Mutability) >> + self_: syn!(Self_) >> + not!(syn!(Colon)) >> + (ArgSelfRef { + lifetime: lt, + mutbl: mutability, + and_token: and, + self_token: self_, + }.into()) + ) + | + do_parse!( + mutability: syn!(Mutability) >> + self_: syn!(Self_) >> + not!(syn!(Colon)) >> + (ArgSelf { + mutbl: mutability, + self_token: self_, + }.into()) + ) + | + do_parse!( + pat: syn!(Pat) >> + colon: syn!(Colon) >> + ty: syn!(Ty) >> + (ArgCaptured { + pat: pat, + ty: ty, + colon_token: colon, + }.into()) + ) + | + syn!(Ty) => { FnArg::Ignored } + } + } + } named!(item_mod -> Item, do_parse!( - outer_attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("mod") >> - id: ident >> + outer_attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + mod_: syn!(Mod) >> + id: syn!(Ident) >> content: alt!( - punct!(";") => { |_| None } + syn!(Semi) => { Ok } | - delim!( - Brace, + braces!( tuple!( - many0!(inner_attr), - items + many0!(call!(Attribute::parse_inner)), + many0!(syn!(Item)) ) - ) => { Some } + ) => { Err } ) >> (match content { - Some((inner_attrs, items)) => Item { + Err(((inner_attrs, items), braces)) => Item { ident: id, vis: vis, attrs: { @@ -774,128 +793,133 @@ pub mod parsing { attrs }, node: ItemMod { - mod_token: tokens::Mod::default(), + mod_token: mod_, semi_token: None, - items: Some((items, tokens::Brace::default())), + items: Some((items, braces)), }.into(), }, - None => Item { + Ok(semi) => Item { ident: id, vis: vis, attrs: outer_attrs, node: ItemMod { items: None, - mod_token: tokens::Mod::default(), - semi_token: Some(tokens::Semi::default()), + mod_token: mod_, + semi_token: Some(semi), }.into(), }, }) )); named!(item_foreign_mod -> Item, do_parse!( - attrs: many0!(outer_attr) >> - abi: abi >> - items: delim!(Brace, many0!(foreign_item)) >> + attrs: many0!(call!(Attribute::parse_outer)) >> + abi: syn!(Abi) >> + items: braces!(many0!(syn!(ForeignItem))) >> (Item { ident: "".into(), vis: VisInherited {}.into(), attrs: attrs, node: ItemForeignMod { - brace_token: tokens::Brace::default(), + brace_token: items.1, abi: abi, - items: items, + items: items.0, }.into(), }) )); - named!(foreign_item -> ForeignItem, alt!( - foreign_fn - | - foreign_static - )); + impl Synom for ForeignItem { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + foreign_fn + | + foreign_static + } + } + } named!(foreign_fn -> ForeignItem, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("fn") >> - name: ident >> - generics: generics >> - punct!("(") >> - inputs: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - fn_arg) >> - variadic: cond!(inputs.is_empty() || inputs.trailing_delim(), - option!(punct!("..."))) >> - punct!(")") >> - ret: fn_ret_ty >> - where_clause: where_clause >> - punct!(";") >> - (ForeignItem { - ident: name, - attrs: attrs, - semi_token: tokens::Semi::default(), - node: ForeignItemFn { - decl: Box::new(FnDecl { - fn_token: tokens::Fn::default(), - paren_token: tokens::Paren::default(), - inputs: inputs, - variadic: variadic.map(|m| m.is_some()).unwrap_or(false), - dot_tokens: if variadic.map(|m| m.is_some()).unwrap_or(false) { - Some(tokens::Dot3::default()) - } else { - None - }, - output: ret, - generics: Generics { - where_clause: where_clause, - .. generics - }, - }), - }.into(), - vis: vis, + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + fn_: syn!(Fn_) >> + name: syn!(Ident) >> + generics: syn!(Generics) >> + inputs: parens!(do_parse!( + args: call!(Delimited::parse_terminated) >> + variadic: cond!(args.is_empty() || args.trailing_delim(), + option!(syn!(Dot3))) >> + (args, variadic) + )) >> + ret: syn!(FunctionRetTy) >> + where_clause: syn!(WhereClause) >> + semi: syn!(Semi) >> + ({ + let ((inputs, variadic), parens) = inputs; + let variadic = variadic.and_then(|v| v); + ForeignItem { + ident: name, + attrs: attrs, + semi_token: semi, + node: ForeignItemFn { + decl: Box::new(FnDecl { + fn_token: fn_, + paren_token: parens, + inputs: inputs, + variadic: variadic.is_some(), + dot_tokens: variadic, + output: ret, + generics: Generics { + where_clause: where_clause, + .. generics + }, + }), + }.into(), + vis: vis, + } }) )); named!(foreign_static -> ForeignItem, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("static") >> - mutability: mutability >> - id: ident >> - punct!(":") >> - ty: ty >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + static_: syn!(Static) >> + mutability: syn!(Mutability) >> + id: syn!(Ident) >> + colon: syn!(Colon) >> + ty: syn!(Ty) >> + semi: syn!(Semi) >> (ForeignItem { ident: id, attrs: attrs, - semi_token: tokens::Semi::default(), + semi_token: semi, node: ForeignItemStatic { ty: Box::new(ty), mutbl: mutability, - static_token: tokens::Static::default(), - colon_token: tokens::Colon::default(), + static_token: static_, + colon_token: colon, }.into(), vis: vis, }) )); named!(item_ty -> Item, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("type") >> - id: ident >> - generics: generics >> - where_clause: where_clause >> - punct!("=") >> - ty: ty >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + type_: syn!(Type) >> + id: syn!(Ident) >> + generics: syn!(Generics) >> + where_clause: syn!(WhereClause) >> + eq: syn!(Eq) >> + ty: syn!(Ty) >> + semi: syn!(Semi) >> (Item { ident: id, vis: vis, attrs: attrs, node: ItemTy { - type_token: tokens::Type::default(), - eq_token: tokens::Eq::default(), - semi_token: tokens::Semi::default(), + type_token: type_, + eq_token: eq, + semi_token: semi, ty: Box::new(ty), generics: Generics { where_clause: where_clause, @@ -906,7 +930,7 @@ pub mod parsing { )); named!(item_struct_or_enum -> Item, map!( - derive_input, + syn!(DeriveInput), |def: DeriveInput| Item { ident: def.ident, vis: def.vis, @@ -933,19 +957,20 @@ pub mod parsing { )); named!(item_union -> Item, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - keyword!("union") >> - id: ident >> - generics: generics >> - where_clause: where_clause >> - fields: struct_like_body >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + union_: syn!(Union) >> + id: syn!(Ident) >> + generics: syn!(Generics) >> + where_clause: syn!(WhereClause) >> + fields: braces!(call!(Delimited::parse_terminated_with, + Field::parse_struct)) >> (Item { ident: id, vis: vis, attrs: attrs, node: ItemUnion { - union_token: tokens::Union::default(), + union_token: union_, data: VariantData::Struct(fields.0, fields.1), generics: Generics { where_clause: where_clause, @@ -956,46 +981,45 @@ pub mod parsing { )); named!(item_trait -> Item, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - unsafety: unsafety >> - keyword!("trait") >> - id: ident >> - generics: generics >> - colon: option!(punct!(":")) >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + unsafety: syn!(Unsafety) >> + trait_: syn!(Trait) >> + id: syn!(Ident) >> + generics: syn!(Generics) >> + colon: option!(syn!(Colon)) >> bounds: cond!(colon.is_some(), - separated_nonempty_list!(map!(punct!("+"), |_| tokens::Add::default()), - ty_param_bound) + call!(Delimited::parse_separated_nonempty) ) >> - where_clause: where_clause >> - body: delim!(Brace, many0!(trait_item)) >> + where_clause: syn!(WhereClause) >> + body: braces!(many0!(syn!(TraitItem))) >> (Item { ident: id, vis: vis, attrs: attrs, node: ItemTrait { - trait_token: tokens::Trait::default(), - brace_token: tokens::Brace::default(), - colon_token: colon.map(|_| tokens::Colon::default()), + trait_token: trait_, + brace_token: body.1, + colon_token: colon, unsafety: unsafety, generics: Generics { where_clause: where_clause, .. generics }, supertraits: bounds.unwrap_or_default(), - items: body, + items: body.0, }.into(), }) )); named!(item_default_impl -> Item, do_parse!( - attrs: many0!(outer_attr) >> - unsafety: unsafety >> - keyword!("impl") >> - path: path >> - keyword!("for") >> - punct!("..") >> - delim!(Brace, epsilon!()) >> + attrs: many0!(call!(Attribute::parse_outer)) >> + unsafety: syn!(Unsafety) >> + impl_: syn!(Impl) >> + path: syn!(Path) >> + for_: syn!(For) >> + dot2: syn!(Dot2) >> + braces: braces!(epsilon!()) >> (Item { ident: "".into(), vis: VisInherited {}.into(), @@ -1003,67 +1027,70 @@ pub mod parsing { node: ItemDefaultImpl { unsafety: unsafety, path: path, - impl_token: tokens::Impl::default(), - for_token: tokens::For::default(), - dot2_token: tokens::Dot2::default(), - brace_token: tokens::Brace::default(), + impl_token: impl_, + for_token: for_, + dot2_token: dot2, + brace_token: braces.1, }.into(), }) )); - named!(trait_item -> TraitItem, alt!( - trait_item_const - | - trait_item_method - | - trait_item_type - | - trait_item_mac - )); + impl Synom for TraitItem { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + trait_item_const + | + trait_item_method + | + trait_item_type + | + trait_item_mac + } + } + } named!(trait_item_const -> TraitItem, do_parse!( - attrs: many0!(outer_attr) >> - keyword!("const") >> - id: ident >> - punct!(":") >> - ty: ty >> - value: option!(preceded!(punct!("="), expr)) >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + const_: syn!(Const) >> + id: syn!(Ident) >> + colon: syn!(Colon) >> + ty: syn!(Ty) >> + value: option!(tuple!(syn!(Eq), syn!(Expr))) >> + semi: syn!(Semi) >> (TraitItem { ident: id, attrs: attrs, node: TraitItemConst { ty: ty, - const_token: tokens::Const::default(), - colon_token: tokens::Colon::default(), - eq_token: value.as_ref().map(|_| tokens::Eq::default()), - default: value, - semi_token: tokens::Semi::default(), + const_token: const_, + colon_token: colon, + eq_token: value.as_ref().map(|p| Eq((p.0).0)), + default: value.map(|p| p.1), + semi_token: semi, }.into(), }) )); named!(trait_item_method -> TraitItem, do_parse!( - outer_attrs: many0!(outer_attr) >> - constness: constness >> - unsafety: unsafety >> - abi: option!(abi) >> - keyword!("fn") >> - name: ident >> - generics: generics >> - punct!("(") >> - inputs: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), fn_arg) >> - punct!(")") >> - ret: fn_ret_ty >> - where_clause: where_clause >> - body: option!(delim!( - Brace, - tuple!(many0!(inner_attr), within_block) + outer_attrs: many0!(call!(Attribute::parse_outer)) >> + constness: syn!(Constness) >> + unsafety: syn!(Unsafety) >> + abi: option!(syn!(Abi)) >> + fn_: syn!(Fn_) >> + name: syn!(Ident) >> + generics: syn!(Generics) >> + inputs: parens!(call!(Delimited::parse_terminated)) >> + ret: syn!(FunctionRetTy) >> + where_clause: syn!(WhereClause) >> + body: option!(braces!( + tuple!(many0!(call!(Attribute::parse_inner)), + call!(Block::parse_within)) )) >> - semi: cond!(body.is_none(), punct!(";")) >> + semi: cond!(body.is_none(), syn!(Semi)) >> ({ let (inner_attrs, stmts) = match body { - Some((inner_attrs, stmts)) => (inner_attrs, Some(stmts)), + Some(((inner_attrs, stmts), b)) => (inner_attrs, Some((stmts, b))), None => (Vec::new(), None), }; TraitItem { @@ -1074,17 +1101,17 @@ pub mod parsing { attrs }, node: TraitItemMethod { - semi_token: semi.map(|_| tokens::Semi::default()), + semi_token: semi, sig: MethodSig { unsafety: unsafety, constness: constness, abi: abi, decl: FnDecl { - inputs: inputs, + inputs: inputs.0, output: ret, variadic: false, - fn_token: tokens::Fn::default(), - paren_token: tokens::Paren::default(), + fn_token: fn_, + paren_token: inputs.1, dot_tokens: None, generics: Generics { where_clause: where_clause, @@ -1094,8 +1121,8 @@ pub mod parsing { }, default: stmts.map(|stmts| { Block { - stmts: stmts, - brace_token: tokens::Brace::default(), + stmts: stmts.0, + brace_token: stmts.1, } }), }.into(), @@ -1104,73 +1131,66 @@ pub mod parsing { )); named!(trait_item_type -> TraitItem, do_parse!( - attrs: many0!(outer_attr) >> - keyword!("type") >> - id: ident >> - colon: option!(punct!(":")) >> + attrs: many0!(call!(Attribute::parse_outer)) >> + type_: syn!(Type) >> + id: syn!(Ident) >> + colon: option!(syn!(Colon)) >> bounds: cond!(colon.is_some(), - separated_nonempty_list!(map!(punct!("+"), |_| tokens::Add::default()), - ty_param_bound) + call!(Delimited::parse_separated_nonempty) ) >> - default: option!(preceded!(punct!("="), ty)) >> - punct!(";") >> + default: option!(tuple!(syn!(Eq), syn!(Ty))) >> + semi: syn!(Semi) >> (TraitItem { ident: id, attrs: attrs, node: TraitItemType { - type_token: tokens::Type::default(), - colon_token: colon.map(|_| tokens::Colon::default()), + type_token: type_, + colon_token: colon, + eq_token: default.as_ref().map(|p| Eq((p.0).0)), bounds: bounds.unwrap_or_default(), - eq_token: default.as_ref().map(|_| tokens::Eq::default()), - semi_token: tokens::Semi::default(), - default: default, + semi_token: semi, + default: default.map(|p| p.1), }.into(), }) )); named!(trait_item_mac -> TraitItem, do_parse!( - attrs: many0!(outer_attr) >> - what: path >> - punct!("!") >> - body: delimited >> - cond!(!body.is_braced(), punct!(";")) >> + attrs: many0!(call!(Attribute::parse_outer)) >> + mac: syn!(Mac) >> + cond!(!mac.is_braced(), syn!(Semi)) >> (TraitItem { ident: "".into(), attrs: attrs, - node: TraitItemKind::Macro(Mac { - path: what, - bang_token: tokens::Bang::default(), - tokens: vec![body], - }), + node: TraitItemKind::Macro(mac), }) )); named!(item_impl -> Item, do_parse!( - attrs: many0!(outer_attr) >> - unsafety: unsafety >> - keyword!("impl") >> - generics: generics >> + attrs: many0!(call!(Attribute::parse_outer)) >> + unsafety: syn!(Unsafety) >> + impl_: syn!(Impl) >> + generics: syn!(Generics) >> polarity_path: alt!( do_parse!( - polarity: impl_polarity >> - path: path >> - keyword!("for") >> - (polarity, Some(path)) + polarity: syn!(ImplPolarity) >> + path: syn!(Path) >> + for_: syn!(For) >> + (polarity, Some(path), Some(for_)) ) | - epsilon!() => { |_| (ImplPolarity::Positive, None) } + epsilon!() => { |_| (ImplPolarity::Positive, None, None) } ) >> - self_ty: ty >> - where_clause: where_clause >> - body: delim!(Brace, many0!(impl_item)) >> + self_ty: syn!(Ty) >> + where_clause: syn!(WhereClause) >> + body: braces!(many0!(syn!(ImplItem))) >> (Item { ident: "".into(), vis: VisInherited {}.into(), attrs: attrs, node: ItemImpl { - impl_token: tokens::Impl::default(), - brace_token: tokens::Brace::default(), - for_token: polarity_path.1.as_ref().map(|_| tokens::For::default()), + impl_token: impl_, + brace_token: body.1, + for_token: polarity_path.2, unsafety: unsafety, polarity: polarity_path.0, generics: Generics { @@ -1179,32 +1199,37 @@ pub mod parsing { }, trait_: polarity_path.1, self_ty: Box::new(self_ty), - items: body, + items: body.0, }.into(), }) )); - named!(impl_item -> ImplItem, alt!( - impl_item_const - | - impl_item_method - | - impl_item_type - | - impl_item_macro - )); + impl Synom for ImplItem { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + impl_item_const + | + impl_item_method + | + impl_item_type + | + impl_item_macro + } + } + } named!(impl_item_const -> ImplItem, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - defaultness: defaultness >> - keyword!("const") >> - id: ident >> - punct!(":") >> - ty: ty >> - punct!("=") >> - value: expr >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + defaultness: syn!(Defaultness) >> + const_: syn!(Const) >> + id: syn!(Ident) >> + colon: syn!(Colon) >> + ty: syn!(Ty) >> + eq: syn!(Eq) >> + value: syn!(Expr) >> + semi: syn!(Semi) >> (ImplItem { ident: id, vis: vis, @@ -1213,32 +1238,30 @@ pub mod parsing { node: ImplItemConst { ty: ty, expr: value, - const_token: tokens::Const::default(), - colon_token: tokens::Colon::default(), - eq_token: tokens::Eq::default(), - semi_token: tokens::Semi::default(), + const_token: const_, + colon_token: colon, + eq_token: eq, + semi_token: semi, }.into(), }) )); named!(impl_item_method -> ImplItem, do_parse!( - outer_attrs: many0!(outer_attr) >> - vis: visibility >> - defaultness: defaultness >> - constness: constness >> - unsafety: unsafety >> - abi: option!(abi) >> - keyword!("fn") >> - name: ident >> - generics: generics >> - punct!("(") >> - inputs: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - fn_arg) >> - punct!(")") >> - ret: fn_ret_ty >> - where_clause: where_clause >> - inner_attrs_stmts: delim!(Brace, tuple!( - many0!(inner_attr), within_block + outer_attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + defaultness: syn!(Defaultness) >> + constness: syn!(Constness) >> + unsafety: syn!(Unsafety) >> + abi: option!(syn!(Abi)) >> + fn_: syn!(Fn_) >> + name: syn!(Ident) >> + generics: syn!(Generics) >> + inputs: parens!(call!(Delimited::parse_terminated)) >> + ret: syn!(FunctionRetTy) >> + where_clause: syn!(WhereClause) >> + inner_attrs_stmts: braces!(tuple!( + many0!(call!(Attribute::parse_inner)), + call!(Block::parse_within) )) >> (ImplItem { ident: name, @@ -1246,7 +1269,7 @@ pub mod parsing { defaultness: defaultness, attrs: { let mut attrs = outer_attrs; - attrs.extend(inner_attrs_stmts.0); + attrs.extend((inner_attrs_stmts.0).0); attrs }, node: ImplItemMethod { @@ -1255,9 +1278,9 @@ pub mod parsing { constness: constness, abi: abi, decl: FnDecl { - fn_token: tokens::Fn::default(), - paren_token: tokens::Paren::default(), - inputs: inputs, + fn_token: fn_, + paren_token: inputs.1, + inputs: inputs.0, output: ret, variadic: false, generics: Generics { @@ -1268,72 +1291,81 @@ pub mod parsing { }, }, block: Block { - brace_token: tokens::Brace::default(), - stmts: stmts, + brace_token: inner_attrs_stmts.1, + stmts: (inner_attrs_stmts.0).1, }, }.into(), }) )); named!(impl_item_type -> ImplItem, do_parse!( - attrs: many0!(outer_attr) >> - vis: visibility >> - defaultness: defaultness >> - keyword!("type") >> - id: ident >> - punct!("=") >> - ty: ty >> - punct!(";") >> + attrs: many0!(call!(Attribute::parse_outer)) >> + vis: syn!(Visibility) >> + defaultness: syn!(Defaultness) >> + type_: syn!(Type) >> + id: syn!(Ident) >> + eq: syn!(Eq) >> + ty: syn!(Ty) >> + semi: syn!(Semi) >> (ImplItem { ident: id, vis: vis, defaultness: defaultness, attrs: attrs, node: ImplItemType { - type_token: tokens::Type::default(), - eq_token: tokens::Eq::default(), - semi_token: tokens::Semi::default(), + type_token: type_, + eq_token: eq, + semi_token: semi, ty: ty, }.into(), }) )); named!(impl_item_macro -> ImplItem, do_parse!( - attrs: many0!(outer_attr) >> - what: path >> - punct!("!") >> - body: delimited >> - cond!(!body.is_braced(), punct!(";")) >> + attrs: many0!(call!(Attribute::parse_outer)) >> + mac: syn!(Mac) >> + cond!(!mac.is_braced(), syn!(Semi)) >> (ImplItem { ident: "".into(), vis: VisInherited {}.into(), defaultness: Defaultness::Final, attrs: attrs, - node: ImplItemKind::Macro(Mac { - path: what, - bang_token: tokens::Bang::default(), - tokens: vec![body], - }), + node: ImplItemKind::Macro(mac), }) )); - named!(impl_polarity -> ImplPolarity, alt!( - punct!("!") => { |_| ImplPolarity::Negative(tokens::Bang::default()) } - | - epsilon!() => { |_| ImplPolarity::Positive } - )); + impl Synom for ImplPolarity { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(Bang) => { ImplPolarity::Negative } + | + epsilon!() => { |_| ImplPolarity::Positive } + } + } + } - named!(constness -> Constness, alt!( - keyword!("const") => { |_| Constness::Const(tokens::Const::default()) } - | - epsilon!() => { |_| Constness::NotConst } - )); + impl Synom for Constness { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(Const) => { Constness::Const } + | + epsilon!() => { |_| Constness::NotConst } + } + } + } - named!(defaultness -> Defaultness, alt!( - keyword!("default") => { |_| Defaultness::Default(tokens::Default::default()) } - | - epsilon!() => { |_| Defaultness::Final } - )); + impl Synom for Defaultness { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(Default_) => { Defaultness::Default } + | + epsilon!() => { |_| Defaultness::Final } + } + } + } } #[cfg(feature = "printing")] diff --git a/src/krate.rs b/src/krate.rs index 7978a73890..8e86430792 100644 --- a/src/krate.rs +++ b/src/krate.rs @@ -11,32 +11,51 @@ ast_struct! { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use attr::parsing::inner_attr; - use item::parsing::items; - - named!(pub krate -> Crate, do_parse!( - // NOTE: The byte order mark and shebang are not tokens which can appear - // in a TokenStream, so we can't parse them anymore. - - //option!(byte_order_mark) >> - //shebang: option!(shebang) >> - attrs: many0!(inner_attr) >> - items: items >> - (Crate { - shebang: None, - attrs: attrs, - items: items, - }) - )); - - // named!(byte_order_mark -> &str, tag!("\u{feff}")); - - // named!(shebang -> String, do_parse!( - // tag!("#!") >> - // not!(tag!("[")) >> - // content: take_until!("\n") >> - // (format!("#!{}", content)) - // )); + + use synom::{IResult, Synom, ParseError}; + use proc_macro2::TokenTree; + + impl Synom for Crate { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + attrs: many0!(call!(Attribute::parse_inner)) >> + items: many0!(syn!(Item)) >> + (Crate { + shebang: None, + attrs: attrs, + items: items, + }) + } + } + + fn description() -> Option<&'static str> { + Some("crate") + } + + fn parse_str_all(mut input: &str) -> Result { + // Strip the BOM if it is present + const BOM: &'static str = "\u{feff}"; + if input.starts_with(BOM) { + input = &input[BOM.len()..]; + } + + let mut shebang = None; + if input.starts_with("#!") && !input.starts_with("#![") { + if let Some(idx) = input.find('\n') { + shebang = Some(input[..idx].to_string()); + input = &input[idx..]; + } else { + shebang = Some(input.to_string()); + input = ""; + } + } + + let mut krate: Crate = Self::parse_all(input.parse()?)?; + krate.shebang = shebang; + Ok(krate) + } + } } #[cfg(feature = "printing")] @@ -47,10 +66,6 @@ mod printing { impl ToTokens for Crate { fn to_tokens(&self, tokens: &mut Tokens) { - // TODO: how to handle shebang? - // if let Some(ref shebang) = self.shebang { - // tokens.append(&format!("{}\n", shebang)); - // } tokens.append_all(self.attrs.inner()); tokens.append_all(&self.items); } diff --git a/src/lib.rs b/src/lib.rs index e66a7e07d2..55f93efa04 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,9 +7,6 @@ extern crate proc_macro2; #[cfg(feature = "printing")] extern crate quote; -#[cfg(feature = "parsing")] -extern crate relex; - #[cfg_attr(feature = "parsing", macro_use)] extern crate synom; @@ -97,10 +94,8 @@ pub use ty::{Abi, AngleBracketedParameterData, BareFnArg, BareFnTy, FunctionRetT #[cfg(feature = "printing")] pub use ty::PathTokens; -mod span; -pub use span::Span; - -pub mod tokens; +pub use synom::span::Span; +pub use synom::tokens; pub use synom::delimited; #[cfg(feature = "visit")] @@ -109,273 +104,44 @@ pub mod visit; #[cfg(feature = "fold")] pub mod fold; -#[cfg(feature = "parsing")] -pub use parsing::*; - #[cfg(feature = "parsing")] mod parsing { use std::str::FromStr; use super::*; - use {derive, generics, ident, mac, ty, attr}; - use synom::{IResult, TokenStream}; - - use std::convert::From; - use std::error::Error; - use std::fmt; - - #[cfg(feature = "full")] - use {expr, item, krate}; - - #[derive(Debug)] - pub struct ParseError(String); - - impl Error for ParseError { - fn description(&self) -> &str { - &self.0 - } - } - - impl fmt::Display for ParseError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - ::fmt(&self.0, f) - } - } - - impl From for ParseError { - fn from(_: synom::LexError) -> ParseError { - ParseError("error while lexing input string".to_owned()) - } - } - - /// Parse the stringified representation of a struct or enum passed - /// to a `proc_macro_derive` function. - pub fn parse_derive_input(input: TokenStream) -> Result { - unwrap("derive input", derive::parsing::derive_input, input) - } - - /// Parse an entire crate into an AST. This function takes a string as input - /// instead of a TokenStream, as we need to handle parsing the BOM and - /// shebang from the string. - #[cfg(feature = "full")] - pub fn parse_crate(mut input: &str) -> Result { - // Strip the BOM if it is present - const BOM: &str = "\u{feff}"; - if input.starts_with(BOM) { - input = &input[BOM.len()..]; - } - - let mut shebang = None; - if input.starts_with("#!") && !input.starts_with("#![") { - if let Some(idx) = input.find('\n') { - shebang = Some(input[..idx].to_string()); - input = &input[idx..]; - } else { - shebang = Some(input.to_string()); - input = ""; + use synom::{Synom, ParseError}; + use proc_macro2::TokenStream; + + macro_rules! traits { + ($($ty:ident,)*) => ($( + impl From for $ty { + fn from(stream: TokenStream) -> $ty { + $ty::parse_all_unwrap(stream) + } } - } - let mut krate = unwrap("crate", krate::parsing::krate, - input.parse()?)?; - krate.shebang = shebang; - Ok(krate) + impl FromStr for $ty { + type Err = ParseError; - } - - #[cfg(feature = "full")] - pub fn parse_item(input: TokenStream) -> Result { - unwrap("item", item::parsing::item, input) - } - - #[cfg(feature = "full")] - pub fn parse_items(input: TokenStream) -> Result, ParseError> { - unwrap("items", item::parsing::items, input) - } - - #[cfg(feature = "full")] - pub fn parse_expr(input: TokenStream) -> Result { - unwrap("expression", expr::parsing::expr, input) - } - - pub fn parse_type(input: TokenStream) -> Result { - unwrap("type", ty::parsing::ty, input) - } - - /// Parse a path, such as `std::str::FromStr` or `::syn::parse_path`. - pub fn parse_path(input: TokenStream) -> Result { - unwrap("path", ty::parsing::path, input) - } - - pub fn parse_where_clause(input: TokenStream) -> Result { - unwrap("where clause", generics::parsing::where_clause, input) - } - - pub fn parse_token_trees(input: TokenStream) -> Result, ParseError> { - unwrap("token trees", mac::parsing::token_trees, input) - } - - pub fn parse_ident(input: TokenStream) -> Result { - unwrap("identifier", ident::parsing::ident, input) - } - - pub fn parse_ty_param_bound(input: TokenStream) -> Result { - unwrap("type parameter bound", - generics::parsing::ty_param_bound, - input) - } - - /// Parse an attribute declared outside the item it annotates, such as - /// a struct annotation. They are written as `#[...]`. - pub fn parse_outer_attr(input: TokenStream) -> Result { - unwrap("outer attribute", attr::parsing::outer_attr, input) - } - - /// Parse an attribute declared inside the item it annotates. These are used - /// for crate annotations or for mod-level declarations when modules are in - /// their own files. They are written as `#![...]`. - #[cfg(feature = "full")] - pub fn parse_inner_attr(input: TokenStream) -> Result { - unwrap("inner attribute", attr::parsing::inner_attr, input) - } - - /// Deprecated: Use `parse_derive_input` instead. - #[doc(hidden)] - #[deprecated(since="0.11.0", note = "Use `parse_derive_input` instead")] - pub fn parse_macro_input(input: TokenStream) -> Result { - parse_derive_input(input) - } - - /// Alias for `syn::parse_derive_input`. - impl FromStr for DeriveInput { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_derive_input(s.parse()?) - } - } - - /// Alias for `syn::parse_crate`. - #[cfg(feature = "full")] - impl FromStr for Crate { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_crate(s) - } - } - - /// Alias for `syn::parse_item`. - #[cfg(feature = "full")] - impl FromStr for Item { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_item(s.parse()?) - } - } - - /// Alias for `syn::parse_expr`. - #[cfg(feature = "full")] - impl FromStr for Expr { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_expr(s.parse()?) - } - } - - /// Alias for `syn::parse_type`. - impl FromStr for Ty { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_type(s.parse()?) - } - } - - /// Alias for `syn::parse_path`. - impl FromStr for Path { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_path(s.parse()?) - } - } - - /// Alias for `syn::parse_where_clause`. - impl FromStr for WhereClause { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_where_clause(s.parse()?) - } - } - - /// Alias for `syn::parse_ident`. - impl FromStr for Ident { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_ident(s.parse()?) - } - } - - /// Alias for `syn::parse_ty_param_bound`. - impl FromStr for TyParamBound { - type Err = ParseError; - - fn from_str(s: &str) -> Result { - parse_ty_param_bound(s.parse()?) - } - } - - fn unwrap(name: &'static str, - f: fn(&[synom::TokenTree]) -> IResult<&[synom::TokenTree], T>, - input: TokenStream) - -> Result { - let input = synom::InputBuf::new(input); - match f(&input) { - IResult::Done(rest, t) => { - if rest.is_empty() { - Ok(t) - } else if rest.len() == input.len() { - // parsed nothing - Err(ParseError(format!("failed to parse {}", name))) - } else { - Err(ParseError(format!("unparsed tokens after {}", name))) + fn from_str(s: &str) -> Result { + $ty::parse_str_all(s) } } - IResult::Error => Err(ParseError(format!("failed to parse {}", name))), - } + )*) } -} -#[cfg(feature = "parsing")] -pub mod parse { - //! This module contains a set of exported nom parsers which can be used to - //! parse custom grammars when used alongside the `synom` crate. - //! - //! Internally, `syn` uses a fork of `nom` called `synom` which resolves a - //! persistent pitfall of using `nom` to parse Rust by eliminating the - //! `IResult::Incomplete` variant. The `synom` crate should be used instead - //! of `nom` when working with the parsers in this module. - - pub use synom::IResult; - - #[cfg(feature = "full")] - pub use item::parsing::item; + traits! { + DeriveInput, + TyParamBound, + Ident, + WhereClause, + Ty, + } #[cfg(feature = "full")] - pub use expr::parsing::{expr, pat, block, stmt}; - - pub use lit::parsing::{lit, string, byte_string, byte, character, float, int, boolean}; - - pub use ty::parsing::{ty, path}; - - pub use mac::parsing::token_tree as tt; - - pub use ident::parsing::ident; - - pub use generics::parsing::lifetime; + traits! { + Expr, + Item, + Crate, + } } diff --git a/src/lit.rs b/src/lit.rs index fbec4a21e3..0be32d5651 100644 --- a/src/lit.rs +++ b/src/lit.rs @@ -101,308 +101,26 @@ impl Hash for LitKind { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use escape::{cooked_byte, cooked_byte_string, cooked_char, cooked_string, raw_string}; - use proc_macro2::Literal; - use synom::IResult; - use synom::space::skip_whitespace; - use unicode_xid::UnicodeXID; - - fn l>(t: T) -> Lit { - Lit { - value: LitKind::Other(t.into()), - span: Default::default(), - } - } - - named!(pub lit -> Lit, alt!( - string - | - byte_string - | - byte - | - character - | - float - | - int - | - boolean - )); - - named!(pub string -> Lit, alt!( - quoted_string => { |s: String| l(&s[..]) } - | - preceded!( - punct!("r"), - raw_string - ) => { |(s, n): (String, _)| l(Literal::raw_string(&s[..], n)) } - )); - - named!(pub quoted_string -> String, delimited!( - punct!("\""), - cooked_string, - tag!("\"") - )); - - named!(pub byte_string -> Lit, alt!( - delimited!( - punct!("b\""), - cooked_byte_string, - tag!("\"") - ) => { |vec: Vec| l(Literal::byte_string(&vec)) } - | - preceded!( - punct!("br"), - raw_string - ) => { |(s, n): (String, _)| l(Literal::raw_byte_string(&s, n)) } - )); - - named!(pub byte -> Lit, do_parse!( - punct!("b") >> - tag!("'") >> - b: cooked_byte >> - tag!("'") >> - (l(Literal::byte_char(b))) - )); - - named!(pub character -> Lit, do_parse!( - punct!("'") >> - ch: cooked_char >> - tag!("'") >> - (l(ch)) - )); - - named!(pub float -> Lit, do_parse!( - value: float_string >> - suffix: alt!( - tag!("f32") - | - tag!("f64") - | - epsilon!() => { |_| "" } - ) >> - (l(Literal::float(&format!("{}{}", value, suffix)))) - )); - - named!(pub int -> Lit, do_parse!( - value: digits >> - suffix: alt!( - tag!("isize") - | - tag!("i8") - | - tag!("i16") - | - tag!("i32") - | - tag!("i64") - | - tag!("usize") - | - tag!("u8") - | - tag!("u16") - | - tag!("u32") - | - tag!("u64") - | - epsilon!() => { |_| "" } - ) >> - (l(Literal::integer(&format!("{}{}", value, suffix)))) - )); - - named!(pub boolean -> Lit, alt!( - keyword!("true") => { |_| Lit { - span: Span::default(), - value: LitKind::Bool(true), - } } - | - keyword!("false") => { |_| Lit { - span: Span::default(), - value: LitKind::Bool(false), - } } - )); - - fn float_string(mut input: &str) -> IResult<&str, String> { - input = skip_whitespace(input); - - let mut chars = input.chars().peekable(); - match chars.next() { - Some(ch) if ch >= '0' && ch <= '9' => {} - _ => return IResult::Error, - } - - let mut len = 1; - let mut has_dot = false; - let mut has_exp = false; - while let Some(&ch) = chars.peek() { - match ch { - '0'...'9' | '_' => { - chars.next(); - len += 1; - } - '.' => { - if has_dot { - break; - } - relex::LToken::Lit(relex::Lit::Char(c)) => { - Lit::Char(c) - } - relex::LToken::Lit(relex::Lit::Integer(v, suffix)) => { - let suffix = match suffix { - relex::IntSuffix::Unsuffixed => - IntTy::Unsuffixed, - relex::IntSuffix::Isize => - IntTy::Isize, - relex::IntSuffix::Usize => - IntTy::Usize, - relex::IntSuffix::I8 => - IntTy::I8, - relex::IntSuffix::U8 => - IntTy::U8, - relex::IntSuffix::I16 => - IntTy::I16, - relex::IntSuffix::U16 => - IntTy::U16, - relex::IntSuffix::I32 => - IntTy::I32, - relex::IntSuffix::U32 => - IntTy::U32, - relex::IntSuffix::I64 => - IntTy::I64, - relex::IntSuffix::U64 => - IntTy::U64, - }; - Lit::Int(v, suffix) - } - relex::LToken::Lit(relex::Lit::Float(v, suffix)) => { - let suffix = match suffix { - relex::FloatSuffix::Unsuffixed => - FloatTy::Unsuffixed, - relex::FloatSuffix::F32 => - FloatTy::F32, - relex::FloatSuffix::F64 => - FloatTy::F64, - }; - Lit::Float(v, suffix) - } - relex::LToken::Lit(relex::Lit::Str(s, relex::StrStyle::Cooked)) => { - Lit::Str(s, StrStyle::Cooked) - } - relex::LToken::Lit(relex::Lit::Str(s, relex::StrStyle::Raw(n))) => { - Lit::Str(s, StrStyle::Raw(n)) - } - relex::LToken::Lit(relex::Lit::ByteStr(s, relex::StrStyle::Cooked)) => { - Lit::ByteStr(s, StrStyle::Cooked) - } - relex::LToken::Lit(relex::Lit::ByteStr(s, relex::StrStyle::Raw(n))) => { - Lit::ByteStr(s, StrStyle::Raw(n)) - } - _ => return IResult::Error - }; - - IResult::Done(&i[1..], lit) - } - Some(&TokenTree{ kind: TokenKind::Word(ref w), .. }) => { - if &**w == "true" { - IResult::Done(&i[1..], Lit::Bool(true)) - } else if &**w == "false" { - IResult::Done(&i[1..], Lit::Bool(false)) - } else { - IResult::Error - } - } - _ => IResult::Error - } - } - - #[cfg(feature = "full")] - pub fn digits(i: &[TokenTree]) -> IResult<&[TokenTree], u64> { - if let IResult::Done(r, Lit::Int(v, IntTy::Unsuffixed)) = lit(i) { - IResult::Done(r, v) - } else { - IResult::Error - } - } - - pub fn string(i: &[TokenTree]) -> IResult<&[TokenTree], String> { - if let IResult::Done(r, Lit::Str(v, _)) = lit(i) { - IResult::Done(r, v) - } else { - IResult::Error - } - } - - pub fn byte_string(i: &[TokenTree]) -> IResult<&[TokenTree], Vec> { - if let IResult::Done(r, Lit::ByteStr(v, _)) = lit(i) { - IResult::Done(r, v) - } else { - IResult::Error - } - } - - pub fn digits(mut input: &str) -> IResult<&str, &str> { - input = skip_whitespace(input); - - let base = if input.starts_with("0x") { - 16 - } else if input.starts_with("0o") { - 8 - } else if input.starts_with("0b") { - 2 - } else { - IResult::Error - } - } - - let mut value = 0u64; - let mut len = if base == 10 {0} else {2}; - let mut empty = true; - for b in input[len..].bytes() { - let digit = match b { - b'0'...b'9' => (b - b'0') as u64, - b'a'...b'f' => 10 + (b - b'a') as u64, - b'A'...b'F' => 10 + (b - b'A') as u64, - b'_' => { - if empty && base == 10 { - return IResult::Error; - } - len += 1; - continue; - } - _ => break, - }; - if digit >= base { - return IResult::Error; - } - value = match value.checked_mul(base) { - Some(value) => value, + use proc_macro2::TokenTree; + use synom::{Synom, IResult}; + + impl Synom for Lit { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + let mut tokens = input.iter(); + let token = match tokens.next() { + Some(token) => token, None => return IResult::Error, }; - value = match value.checked_add(digit) { - Some(value) => value, - None => return IResult::Error, + let kind = match token.kind { + TokenKind::Literal(ref l) => LitKind::Other(l.clone()), + TokenKind::Word(ref s) if s.as_str() == "true" => LitKind::Bool(true), + TokenKind::Word(ref s) if s.as_str() == "false" => LitKind::Bool(false), + _ => return IResult::Error, }; - len += 1; - empty = false; - } - } - - pub fn int(i: &[TokenTree]) -> IResult<&[TokenTree], u64> { - if let IResult::Done(r, Lit::Int(v, _)) = lit(i) { - IResult::Done(r, v) - } else { - IResult::Error - } - } - - pub fn boolean(i: &[TokenTree]) -> IResult<&[TokenTree], bool> { - if let IResult::Done(r, Lit::Bool(b)) = lit(i) { - IResult::Done(r, b) - } else { - IResult::Done(&input[len..], &input[..len]) + IResult::Done(tokens.as_slice(), Lit { + span: Span(token.span), + value: kind, + }) } } } diff --git a/src/mac.rs b/src/mac.rs index 8ce8c33d53..5e6d02e003 100644 --- a/src/mac.rs +++ b/src/mac.rs @@ -134,195 +134,42 @@ impl fmt::Debug for TokenTree { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use {Lifetime}; - use generics::parsing::lifetime; - use ident::parsing::word; - use lit::parsing::lit; - use synom::IResult; - use synom::space::{block_comment, whitespace, skip_whitespace}; - use ty::parsing::path; - use proc_macro2::{self, TokenStream, TokenKind, Delimiter, OpKind, Literal}; - fn tt(kind: TokenKind) -> TokenTree { - TokenTree(proc_macro2::TokenTree { - kind: kind, - span: Default::default(), - }) + use proc_macro2::{TokenKind, TokenTree}; + use synom::tokens::*; + use synom::{Synom, IResult}; + + impl Synom for Mac { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + what: syn!(Path) >> + bang: syn!(Bang) >> + body: call!(::TokenTree::parse_delimited) >> + (Mac { + path: what, + bang_token: bang, + tokens: vec![body], + }) + } + } } - named!(pub mac -> Mac, do_parse!( - what: path >> - punct!("!") >> - body: delimited >> - (Mac { - path: what, - bang_token: tokens::Bang::default(), - tokens: vec![body], - }) - )); - - named!(pub token_trees -> Vec, many0!(token_tree)); - - named!(pub token_stream -> TokenStream, - map!(token_trees, |t: Vec| t.into_iter().map(|t| t.0).collect())); - - named!(pub delimited -> TokenTree, alt!( - delimited!( - punct!("("), - token_stream, - punct!(")") - ) => { |ts| tt(TokenKind::Sequence(Delimiter::Parenthesis, ts)) } - | - delimited!( - punct!("["), - token_stream, - punct!("]") - ) => { |ts| tt(TokenKind::Sequence(Delimiter::Bracket, ts)) } - | - delimited!( - punct!("{"), - token_stream, - punct!("}") - ) => { |ts| tt(TokenKind::Sequence(Delimiter::Brace, ts)) } - )); - - named!(pub token_tree -> TokenTree, alt!( - token - | - delimited - )); - - macro_rules! punct1 { - ($i:expr, $punct:expr) => { - punct1($i, $punct) + impl ::TokenTree { + pub fn parse_list(input: &[TokenTree]) -> IResult<&[TokenTree], Vec> { + IResult::Done(&[], input.iter().cloned().map(::TokenTree).collect()) } - } - fn punct1<'a>(input: &'a str, token: &'static str) -> IResult<&'a str, char> { - let input = skip_whitespace(input); - if input.starts_with(token) { - IResult::Done(&input[1..], token.chars().next().unwrap()) - } else { - IResult::Error + pub fn parse_delimited(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + let mut tokens = input.iter(); + match tokens.next() { + Some(token @ &TokenTree { kind: TokenKind::Sequence(..), .. }) => { + IResult::Done(tokens.as_slice(), ::TokenTree(token.clone())) + } + _ => IResult::Error, + } } } - - named!(token -> TokenTree, alt!( - keyword!("_") => { |_| tt(TokenKind::Op('_', OpKind::Alone)) } - | - punct1!("&&") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } // must be before BinOp - | - punct1!("||") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } // must be before BinOp - | - punct1!("->") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } // must be before BinOp - | - punct1!("<-") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } // must be before Lt - | - punct1!("=>") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } // must be before Eq - | - punct1!("...") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } // must be before DotDot - | - punct1!("..") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } // must be before Dot - | - punct1!(".") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - // must be before bin_op - map!(doc_comment, |s: String| tt(TokenKind::Literal(Literal::doccomment(&s)))) - | - bin_op_eq // must be before bin_op - | - bin_op - | - map!(lit, |l: Lit| l.into_token_tree()) - | - map!(word, |w: Ident| tt(TokenKind::Word(w.sym))) - | - map!(lifetime, |lt: Lifetime| tt(TokenKind::Word(lt.ident.sym))) - | - punct1!("<=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("==") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("!=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!(">=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("::") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("=") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("<") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!(">") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("!") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("~") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("@") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!(",") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!(";") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!(":") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("#") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("$") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("?") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - )); - - named!(bin_op -> TokenTree, alt!( - punct1!("+") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("-") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("*") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("/") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("%") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("^") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("&") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("|") => { |c| tt(TokenKind::Op(c, OpKind::Alone)) } - | - punct1!("<<") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!(">>") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - )); - - named!(bin_op_eq -> TokenTree, alt!( - punct1!("+=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("-=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("*=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("/=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("%=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("^=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("&=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("|=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!("<<=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - | - punct1!(">>=") => { |c| tt(TokenKind::Op(c, OpKind::Joint)) } - )); - - named!(doc_comment -> String, alt!( - inner_doc_comment - | - outer_doc_comment - )); } #[cfg(feature = "printing")] diff --git a/src/op.rs b/src/op.rs index 636b9743ba..e0599299a5 100644 --- a/src/op.rs +++ b/src/op.rs @@ -77,75 +77,90 @@ ast_enum! { #[cfg(feature = "parsing")] pub mod parsing { use super::*; + use synom::{TokenTree, IResult, Synom}; + use synom::tokens::*; - named!(pub binop -> BinOp, alt!( - punct!("&&") => { |_| BinOp::And(tokens::AndAnd::default()) } - | - punct!("||") => { |_| BinOp::Or(tokens::OrOr::default()) } - | - punct!("<<") => { |_| BinOp::Shl(tokens::Shl::default()) } - | - punct!(">>") => { |_| BinOp::Shr(tokens::Shr::default()) } - | - punct!("==") => { |_| BinOp::Eq(tokens::EqEq::default()) } - | - punct!("<=") => { |_| BinOp::Le(tokens::Le::default()) } - | - punct!("!=") => { |_| BinOp::Ne(tokens::Ne::default()) } - | - punct!(">=") => { |_| BinOp::Ge(tokens::Ge::default()) } - | - punct!("+") => { |_| BinOp::Add(tokens::Add::default()) } - | - punct!("-") => { |_| BinOp::Sub(tokens::Sub::default()) } - | - punct!("*") => { |_| BinOp::Mul(tokens::Star::default()) } - | - punct!("/") => { |_| BinOp::Div(tokens::Div::default()) } - | - punct!("%") => { |_| BinOp::Rem(tokens::Rem::default()) } - | - punct!("^") => { |_| BinOp::BitXor(tokens::Caret::default()) } - | - punct!("&") => { |_| BinOp::BitAnd(tokens::And::default()) } - | - punct!("|") => { |_| BinOp::BitOr(tokens::Or::default()) } - | - punct!("<") => { |_| BinOp::Lt(tokens::Lt::default()) } - | - punct!(">") => { |_| BinOp::Gt(tokens::Gt::default()) } - )); + impl BinOp { + pub fn parse_binop(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(AndAnd) => { BinOp::And } + | + syn!(OrOr) => { BinOp::Or } + | + syn!(Shl) => { BinOp::Shl } + | + syn!(Shr) => { BinOp::Shr } + | + syn!(EqEq) => { BinOp::Eq } + | + syn!(Le) => { BinOp::Le } + | + syn!(Ne) => { BinOp::Ne } + | + syn!(Ge) => { BinOp::Ge } + | + syn!(Add) => { BinOp::Add } + | + syn!(Sub) => { BinOp::Sub } + | + syn!(Star) => { BinOp::Mul } + | + syn!(Div) => { BinOp::Div } + | + syn!(Rem) => { BinOp::Rem } + | + syn!(Caret) => { BinOp::BitXor } + | + syn!(And) => { BinOp::BitAnd } + | + syn!(Or) => { BinOp::BitOr } + | + syn!(Lt) => { BinOp::Lt } + | + syn!(Gt) => { BinOp::Gt } + } + } - #[cfg(feature = "full")] - named!(pub assign_op -> BinOp, alt!( - punct!("+=") => { |_| BinOp::AddEq(tokens::AddEq::default()) } - | - punct!("-=") => { |_| BinOp::SubEq(tokens::SubEq::default()) } - | - punct!("*=") => { |_| BinOp::MulEq(tokens::MulEq::default()) } - | - punct!("/=") => { |_| BinOp::DivEq(tokens::DivEq::default()) } - | - punct!("%=") => { |_| BinOp::RemEq(tokens::RemEq::default()) } - | - punct!("^=") => { |_| BinOp::BitXorEq(tokens::CaretEq::default()) } - | - punct!("&=") => { |_| BinOp::BitAndEq(tokens::AndEq::default()) } - | - punct!("|=") => { |_| BinOp::BitOrEq(tokens::OrEq::default()) } - | - punct!("<<=") => { |_| BinOp::ShlEq(tokens::ShlEq::default()) } - | - punct!(">>=") => { |_| BinOp::ShrEq(tokens::ShrEq::default()) } - )); + #[cfg(feature = "full")] + pub fn parse_assign_op(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(AddEq) => { BinOp::AddEq } + | + syn!(SubEq) => { BinOp::SubEq } + | + syn!(MulEq) => { BinOp::MulEq } + | + syn!(DivEq) => { BinOp::DivEq } + | + syn!(RemEq) => { BinOp::RemEq } + | + syn!(CaretEq) => { BinOp::BitXorEq } + | + syn!(AndEq) => { BinOp::BitAndEq } + | + syn!(OrEq) => { BinOp::BitOrEq } + | + syn!(ShlEq) => { BinOp::ShlEq } + | + syn!(ShrEq) => { BinOp::ShrEq } + } + } + } - named!(pub unop -> UnOp, alt!( - punct!("*") => { |_| UnOp::Deref(tokens::Star::default()) } - | - punct!("!") => { |_| UnOp::Not(tokens::Bang::default()) } - | - punct!("-") => { |_| UnOp::Neg(tokens::Sub::default()) } - )); + impl Synom for UnOp { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(Star) => { UnOp::Deref } + | + syn!(Bang) => { UnOp::Not } + | + syn!(Sub) => { UnOp::Neg } + } + } + } } #[cfg(feature = "printing")] diff --git a/src/ty.rs b/src/ty.rs index c6e8539641..9c3649b8a7 100644 --- a/src/ty.rs +++ b/src/ty.rs @@ -257,7 +257,7 @@ ast_struct! { pub lifetimes: Option, pub unsafety: Unsafety, pub abi: Option, - pub fn_token: tokens::Fn, + pub fn_token: tokens::Fn_, pub paren_token: tokens::Paren, pub inputs: Delimited, pub variadic: Option, @@ -314,168 +314,213 @@ ast_enum! { #[cfg(feature = "parsing")] pub mod parsing { use super::*; - use {TyParamBound, TraitBoundModifier}; - #[cfg(feature = "full")] - use ConstExpr; - #[cfg(feature = "full")] - use constant::parsing::const_expr; - #[cfg(feature = "full")] - use expr::parsing::expr; - use generics::parsing::{lifetime, ty_param_bound, bound_lifetimes}; - use ident::parsing::ident; - use lit::parsing::string; - use mac::parsing::mac; - #[cfg(feature = "full")] - use synom::{IResult, TokenTree}; - - named!(pub ty -> Ty, alt!( - ty_paren // must be before ty_tup - | - ty_mac // must be before ty_path - | - ty_path // must be before ty_poly_trait_ref - | - ty_vec - | - ty_array - | - ty_ptr - | - ty_rptr - | - ty_bare_fn - | - ty_never - | - ty_tup - | - ty_poly_trait_ref - | - ty_impl_trait - )); + // use {TyParamBound, TraitBoundModifier}; + // #[cfg(feature = "full")] + // use ConstExpr; + // #[cfg(feature = "full")] + // use constant::parsing::const_expr; + // #[cfg(feature = "full")] + // use expr::parsing::expr; + // use generics::parsing::{lifetime, ty_param_bound, bound_lifetimes}; + // use ident::parsing::ident; + // use lit::parsing::string; + // use mac::parsing::mac; + use proc_macro2::TokenTree; + use synom::{IResult, Synom}; + use synom::tokens::*; + + impl Synom for Ty { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + // must be before mac + syn!(TyParen) => { Ty::Paren } + | + // must be before path + syn!(Mac) => { Ty::Mac } + | + // must be before ty_poly_trait_ref + ty_path + | + syn!(TySlice) => { Ty::Slice } + | + syn!(TyArray) => { Ty::Array } + | + syn!(TyPtr) => { Ty::Ptr } + | + syn!(TyRptr) => { Ty::Rptr } + | + syn!(TyBareFn) => { Ty::BareFn } + | + syn!(TyNever) => { Ty::Never } + | + syn!(TyTup) => { Ty::Tup } + | + ty_poly_trait_ref + | + syn!(TyImplTrait) => { Ty::ImplTrait } + } + } - named!(ty_mac -> Ty, map!(mac, Ty::Mac)); + fn description() -> Option<&'static str> { + Some("type") + } + } - named!(ty_vec -> Ty, delim!(Bracket, do_parse!( - elem: ty >> - punct!("]") >> - (TySlice { - ty: Box::new(elem), - bracket_token: tokens::Bracket::default(), - }.into()) - )); + impl Synom for TySlice { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + map! { + input, + brackets!(syn!(Ty)), + |(ty, b)| TySlice { + ty: Box::new(ty), + bracket_token: b, + } + } + } + } - named!(ty_array -> Ty, delim!(Bracket, do_parse!( - elem: ty >> - punct!(";") >> - len: array_len >> - punct!("]") >> - (TyArray { - ty: Box::new(elem), - amt: len, - bracket_token: tokens::Bracket::default(), - semi_token: tokens::Semi::default(), - }.into()) - )); + impl Synom for TyArray { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + map! { + input, + brackets!(do_parse!( + elem: syn!(Ty) >> + semi: syn!(Semi) >> + len: array_len >> + (elem, semi, len) + )), + |((elem, semi, len), brackets)| { + TyArray { + ty: Box::new(elem), + amt: len, + bracket_token: brackets, + semi_token: semi, + } + } + } + } + } #[cfg(not(feature = "full"))] - use constant::parsing::const_expr as array_len; + named!(array_len -> ConstExpr, syn!(ConstExpr)); #[cfg(feature = "full")] named!(array_len -> ConstExpr, alt!( - terminated!(const_expr, input_end!()) + terminated!(syn!(ConstExpr), input_end!()) | - terminated!(expr, input_end!()) => { ConstExpr::Other } + terminated!(syn!(Expr), input_end!()) => { ConstExpr::Other } )); - named!(ty_ptr -> Ty, do_parse!( - punct!("*") >> - mutability: alt!( - keyword!("const") => { |_| Mutability::Immutable } - | - keyword!("mut") => { |_| Mutability::Mutable(tokens::Mut::default()) } - ) >> - target: ty >> - (TyPtr { - const_token: match mutability { - Mutability::Mutable(_) => None, - Mutability::Immutable => Some(tokens::Const::default()), - }, - star_token: tokens::Star::default(), - ty: Box::new(MutTy { - ty: target, - mutability: mutability, - }), - }.into()) - )); + impl Synom for TyPtr { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + star: syn!(Star) >> + mutability: alt!( + syn!(Const) => { |c| (Mutability::Immutable, Some(c)) } + | + syn!(Mut) => { |m| (Mutability::Mutable(m), None) } + ) >> + target: syn!(Ty) >> + (TyPtr { + const_token: mutability.1, + star_token: star, + ty: Box::new(MutTy { + ty: target, + mutability: mutability.0, + }), + }) + } + } + } - named!(ty_rptr -> Ty, do_parse!( - punct!("&") >> - life: option!(lifetime) >> - mutability: mutability >> - target: ty >> - (TyRptr { - lifetime: life, - ty: Box::new(MutTy { - ty: target, - mutability: mutability, - }), - and_token: tokens::And::default(), - }.into()) - )); + impl Synom for TyRptr { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + amp: syn!(And) >> + life: option!(syn!(Lifetime)) >> + mutability: syn!(Mutability) >> + target: syn!(Ty) >> + (TyRptr { + lifetime: life, + ty: Box::new(MutTy { + ty: target, + mutability: mutability, + }), + and_token: amp, + }) + } + } + } - named!(ty_bare_fn -> Ty, do_parse!( - lifetimes: bound_lifetimes >> - unsafety: unsafety >> - abi: option!(abi) >> - keyword!("fn") >> - punct!("(") >> - inputs: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - fn_arg) >> - variadic: option!(cond_reduce!(inputs.is_empty() || inputs.trailing_delim(), - punct!("..."))) >> - punct!(")") >> - output: fn_ret_ty >> - (TyBareFn { - ty: Box::new(BareFnTy { - unsafety: unsafety, - abi: abi, - lifetimes: lifetimes, - inputs: inputs, - output: output, - variadic: variadic.map(|_| tokens::Dot3::default()), - fn_token: tokens::Fn::default(), - paren_token: tokens::Paren::default(), - }), - }.into()) - )); + impl Synom for TyBareFn { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + lifetimes: option!(syn!(BoundLifetimes)) >> + unsafety: syn!(Unsafety) >> + abi: option!(syn!(Abi)) >> + fn_: syn!(Fn_) >> + parens: parens!(do_parse!( + inputs: call!(Delimited::parse_terminated) >> + variadic: option!(cond_reduce!(inputs.is_empty() || inputs.trailing_delim(), + syn!(Dot3))) >> + (inputs, variadic) + )) >> + output: syn!(FunctionRetTy) >> + (TyBareFn { + ty: Box::new(BareFnTy { + unsafety: unsafety, + abi: abi, + lifetimes: lifetimes, + output: output, + variadic: (parens.0).1, + fn_token: fn_, + paren_token: parens.1, + inputs: (parens.0).0, + }), + }) + } + } + } - named!(ty_never -> Ty, map!(punct!("!"), |_| TyNever { - bang_token: tokens::Bang::default(), - }.into())); - - named!(ty_tup -> Ty, do_parse!( - punct!("(") >> - elems: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - ty) >> - punct!(")") >> - (TyTup { - tys: elems, - paren_token: tokens::Paren::default(), - lone_comma: None, // TODO: does this just not parse? - }.into()) - )); + impl Synom for TyNever { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + map! { + input, + syn!(Bang), + |b| TyNever { bang_token: b } + } + } + } + + impl Synom for TyTup { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + data: parens!(call!(Delimited::parse_terminated)) >> + (TyTup { + tys: data.0, + paren_token: data.1, + lone_comma: None, // TODO: does this just not parse? + }) + } + } + } named!(ty_path -> Ty, do_parse!( qpath: qpath >> parenthesized: cond!( qpath.1.segments.get(qpath.1.segments.len() - 1).item().parameters.is_empty(), - option!(parenthesized_parameter_data) + option!(syn!(ParenthesizedParameterData)) ) >> - bounds: many0!(preceded!(punct!("+"), ty_param_bound)) >> + bounds: many0!(tuple!(syn!(Add), syn!(TyParamBound))) >> ({ let (qself, mut path) = qpath; if let Some(Some(parenthesized)) = parenthesized { + let parenthesized = PathParameters::Parenthesized(parenthesized); let len = path.segments.len(); path.segments.get_mut(len - 1).item_mut().parameters = parenthesized; } @@ -491,7 +536,7 @@ pub mod parsing { ); let mut new_bounds = Delimited::new(); new_bounds.push_first(path); - for bound in bounds { + for (_tok, bound) in bounds { new_bounds.push_default(bound); } TyTraitObject { bounds: new_bounds }.into() @@ -499,263 +544,299 @@ pub mod parsing { }) )); - named!(parenthesized_parameter_data -> PathParameters, do_parse!( - punct!("(") >> - inputs: terminated_list!(map!(punct!(","), |_| tokens::Comma::default()), - ty) >> - punct!(")") >> - output: fn_ret_ty >> - (PathParameters::Parenthesized( - ParenthesizedParameterData { - paren_token: tokens::Paren::default(), - inputs: inputs, - output: output, - }, - )) - )); - - named!(pub fn_ret_ty -> FunctionRetTy, map!( - option!(preceded!( - punct!("->"), - ty - )), - |ty| { - match ty { - Some(ty) => FunctionRetTy::Ty(ty, tokens::RArrow::default()), - None => FunctionRetTy::Default, - } - } - )); - named!(pub qpath -> (Option, Path), alt!( - map!(path, |p| (None, p)) + map!(syn!(Path), |p| (None, p)) | do_parse!( - punct!("<") >> - this: map!(ty, Box::new) >> - path: option!(preceded!( - keyword!("as"), - path + lt: syn!(Lt) >> + this: map!(syn!(Ty), Box::new) >> + path: option!(do_parse!( + as_: syn!(As) >> + path: syn!(Path) >> + (as_, path) )) >> - punct!(">") >> - punct!("::") >> - rest: separated_nonempty_list!( - map!(punct!("::"), |_| tokens::Colon2::default()), - path_segment - ) >> + gt: syn!(Gt) >> + colon2: syn!(Colon2) >> + rest: call!(Delimited::parse_separated_nonempty) >> ({ - let as_token = path.as_ref().map(|_| tokens::As::default()); - let (pos, path) = match path { - Some(mut path) => { + let (pos, path, as_) = match path { + Some((as_, mut path)) => { let pos = path.segments.len(); if !path.segments.is_empty() && !path.segments.trailing_delim() { - path.segments.push_trailing(tokens::Colon2::default()); + path.segments.push_trailing(colon2); } for item in rest.into_iter() { path.segments.push(item); } - (pos, path) + (pos, path, Some(as_)) } None => { (0, Path { leading_colon: None, global: false, segments: rest, - }) + }, None) } }; (Some(QSelf { ty: this, position: pos, - gt_token: tokens::Gt::default(), - lt_token: tokens::Lt::default(), - as_token: as_token, + gt_token: gt, + lt_token: lt, + as_token: as_, }), path) }) ) | - map!(keyword!("self"), |_| (None, "self".into())) + map!(syn!(Self_), |s: Self_| (None, s.into())) )); + impl Synom for ParenthesizedParameterData { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + data: parens!(call!(Delimited::parse_terminated)) >> + output: syn!(FunctionRetTy) >> + (ParenthesizedParameterData { + paren_token: data.1, + inputs: data.0, + output: output, + }) + } + } + } + + impl Synom for FunctionRetTy { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + arrow: syn!(RArrow) >> + ty: syn!(Ty) >> + (FunctionRetTy::Ty(ty, arrow)) + ) + | + epsilon!() => { |_| FunctionRetTy::Default } + } + } + } + named!(ty_poly_trait_ref -> Ty, map!( - separated_nonempty_list!(map!(punct!("+"), |_| tokens::Add::default()), - ty_param_bound), + call!(Delimited::parse_separated_nonempty), |x| TyTraitObject { bounds: x }.into() )); - named!(ty_impl_trait -> Ty, do_parse!( - keyword!("impl") >> - elem: separated_nonempty_list!(map!(punct!("+"), |_| tokens::Add::default()), - ty_param_bound) >> - (TyImplTrait { - impl_token: tokens::Impl::default(), - bounds: elem, - }.into()) - )); + impl Synom for TyImplTrait { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + impl_: syn!(Impl) >> + elem: call!(Delimited::parse_separated_nonempty) >> + (TyImplTrait { + impl_token: impl_, + bounds: elem, + }) + } + } + } - named!(ty_paren -> Ty, do_parse!( - punct!("(") >> - elem: ty >> - punct!(")") >> - (TyParen { - paren_token: tokens::Paren::default(), - ty: Box::new(elem), - }.into()) - )); + impl Synom for TyParen { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + data: parens!(syn!(Ty)) >> + (TyParen { + paren_token: data.1, + ty: Box::new(data.0), + }) + } + } + } - named!(pub mutability -> Mutability, alt!( - keyword!("mut") => { |_| Mutability::Mutable(tokens::Mut::default()) } - | - epsilon!() => { |_| Mutability::Immutable } - )); + impl Synom for Mutability { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(Mut) => { Mutability::Mutable } + | + epsilon!() => { |_| Mutability::Immutable } + } + } + } - named!(pub path -> Path, do_parse!( - global: option!(punct!("::")) >> - segments: separated_nonempty_list!(map!(punct!("::"), |_| tokens::Colon2::default()), - path_segment) >> - (Path { - global: global.is_some(), - segments: segments, - leading_colon: global.map(|_| tokens::Colon2::default()), - }) - )); + impl Synom for Path { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + global: option!(syn!(Colon2)) >> + segments: call!(Delimited::parse_separated_nonempty) >> + (Path { + global: global.is_some(), + segments: segments, + leading_colon: global, + }) + } + } + } - named!(path_segment -> PathSegment, alt!( - do_parse!( - id: option!(ident) >> - punct!("<") >> - lifetimes: terminated_list!( - map!(punct!(","), |_| tokens::Comma::default()), - lifetime - ) >> - types: cond!( - lifetimes.is_empty() || lifetimes.trailing_delim(), - terminated_list!( - map!(punct!(","), |_| tokens::Comma::default()), - terminated!(ty, not!(punct!("="))) - ) - ) >> - bindings: cond!( - match types { - Some(ref t) => t.is_empty() || t.trailing_delim(), - None => lifetimes.is_empty() || lifetimes.trailing_delim(), - }, - terminated_list!( - map!(punct!(","), |_| tokens::Comma::default()), - type_binding + impl Synom for PathSegment { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + do_parse!( + id: option!(syn!(Ident)) >> + lt: syn!(Lt) >> + lifetimes: call!(Delimited::parse_terminated) >> + types: cond!( + lifetimes.is_empty() || lifetimes.trailing_delim(), + call!(Delimited::parse_terminated_with, + ty_no_eq_after) + ) >> + bindings: cond!( + match types { + Some(ref t) => t.is_empty() || t.trailing_delim(), + None => lifetimes.is_empty() || lifetimes.trailing_delim(), + }, + call!(Delimited::parse_terminated) + ) >> + gt: syn!(Gt) >> + (PathSegment { + ident: id.unwrap_or_else(|| "".into()), + parameters: PathParameters::AngleBracketed( + AngleBracketedParameterData { + gt_token: Some(gt), + lt_token: Some(lt), + lifetimes: lifetimes, + types: types.unwrap_or_default(), + bindings: bindings.unwrap_or_default(), + } + ), + }) ) - ) >> - punct!(">") >> - (PathSegment { - ident: id.unwrap_or_else(|| "".into()), - parameters: PathParameters::AngleBracketed( - AngleBracketedParameterData { - gt_token: Some(tokens::Gt::default()), - lt_token: Some(tokens::Lt::default()), - lifetimes: lifetimes, - types: types.unwrap_or_default(), - bindings: bindings.unwrap_or_default(), - } - ), - }) - ) - | - map!(ident, Into::into) - | - map!(alt!( - keyword!("super") - | - keyword!("self") - | - keyword!("Self") - ), Into::into) - )); - - named!(pub mod_style_path -> Path, do_parse!( - global: option!(punct!("::")) >> - segments: separated_nonempty_list!(map!(punct!("::"), |_| tokens::Colon2::default()), - mod_style_path_segment) >> - (Path { - global: global.is_some(), - segments: segments, - leading_colon: global.map(|_| tokens::Colon2::default()), - }) - )); + | + mod_style_path_segment + } + } + } + named!(ty_no_eq_after -> Ty, terminated!(syn!(Ty), not!(syn!(Eq)))); + + impl Path { + pub fn parse_mod_style(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + global: option!(syn!(Colon2)) >> + segments: call!(Delimited::parse_separated_nonempty_with, + mod_style_path_segment) >> + (Path { + global: global.is_some(), + segments: segments, + leading_colon: global, + }) + } + } + } named!(mod_style_path_segment -> PathSegment, alt!( - map!(ident, Into::into) + map!(syn!(Ident), Into::into) | - map!(alt!( - keyword!("super") + alt!( + syn!(Super) => { Into::into } | - keyword!("self") + syn!(Self_) => { Into::into } | - keyword!("Self") - ), Into::into) - )); - - named!(type_binding -> TypeBinding, do_parse!( - id: ident >> - punct!("=") >> - ty: ty >> - (TypeBinding { - ident: id, - eq_token: tokens::Eq::default(), - ty: ty, - }) + syn!(CapSelf) => { Into::into } + ) )); - named!(pub poly_trait_ref -> PolyTraitRef, do_parse!( - bound_lifetimes: bound_lifetimes >> - trait_ref: path >> - parenthesized: option!(cond_reduce!( - trait_ref.segments.get(trait_ref.segments.len() - 1).item().parameters.is_empty(), - parenthesized_parameter_data - )) >> - ({ - let mut trait_ref = trait_ref; - if let Some(parenthesized) = parenthesized { - let len = trait_ref.segments.len(); - trait_ref.segments.get_mut(len - 1).item_mut().parameters = parenthesized; + impl Synom for TypeBinding { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + id: syn!(Ident) >> + eq: syn!(Eq) >> + ty: syn!(Ty) >> + (TypeBinding { + ident: id, + eq_token: eq, + ty: ty, + }) } - PolyTraitRef { - bound_lifetimes: bound_lifetimes, - trait_ref: trait_ref, + } + } + + impl Synom for PolyTraitRef { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + bound_lifetimes: option!(syn!(BoundLifetimes)) >> + trait_ref: syn!(Path) >> + parenthesized: option!(cond_reduce!( + trait_ref.segments.get(trait_ref.segments.len() - 1).item().parameters.is_empty(), + syn!(ParenthesizedParameterData) + )) >> + ({ + let mut trait_ref = trait_ref; + if let Some(parenthesized) = parenthesized { + let parenthesized = PathParameters::Parenthesized(parenthesized); + let len = trait_ref.segments.len(); + trait_ref.segments.get_mut(len - 1).item_mut().parameters = parenthesized; + } + PolyTraitRef { + bound_lifetimes: bound_lifetimes, + trait_ref: trait_ref, + } + }) } - }) - )); + } + } - named!(pub fn_arg -> BareFnArg, do_parse!( - name: option!(do_parse!( - name: ident >> - not!(punct!("::")) >> - punct!(":") >> - (name) - )) >> - ty: ty >> - (BareFnArg { - name: name.map(|t| (t, tokens::Colon::default())), - ty: ty, - }) - )); + impl Synom for BareFnArg { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + name: option!(do_parse!( + name: syn!(Ident) >> + not!(syn!(Colon2)) >> + colon: syn!(Colon) >> + (name, colon) + )) >> + ty: syn!(Ty) >> + (BareFnArg { + name: name, + ty: ty, + }) + } + } + } - named!(pub unsafety -> Unsafety, alt!( - keyword!("unsafe") => { |_| Unsafety::Unsafe(tokens::Unsafe::default()) } - | - epsilon!() => { |_| Unsafety::Normal } - )); + impl Synom for Unsafety { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + alt! { + input, + syn!(Unsafe) => { Unsafety::Unsafe } + | + epsilon!() => { |_| Unsafety::Normal } + } + } + } - named!(pub abi -> Abi, do_parse!( - keyword!("extern") >> - name: option!(string) >> - (Abi { - extern_token: tokens::Extern::default(), - kind: match name { - Some(name) => AbiKind::Named(name), - None => AbiKind::Default, - }, - }) - )); + impl Synom for Abi { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + do_parse! { + input, + extern_: syn!(Extern) >> + // TODO: this parses all literals, not just strings + name: option!(syn!(Lit)) >> + (Abi { + extern_token: extern_, + kind: match name { + Some(name) => AbiKind::Named(name), + None => AbiKind::Default, + }, + }) + } + } + } } #[cfg(feature = "printing")] diff --git a/synom/src/delimited.rs b/synom/src/delimited.rs index e27f1b682e..2cb0303b6b 100644 --- a/synom/src/delimited.rs +++ b/synom/src/delimited.rs @@ -43,16 +43,12 @@ impl Delimited { Iter { inner: self.inner.iter() } } - pub fn into_iter(self) -> IntoIter { - IntoIter { inner: self.inner.into_iter() } - } - pub fn items(&self) -> Items { Items { inner: self.inner.iter() } } pub fn push(&mut self, token: Element) { - assert!(self.len() == 0 || self.trailing_delim()); + assert!(self.is_empty() || self.trailing_delim()); match token { Element::Delimited(t, d) => self.inner.push((t, Some(d))), Element::End(t) => self.inner.push((t, None)), @@ -76,7 +72,7 @@ impl Delimited { } pub fn push_default(&mut self, token: T) where D: Default { - if self.len() == 0 { + if self.is_empty() { self.inner.push((token, None)); } else { self.push_next(token, D::default()); @@ -131,7 +127,25 @@ impl FromIterator> for Delimited { Element::End(a) => ret.inner.push((a, None)), } } - return ret + ret + } +} + +impl<'a, T, D> IntoIterator for &'a Delimited { + type Item = Element<&'a T, &'a D>; + type IntoIter = Iter<'a, T, D>; + + fn into_iter(self) -> Iter<'a, T, D> { + >::iter(self) + } +} + +impl IntoIterator for Delimited { + type Item = Element; + type IntoIter = IntoIter; + + fn into_iter(self) -> IntoIter { + IntoIter { inner: self.inner.into_iter() } } } @@ -236,22 +250,53 @@ mod parsing { pub fn parse_separated(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { - Self::parse(input, false) + Self::parse(input, T::parse, false) + } + + pub fn parse_separated_nonempty(input: &[TokenTree]) + -> IResult<&[TokenTree], Self> + { + Self::parse_separated_nonempty_with(input, T::parse) } pub fn parse_terminated(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { - Self::parse(input, true) + Self::parse_terminated_with(input, T::parse) + } + } + + impl Delimited + where D: Synom, + { + pub fn parse_separated_nonempty_with( + input: &[TokenTree], + parse: fn(&[TokenTree]) -> IResult<&[TokenTree], T>) + -> IResult<&[TokenTree], Self> + { + match Self::parse(input, parse, false) { + IResult::Done(_, ref b) if b.is_empty() => IResult::Error, + other => other, + } + } + + pub fn parse_terminated_with( + input: &[TokenTree], + parse: fn(&[TokenTree]) -> IResult<&[TokenTree], T>) + -> IResult<&[TokenTree], Self> + { + Self::parse(input, parse, true) } - fn parse(mut input: &[TokenTree], terminated: bool) + fn parse(mut input: &[TokenTree], + parse: fn(&[TokenTree]) -> IResult<&[TokenTree], T>, + terminated: bool) -> IResult<&[TokenTree], Self> { let mut res = Delimited::new(); // get the first element - match T::parse(input) { + match parse(input) { IResult::Error => IResult::Done(input, res), IResult::Done(i, o) => { if i.len() == input.len() { @@ -267,7 +312,7 @@ mod parsing { } // get the element next - if let IResult::Done(i3, o3) = T::parse(i2) { + if let IResult::Done(i3, o3) = parse(i2) { if i3.len() == i2.len() { break; } diff --git a/synom/src/helper.rs b/synom/src/helper.rs index 3b64ac5028..b0bc7cf04c 100644 --- a/synom/src/helper.rs +++ b/synom/src/helper.rs @@ -1,62 +1,3 @@ -use proc_macro2::{TokenTree, TokenKind}; - -use IResult; -use span::Span; - -/// Parse a keyword like "fn" or "struct". -/// -/// See also `punct!` for parsing punctuation, which are subtly different from -/// keywords. -/// -/// - **Syntax:** `keyword!("...")` -/// - **Output:** `&str` -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// -/// use synom::IResult; -/// -/// // Parse zero or more "bang" keywords. -/// named!(many_bangs -> Vec<&str>, -/// terminated!( -/// many0!(keyword!("bang")), -/// punct!(";") -/// ) -/// ); -/// -/// fn main() { -/// let input = "bang bang bang;"; -/// let parsed = many_bangs(input).expect("bangs"); -/// assert_eq!(parsed, ["bang", "bang", "bang"]); -/// -/// let input = "bangbang;"; -/// let err = many_bangs(input); -/// assert_eq!(err, IResult::Error); -/// } -/// ``` -#[macro_export] -macro_rules! keyword { - ($i:expr, $keyword:expr) => { - $crate::helper::keyword($i, $keyword) - }; -} - -// Not public API. -#[doc(hidden)] -pub fn keyword<'a>(input: &'a [TokenTree], token: &'static str) -> IResult<&'a [TokenTree], Span> { - match input.first() { - Some(&TokenTree{ kind: TokenKind::Word(ref symbol), span }) => { - if symbol.as_str() == token { - IResult::Done(&input[1..], Span(span)) - } else { - IResult::Error - } - } - _ => IResult::Error, - } -} - /// Turn a failed parse into `None` and a successful parse into `Some`. /// /// - **Syntax:** `option!(THING)` @@ -66,17 +7,11 @@ pub fn keyword<'a>(input: &'a [TokenTree], token: &'static str) -> IResult<&'a [ /// extern crate syn; /// #[macro_use] extern crate synom; /// -/// named!(maybe_bang -> Option<&str>, option!(punct!("!"))); +/// use syn::tokens::Bang; /// -/// fn main() { -/// let input = "!"; -/// let parsed = maybe_bang(input).expect("maybe bang"); -/// assert_eq!(parsed, Some("!")); +/// named!(maybe_bang -> Option, option!(syn!(Bang))); /// -/// let input = ""; -/// let parsed = maybe_bang(input).expect("maybe bang"); -/// assert_eq!(parsed, None); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! option { @@ -106,30 +41,21 @@ macro_rules! option { /// #[macro_use] extern crate synom; /// /// use syn::{Lifetime, Ty}; -/// use syn::parse::{lifetime, ty}; +/// use syn::delimited::Delimited; +/// use syn::tokens::*; /// /// named!(bound_lifetimes -> (Vec, Ty), tuple!( /// opt_vec!(do_parse!( -/// keyword!("for") >> -/// punct!("<") >> -/// lifetimes: terminated_list!(punct!(","), lifetime) >> -/// punct!(">") >> +/// syn!(For) >> +/// syn!(Lt) >> +/// lifetimes: call!(Delimited::::parse_terminated) >> +/// syn!(Gt) >> /// (lifetimes.into_vec()) /// )), -/// ty +/// syn!(Ty) /// )); /// -/// fn main() { -/// let input = "for<'a, 'b> fn(&'a A) -> &'b B"; -/// let parsed = bound_lifetimes(input).expect("bound lifetimes"); -/// assert_eq!(parsed.0, [Lifetime::new("'a"), Lifetime::new("'b")]); -/// println!("{:?}", parsed); -/// -/// let input = "From"; -/// let parsed = bound_lifetimes(input).expect("bound lifetimes"); -/// assert!(parsed.0.is_empty()); -/// println!("{:?}", parsed); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! opt_vec { @@ -153,23 +79,15 @@ macro_rules! opt_vec { /// #[macro_use] extern crate synom; /// /// use syn::Mutability; +/// use synom::tokens::Mut; /// /// named!(mutability -> Mutability, alt!( -/// keyword!("mut") => { |_| Mutability::Mutable(Default::default()) } +/// syn!(Mut) => { Mutability::Mutable } /// | /// epsilon!() => { |_| Mutability::Immutable } /// )); /// -/// fn main() { -/// let input = "mut"; -/// let parsed = mutability(input).expect("mutability"); -/// assert_eq!(parsed, Mutability::Mutable(Default::default())); -/// -/// let input = ""; -/// let parsed = mutability(input).expect("mutability"); -/// assert_eq!(parsed, Mutability::Immutable); -/// } -/// ``` +/// # fn main() {} #[macro_export] macro_rules! epsilon { ($i:expr,) => { @@ -190,11 +108,11 @@ macro_rules! epsilon { /// #[macro_use] extern crate synom; /// /// use syn::{Expr, ExprCall}; -/// use syn::parse::expr; +/// use syn::tokens::RArrow; /// /// named!(expr_with_arrow_call -> Expr, do_parse!( -/// mut e: expr >> -/// many0!(tap!(arg: tuple!(punct!("=>"), expr) => { +/// mut e: syn!(Expr) >> +/// many0!(tap!(arg: tuple!(syn!(RArrow), syn!(Expr)) => { /// e = Expr { /// node: ExprCall { /// func: Box::new(e), @@ -207,13 +125,7 @@ macro_rules! epsilon { /// (e) /// )); /// -/// fn main() { -/// let input = "something => argument1 => argument2"; -/// -/// let parsed = expr_with_arrow_call(input).expect("expr with arrow call"); -/// -/// println!("{:?}", parsed); -/// } +/// # fn main() {} /// ``` #[doc(hidden)] #[macro_export] @@ -233,3 +145,88 @@ macro_rules! tap { tap!($i, $name: call!($f) => $e); }; } + +/// Parse a type through the `Synom` trait. +/// +/// This is a convenience macro used to invoke the `Synom::parse` method for a +/// type, you'll find this in quite a few parsers. This is also the primary way +/// to parse punctuation. +/// +/// - **Syntax:** `syn!(TYPE)` +/// - **Output:** `TYPE` +/// +/// ```rust +/// extern crate syn; +/// #[macro_use] extern crate synom; +/// +/// use syn::Expr; +/// use synom::tokens::Dot; +/// +/// named!(expression -> Expr, syn!(Expr)); +/// +/// named!(expression_dot -> (Expr, Dot), tuple!(syn!(Expr), syn!(Dot))); +/// +/// # fn main() {} +/// ``` +#[macro_export] +macro_rules! syn { + ($i:expr, $t:ty) => { + call!($i, <$t as $crate::Synom>::parse) + }; +} + +/// Parse a parenthesized-surrounded subtree. +/// +/// This macro will invoke a sub-parser inside of all tokens contained in +/// parenthesis. The sub-parser is required to consume all tokens within the +/// parens or else this parser will return an error. +/// +/// - **Syntax:** `parens!(SUBPARSER)` +/// - **Output:** `(SUBPARSER_RET, Paren)` +/// +/// ```rust +/// extern crate syn; +/// #[macro_use] extern crate synom; +/// +/// use syn::Expr; +/// use synom::tokens::Paren; +/// +/// named!(expr_paren -> (Expr, Paren), parens!(syn!(Expr))); +/// +/// # fn main() {} +/// ``` +#[macro_export] +macro_rules! parens { + ($i:expr, $submac:ident!( $($args:tt)* )) => { + $crate::tokens::Paren::parse($i, |i| $submac!(i, $($args)*)) + }; + + ($i:expr, $f:expr) => { + parens!($i, call!($f)); + }; +} + + +/// Same as the `parens` macro, but for brackets. +#[macro_export] +macro_rules! brackets { + ($i:expr, $submac:ident!( $($args:tt)* )) => { + $crate::tokens::Bracket::parse($i, |i| $submac!(i, $($args)*)) + }; + + ($i:expr, $f:expr) => { + brackets!($i, call!($f)); + }; +} + +/// Same as the `parens` macro, but for braces. +#[macro_export] +macro_rules! braces { + ($i:expr, $submac:ident!( $($args:tt)* )) => { + $crate::tokens::Brace::parse($i, |i| $submac!(i, $($args)*)) + }; + + ($i:expr, $f:expr) => { + braces!($i, call!($f)); + }; +} diff --git a/synom/src/lib.rs b/synom/src/lib.rs index a53688834a..c5e364b6f3 100644 --- a/synom/src/lib.rs +++ b/synom/src/lib.rs @@ -28,7 +28,13 @@ extern crate proc_macro2; extern crate quote; #[doc(hidden)] -pub use proc_macro2::TokenTree; +pub use proc_macro2::{TokenTree, TokenStream}; + +use std::convert::From; +use std::error::Error; +use std::fmt; + +use proc_macro2::LexError; #[cfg(feature = "parsing")] #[doc(hidden)] @@ -51,29 +57,6 @@ pub enum IResult { impl<'a, O> IResult<&'a [TokenTree], O> { /// Unwraps the result, asserting the the parse is complete. Panics with a /// message based on the given string if the parse failed or is incomplete. - /// - /// ```rust - /// extern crate syn; - /// #[macro_use] extern crate synom; - /// - /// use syn::Ty; - /// use syn::parse::ty; - /// use synom::delimited::Delimited; - /// - /// // One or more Rust types separated by commas. - /// named!(comma_separated_types -> Delimited, - /// separated_nonempty_list!(punct!(","), ty) - /// ); - /// - /// fn main() { - /// let input = "&str, Map, String"; - /// - /// let parsed = comma_separated_types(input).expect("comma-separated types"); - /// - /// assert_eq!(parsed.len(), 3); - /// println!("{:?}", parsed); - /// } - /// ``` pub fn expect(self, name: &str) -> O { match self { IResult::Done(rest, o) => { @@ -90,6 +73,69 @@ impl<'a, O> IResult<&'a [TokenTree], O> { pub trait Synom: Sized { fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self>; + + fn description() -> Option<&'static str> { + None + } + + fn parse_all(input: TokenStream) -> Result { + let tokens = input.into_iter().collect::>(); + let err = match Self::parse(&tokens) { + IResult::Done(rest, t) => { + if rest.is_empty() { + return Ok(t) + } else if rest.len() == tokens.len() { + // parsed nothing + "failed to parse" + } else { + "unparsed tokens after" + } + } + IResult::Error => "failed to parse" + }; + match Self::description() { + Some(s) => Err(ParseError(format!("{} {}", err, s))), + None => Err(ParseError(err.to_string())), + } + } + + fn parse_str_all(input: &str) -> Result { + Self::parse_all(input.parse()?) + } + + fn parse_all_unwrap(input: TokenStream) -> Self { + // TODO: eventually try to provide super nice error messages here as + // this is what most users will hit. Hopefully the compiler will give us + // an interface one day to give an extra-good error message here. + Self::parse_all(input).unwrap() + } +} + +#[derive(Debug)] +pub struct ParseError(String); + +impl Error for ParseError { + fn description(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + ::fmt(&self.0, f) + } +} + +impl From for ParseError { + fn from(_: LexError) -> ParseError { + ParseError("error while lexing input string".to_owned()) + } +} + +impl Synom for TokenStream { + fn parse(input: &[TokenTree]) -> IResult<&[TokenTree], Self> { + IResult::Done(&[], input.iter().cloned().collect()) + } } /// Define a function from a parser combination. @@ -100,11 +146,11 @@ pub trait Synom: Sized { /// # extern crate syn; /// # #[macro_use] extern crate synom; /// # use syn::Ty; -/// # use syn::parse::ty; /// # use synom::delimited::Delimited; +/// # use synom::tokens::Comma; /// // One or more Rust types separated by commas. -/// named!(pub comma_separated_types -> Delimited, -/// separated_nonempty_list!(punct!(","), ty) +/// named!(pub comma_separated_types -> Delimited, +/// call!(Delimited::parse_separated_nonempty) /// ); /// # fn main() {} /// ``` @@ -127,41 +173,8 @@ macro_rules! named { /// /// - **Syntax:** `call!(FUNCTION, ARGS...)` /// -/// where the signature of the function is `fn(&str, ARGS...) -> IResult<&str, T>` +/// where the signature of the function is `fn(&[U], ARGS...) -> IResult<&[U], T>` /// - **Output:** `T`, the result of invoking the function `FUNCTION` -/// -/// ```rust -/// #[macro_use] extern crate synom; -/// -/// use synom::IResult; -/// -/// // Parses any string up to but not including the given character, returning -/// // the content up to the given character. The given character is required to -/// // be present in the input string. -/// fn skip_until(input: &str, ch: char) -> IResult<&str, &str> { -/// if let Some(pos) = input.find(ch) { -/// IResult::Done(&input[pos..], &input[..pos]) -/// } else { -/// IResult::Error -/// } -/// } -/// -/// // Parses any string surrounded by tilde characters '~'. Returns the content -/// // between the tilde characters. -/// named!(surrounded_by_tilde -> &str, delimited!( -/// punct!("~"), -/// call!(skip_until, '~'), -/// punct!("~") -/// )); -/// -/// fn main() { -/// let input = "~ abc def ~"; -/// -/// let inner = surrounded_by_tilde(input).expect("surrounded by tilde"); -/// -/// println!("{:?}", inner); -/// } -/// ``` #[macro_export] macro_rules! call { ($i:expr, $fun:expr $(, $args:expr)*) => { @@ -179,7 +192,6 @@ macro_rules! call { /// #[macro_use] extern crate synom; /// /// use syn::{Item, Ident}; -/// use syn::parse::item; /// /// fn get_item_ident(item: Item) -> Ident { /// item.ident @@ -187,21 +199,15 @@ macro_rules! call { /// /// // Parses an item and returns the name (identifier) of the item only. /// named!(item_ident -> Ident, -/// map!(item, get_item_ident) +/// map!(syn!(Item), get_item_ident) /// ); /// /// // Or equivalently: /// named!(item_ident2 -> Ident, -/// map!(item, |i: Item| i.ident) +/// map!(syn!(Item), |i: Item| i.ident) /// ); /// -/// fn main() { -/// let input = "fn foo() {}"; -/// -/// let parsed = item_ident(input).expect("item"); -/// -/// assert_eq!(parsed, "foo"); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! map { @@ -224,30 +230,6 @@ macro_rules! map { /// /// - **Syntax:** `not!(THING)` /// - **Output:** `()` -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// use synom::IResult; -/// -/// // Parses a shebang line like `#!/bin/bash` and returns the part after `#!`. -/// // Note that a line starting with `#![` is an inner attribute, not a -/// // shebang. -/// named!(shebang -> &str, preceded!( -/// tuple!(tag!("#!"), not!(tag!("["))), -/// take_until!("\n") -/// )); -/// -/// fn main() { -/// let bin_bash = "#!/bin/bash\n"; -/// let parsed = shebang(bin_bash).expect("shebang"); -/// assert_eq!(parsed, "/bin/bash"); -/// -/// let inner_attr = "#![feature(specialization)]\n"; -/// let err = shebang(inner_attr); -/// assert_eq!(err, IResult::Error); -/// } -/// ``` #[macro_export] macro_rules! not { ($i:expr, $submac:ident!( $($args:tt)* )) => { @@ -264,63 +246,6 @@ macro_rules! not { /// /// - **Syntax:** `cond!(CONDITION, THING)` /// - **Output:** `Some(THING)` if the condition is true, else `None` -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// -/// named!(boolean -> bool, alt!( -/// keyword!("true") => { |_| true } -/// | -/// keyword!("false") => { |_| false } -/// )); -/// -/// // Parses a tuple of booleans like `(true, false, false)`, possibly with a -/// // dotdot indicating omitted values like `(true, true, .., true)`. Returns -/// // separate vectors for the booleans before and after the dotdot. The second -/// // vector is None if there was no dotdot. -/// named!(bools_with_dotdot -> (Vec, Option>), do_parse!( -/// punct!("(") >> -/// before: separated_list!(punct!(","), boolean) >> -/// after: option!(do_parse!( -/// // Only allow comma if there are elements before dotdot, i.e. cannot -/// // be `(, .., true)`. -/// cond!(!before.is_empty(), punct!(",")) >> -/// punct!("..") >> -/// after: many0!(preceded!(punct!(","), boolean)) >> -/// // Only allow trailing comma if there are elements after dotdot, -/// // i.e. cannot be `(true, .., )`. -/// cond!(!after.is_empty(), option!(punct!(","))) >> -/// (after) -/// )) >> -/// // Allow trailing comma if there is no dotdot but there are elements. -/// cond!(!before.is_empty() && after.is_none(), option!(punct!(","))) >> -/// punct!(")") >> -/// (before.into_vec(), after) -/// )); -/// -/// fn main() { -/// let input = "(true, false, false)"; -/// let parsed = bools_with_dotdot(input).expect("bools with dotdot"); -/// assert_eq!(parsed, (vec![true, false, false], None)); -/// -/// let input = "(true, true, .., true)"; -/// let parsed = bools_with_dotdot(input).expect("bools with dotdot"); -/// assert_eq!(parsed, (vec![true, true], Some(vec![true]))); -/// -/// let input = "(.., true)"; -/// let parsed = bools_with_dotdot(input).expect("bools with dotdot"); -/// assert_eq!(parsed, (vec![], Some(vec![true]))); -/// -/// let input = "(true, true, ..)"; -/// let parsed = bools_with_dotdot(input).expect("bools with dotdot"); -/// assert_eq!(parsed, (vec![true, true], Some(vec![]))); -/// -/// let input = "(..)"; -/// let parsed = bools_with_dotdot(input).expect("bools with dotdot"); -/// assert_eq!(parsed, (vec![], Some(vec![]))); -/// } -/// ``` #[macro_export] macro_rules! cond { ($i:expr, $cond:expr, $submac:ident!( $($args:tt)* )) => { @@ -345,62 +270,6 @@ macro_rules! cond { /// /// - **Syntax:** `cond_reduce!(CONDITION, THING)` /// - **Output:** `THING` -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// -/// #[derive(Debug, PartialEq)] -/// struct VariadicBools { -/// data: Vec, -/// variadic: bool, -/// } -/// -/// named!(boolean -> bool, alt!( -/// keyword!("true") => { |_| true } -/// | -/// keyword!("false") => { |_| false } -/// )); -/// -/// // Parse one or more comma-separated booleans, possibly ending in "..." to -/// // indicate there may be more. -/// named!(variadic_bools -> VariadicBools, do_parse!( -/// data: separated_nonempty_list!(punct!(","), boolean) >> -/// trailing_comma: option!(punct!(",")) >> -/// // Only allow "..." if there is a comma after the last boolean. Using -/// // `cond_reduce!` is more convenient here than using `cond!`. The -/// // alternatives are: -/// // -/// // - `cond!(c, option!(p))` or `option!(cond!(c, p))` -/// // Gives `Some(Some("..."))` for variadic and `Some(None)` or `None` -/// // which both mean not variadic. -/// // - `cond_reduce!(c, option!(p))` -/// // Incorrect; would fail to parse if there is no trailing comma. -/// // - `option!(cond_reduce!(c, p))` -/// // Gives `Some("...")` for variadic and `None` otherwise. Perfect! -/// variadic: option!(cond_reduce!(trailing_comma.is_some(), punct!("..."))) >> -/// (VariadicBools { -/// data: data.into_vec(), -/// variadic: variadic.is_some(), -/// }) -/// )); -/// -/// fn main() { -/// let input = "true, true"; -/// let parsed = variadic_bools(input).expect("variadic bools"); -/// assert_eq!(parsed, VariadicBools { -/// data: vec![true, true], -/// variadic: false, -/// }); -/// -/// let input = "true, ..."; -/// let parsed = variadic_bools(input).expect("variadic bools"); -/// assert_eq!(parsed, VariadicBools { -/// data: vec![true], -/// variadic: true, -/// }); -/// } -/// ``` #[macro_export] macro_rules! cond_reduce { ($i:expr, $cond:expr, $submac:ident!( $($args:tt)* )) => { @@ -416,53 +285,6 @@ macro_rules! cond_reduce { }; } -/// Parse two things, returning the value of the second. -/// -/// - **Syntax:** `preceded!(BEFORE, THING)` -/// - **Output:** `THING` -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// -/// use syn::Expr; -/// use syn::parse::expr; -/// -/// // An expression preceded by ##. -/// named!(pound_pound_expr -> Expr, -/// preceded!(punct!("##"), expr) -/// ); -/// -/// fn main() { -/// let input = "## 1 + 1"; -/// -/// let parsed = pound_pound_expr(input).expect("pound pound expr"); -/// -/// println!("{:?}", parsed); -/// } -/// ``` -#[macro_export] -macro_rules! preceded { - ($i:expr, $submac:ident!( $($args:tt)* ), $submac2:ident!( $($args2:tt)* )) => { - match tuple!($i, $submac!($($args)*), $submac2!($($args2)*)) { - $crate::IResult::Done(remaining, (_, o)) => $crate::IResult::Done(remaining, o), - $crate::IResult::Error => $crate::IResult::Error, - } - }; - - ($i:expr, $submac:ident!( $($args:tt)* ), $g:expr) => { - preceded!($i, $submac!($($args)*), call!($g)) - }; - - ($i:expr, $f:expr, $submac:ident!( $($args:tt)* )) => { - preceded!($i, call!($f), $submac!($($args)*)) - }; - - ($i:expr, $f:expr, $g:expr) => { - preceded!($i, call!($f), call!($g)) - }; -} - /// Parse two things, returning the value of the first. /// /// - **Syntax:** `terminated!(THING, AFTER)` @@ -473,20 +295,14 @@ macro_rules! preceded { /// #[macro_use] extern crate synom; /// /// use syn::Expr; -/// use syn::parse::expr; +/// use synom::tokens::Pound; /// /// // An expression terminated by ##. /// named!(expr_pound_pound -> Expr, -/// terminated!(expr, punct!("##")) +/// terminated!(syn!(Expr), tuple!(syn!(Pound), syn!(Pound))) /// ); /// -/// fn main() { -/// let input = "1 + 1 ##"; -/// -/// let parsed = expr_pound_pound(input).expect("expr pound pound"); -/// -/// println!("{:?}", parsed); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! terminated { @@ -517,31 +333,19 @@ macro_rules! terminated { /// /// You may also be looking for: /// -/// - `separated_list!` - zero or more values with separator -/// - `separated_nonempty_list!` - one or more values -/// - `terminated_list!` - zero or more, allows trailing separator +/// - `call!(Delimited::parse_separated)` - zero or more values with separator +/// - `call!(Delimited::parse_separated_nonempty)` - one or more values +/// - `call!(Delimited::parse_terminated)` - zero or more, allows trailing separator /// /// ```rust /// extern crate syn; /// #[macro_use] extern crate synom; /// /// use syn::Item; -/// use syn::parse::item; -/// -/// named!(items -> Vec, many0!(item)); /// -/// fn main() { -/// let input = " -/// fn a() {} -/// fn b() {} -/// "; +/// named!(items -> Vec, many0!(syn!(Item))); /// -/// let parsed = items(input).expect("items"); -/// -/// assert_eq!(parsed.len(), 2); -/// println!("{:?}", parsed); -/// } -/// ``` +/// # fn main() {} #[macro_export] macro_rules! many0 { ($i:expr, $submac:ident!( $($args:tt)* )) => {{ @@ -622,26 +426,15 @@ pub fn many0<'a, T>(mut input: &'a [TokenTree], /// extern crate syn; /// #[macro_use] extern crate synom; /// -/// use syn::Expr; -/// use syn::parse::{ident, expr}; +/// use syn::{Expr, Ident}; /// use synom::IResult; /// /// // Parse an expression that begins with an identifier. -/// named!(ident_expr -> Expr, -/// preceded!(peek!(ident), expr) +/// named!(ident_expr -> (Ident, Expr), +/// tuple!(peek!(syn!(Ident)), syn!(Expr)) /// ); /// -/// fn main() { -/// // begins with an identifier -/// let input = "banana + 1"; -/// let parsed = ident_expr(input).expect("ident"); -/// println!("{:?}", parsed); -/// -/// // does not begin with an identifier -/// let input = "1 + banana"; -/// let err = ident_expr(input); -/// assert_eq!(err, IResult::Error); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! peek { @@ -657,110 +450,6 @@ macro_rules! peek { }; } -/// Parse the part of the input up to but not including the given string. Fail -/// to parse if the given string is not present in the input. -/// -/// - **Syntax:** `take_until!("...")` -/// - **Output:** `&str` -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// use synom::IResult; -/// -/// // Parse a single line doc comment: /// ... -/// named!(single_line_doc -> &str, -/// preceded!(punct!("///"), take_until!("\n")) -/// ); -/// -/// fn main() { -/// let comment = "/// comment\n"; -/// let parsed = single_line_doc(comment).expect("single line doc comment"); -/// assert_eq!(parsed, " comment"); -/// } -/// ``` -#[macro_export] -macro_rules! take_until { - ($i:expr, $substr:expr) => {{ - if $substr.len() > $i.len() { - $crate::IResult::Error - } else { - let substr_vec: Vec = $substr.chars().collect(); - let mut window: Vec = vec![]; - let mut offset = $i.len(); - let mut parsed = false; - for (o, c) in $i.char_indices() { - window.push(c); - if window.len() > substr_vec.len() { - window.remove(0); - } - if window == substr_vec { - parsed = true; - window.pop(); - let window_len: usize = window.iter() - .map(|x| x.len_utf8()) - .fold(0, |x, y| x + y); - offset = o - window_len; - break; - } - } - if parsed { - $crate::IResult::Done(&$i[offset..], &$i[..offset]) - } else { - $crate::IResult::Error - } - } - }}; -} - -/// Parse the given string from exactly the current position in the input. You -/// almost always want `punct!` or `keyword!` instead of this. -/// -/// The `tag!` parser is equivalent to `punct!` but does not ignore leading -/// whitespace. Both `punct!` and `keyword!` skip over leading whitespace. See -/// an explanation of synom's whitespace handling strategy in the top-level -/// crate documentation. -/// -/// - **Syntax:** `tag!("...")` -/// - **Output:** `"..."` -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// -/// use syn::Lit; -/// use syn::parse::string; -/// use synom::IResult; -/// -/// // Parse a proposed syntax for an owned string literal: "abc"s -/// named!(owned_string -> String, -/// map!( -/// terminated!(string, tag!("s")), -/// |lit: Lit| lit.to_string() -/// ) -/// ); -/// -/// fn main() { -/// let input = r#" "abc"s "#; -/// let parsed = owned_string(input).expect("owned string literal"); -/// println!("{:?}", parsed); -/// -/// let input = r#" "abc" s "#; -/// let err = owned_string(input); -/// assert_eq!(err, IResult::Error); -/// } -/// ``` -#[macro_export] -macro_rules! tag { - ($i:expr, $tag:expr) => { - if $i.starts_with($tag) { - $crate::IResult::Done(&$i[$tag.len()..], &$i[..$tag.len()]) - } else { - $crate::IResult::Error - } - }; -} - /// Pattern-match the result of a parser to select which other parser to run. /// /// - **Syntax:** `switch!(TARGET, PAT1 => THEN1 | PAT2 => THEN2 | ...)` @@ -771,7 +460,7 @@ macro_rules! tag { /// #[macro_use] extern crate synom; /// /// use syn::{Ident, Ty}; -/// use syn::parse::{ident, ty}; +/// use synom::tokens::*; /// /// #[derive(Debug)] /// enum UnitType { @@ -786,19 +475,23 @@ macro_rules! tag { /// /// // Parse a unit struct or enum: either `struct S;` or `enum E { V }`. /// named!(unit_type -> UnitType, do_parse!( -/// which: alt!(keyword!("struct") | keyword!("enum")) >> -/// id: ident >> +/// which: alt!( +/// syn!(Struct) => { |_| 0 } +/// | +/// syn!(Enum) => { |_| 1 } +/// ) >> +/// id: syn!(Ident) >> /// item: switch!(value!(which), -/// "struct" => map!( -/// punct!(";"), +/// 0 => map!( +/// syn!(Semi), /// move |_| UnitType::Struct { /// name: id, /// } /// ) /// | -/// "enum" => map!( -/// delimited!(punct!("{"), ident, punct!("}")), -/// move |variant| UnitType::Enum { +/// 1 => map!( +/// braces!(syn!(Ident)), +/// move |(variant, _)| UnitType::Enum { /// name: id, /// variant: variant, /// } @@ -807,15 +500,7 @@ macro_rules! tag { /// (item) /// )); /// -/// fn main() { -/// let input = "struct S;"; -/// let parsed = unit_type(input).expect("unit struct or enum"); -/// println!("{:?}", parsed); -/// -/// let input = "enum E { V }"; -/// let parsed = unit_type(input).expect("unit struct or enum"); -/// println!("{:?}", parsed); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! switch { @@ -832,6 +517,7 @@ macro_rules! switch { }; } + /// Produce the given value without parsing anything. Useful as an argument to /// `switch!`. /// @@ -843,7 +529,7 @@ macro_rules! switch { /// #[macro_use] extern crate synom; /// /// use syn::{Ident, Ty}; -/// use syn::parse::{ident, ty}; +/// use synom::tokens::*; /// /// #[derive(Debug)] /// enum UnitType { @@ -858,19 +544,23 @@ macro_rules! switch { /// /// // Parse a unit struct or enum: either `struct S;` or `enum E { V }`. /// named!(unit_type -> UnitType, do_parse!( -/// which: alt!(keyword!("struct") | keyword!("enum")) >> -/// id: ident >> +/// which: alt!( +/// syn!(Struct) => { |_| 0 } +/// | +/// syn!(Enum) => { |_| 1 } +/// ) >> +/// id: syn!(Ident) >> /// item: switch!(value!(which), -/// "struct" => map!( -/// punct!(";"), +/// 0 => map!( +/// syn!(Semi), /// move |_| UnitType::Struct { /// name: id, /// } /// ) /// | -/// "enum" => map!( -/// delimited!(punct!("{"), ident, punct!("}")), -/// move |variant| UnitType::Enum { +/// 1 => map!( +/// braces!(syn!(Ident)), +/// move |(variant, _)| UnitType::Enum { /// name: id, /// variant: variant, /// } @@ -879,15 +569,7 @@ macro_rules! switch { /// (item) /// )); /// -/// fn main() { -/// let input = "struct S;"; -/// let parsed = unit_type(input).expect("unit struct or enum"); -/// println!("{:?}", parsed); -/// -/// let input = "enum E { V }"; -/// let parsed = unit_type(input).expect("unit struct or enum"); -/// println!("{:?}", parsed); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! value { @@ -896,129 +578,6 @@ macro_rules! value { }; } -/// Value surrounded by a pair of delimiters. -/// -/// - **Syntax:** `delimited!(OPEN, THING, CLOSE)` -/// - **Output:** `THING` -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// -/// use syn::Expr; -/// use syn::parse::expr; -/// -/// // An expression surrounded by [[ ... ]]. -/// named!(double_bracket_expr -> Expr, -/// delimited!(punct!("[["), expr, punct!("]]")) -/// ); -/// -/// fn main() { -/// let input = "[[ 1 + 1 ]]"; -/// -/// let parsed = double_bracket_expr(input).expect("double bracket expr"); -/// -/// println!("{:?}", parsed); -/// } -/// ``` -#[macro_export] -macro_rules! delimited { - ($i:expr, $submac:ident!( $($args:tt)* ), $($rest:tt)+) => { - match tuple_parser!($i, (), $submac!($($args)*), $($rest)*) { - $crate::IResult::Error => $crate::IResult::Error, - $crate::IResult::Done(i1, (_, o, _)) => $crate::IResult::Done(i1, o) - } - }; - - ($i:expr, $f:expr, $($rest:tt)+) => { - delimited!($i, call!($f), $($rest)*) - }; -} - -/// One or more values separated by some separator. Does not allow a trailing -/// separator. -/// -/// - **Syntax:** `separated_nonempty_list!(SEPARATOR, THING)` -/// - **Output:** `Vec` -/// -/// You may also be looking for: -/// -/// - `separated_list!` - one or more values -/// - `terminated_list!` - zero or more, allows trailing separator -/// - `many0!` - zero or more, no separator -/// -/// ```rust -/// extern crate syn; -/// #[macro_use] extern crate synom; -/// -/// use syn::Ty; -/// use syn::parse::ty; -/// use synom::delimited::Delimited; -/// -/// // One or more Rust types separated by commas. -/// named!(comma_separated_types -> Delimited, -/// separated_nonempty_list!(punct!(","), ty) -/// ); -/// -/// fn main() { -/// let input = "&str, Map, String"; -/// -/// let parsed = comma_separated_types(input).expect("comma-separated types"); -/// -/// assert_eq!(parsed.len(), 3); -/// println!("{:?}", parsed); -/// } -/// ``` -#[macro_export] -macro_rules! separated_nonempty_list { - ($i:expr, $sep:ident!( $($args:tt)* ), $submac:ident!( $($args2:tt)* )) => {{ - let mut res = $crate::delimited::Delimited::new(); - let mut input = $i; - - // get the first element - match $submac!(input, $($args2)*) { - $crate::IResult::Error => $crate::IResult::Error, - $crate::IResult::Done(i, o) => { - if i.len() == input.len() { - $crate::IResult::Error - } else { - input = i; - res.push_first(o); - - while let $crate::IResult::Done(i2, s) = $sep!(input, $($args)*) { - if i2.len() == input.len() { - break; - } - - if let $crate::IResult::Done(i3, o3) = $submac!(i2, $($args2)*) { - res.push_next(o3, s); - if i3.len() == i2.len() { - break; - } - input = i3; - } else { - break; - } - } - $crate::IResult::Done(input, res) - } - } - } - }}; - - ($i:expr, $submac:ident!( $($args:tt)* ), $g:expr) => { - separated_nonempty_list!($i, $submac!($($args)*), call!($g)) - }; - - ($i:expr, $f:expr, $submac:ident!( $($args:tt)* )) => { - separated_nonempty_list!($i, call!($f), $submac!($($args)*)) - }; - - ($i:expr, $f:expr, $g:expr) => { - separated_nonempty_list!($i, call!($f), call!($g)) - }; -} - /// Run a series of parsers and produce all of the results in a tuple. /// /// - **Syntax:** `tuple!(A, B, C, ...)` @@ -1029,17 +588,10 @@ macro_rules! separated_nonempty_list { /// #[macro_use] extern crate synom; /// /// use syn::Ty; -/// use syn::parse::ty; -/// -/// named!(two_types -> (Ty, Ty), tuple!(ty, ty)); /// -/// fn main() { -/// let input = "&str Map"; +/// named!(two_types -> (Ty, Ty), tuple!(syn!(Ty), syn!(Ty))); /// -/// let parsed = two_types(input).expect("two types"); -/// -/// println!("{:?}", parsed); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! tuple { @@ -1105,25 +657,17 @@ macro_rules! tuple_parser { /// #[macro_use] extern crate synom; /// /// use syn::Ident; -/// use syn::parse::ident; +/// use synom::tokens::Bang; /// /// named!(ident_or_bang -> Ident, /// alt!( -/// ident +/// syn!(Ident) /// | -/// punct!("!") => { |_| "BANG".into() } +/// syn!(Bang) => { |_| "BANG".into() } /// ) /// ); /// -/// fn main() { -/// let input = "foo"; -/// let parsed = ident_or_bang(input).expect("identifier or `!`"); -/// assert_eq!(parsed, "foo"); -/// -/// let input = "!"; -/// let parsed = ident_or_bang(input).expect("identifier or `!`"); -/// assert_eq!(parsed, "BANG"); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! alt { @@ -1181,24 +725,21 @@ macro_rules! alt { /// ```rust /// extern crate syn; /// #[macro_use] extern crate synom; +/// extern crate proc_macro2; /// /// use syn::{Ident, TokenTree}; -/// use syn::parse::{ident, tt}; +/// use synom::tokens::{Bang, Paren}; +/// use proc_macro2::TokenStream; /// /// // Parse a macro invocation like `stringify!($args)`. -/// named!(simple_mac -> (Ident, TokenTree), do_parse!( -/// name: ident >> -/// punct!("!") >> -/// body: tt >> +/// named!(simple_mac -> (Ident, (TokenStream, Paren)), do_parse!( +/// name: syn!(Ident) >> +/// syn!(Bang) >> +/// body: parens!(syn!(TokenStream)) >> /// (name, body) /// )); /// -/// fn main() { -/// let input = "stringify!($args)"; -/// let (name, body) = simple_mac(input).expect("macro invocation"); -/// println!("{:?}", name); -/// println!("{:?}", body); -/// } +/// # fn main() {} /// ``` #[macro_export] macro_rules! do_parse { diff --git a/synom/src/tokens.rs b/synom/src/tokens.rs index b5a34d58de..595019d438 100644 --- a/synom/src/tokens.rs +++ b/synom/src/tokens.rs @@ -1,3 +1,10 @@ +//! Discrete tokens that can be parsed out by synom. +//! +//! This module contains a number of useful tokens like `+=` and `/` along with +//! keywords like `crate` and such. These structures are used to track the spans +//! of these tokens and all implment the `ToTokens` and `Synom` traits when the +//! corresponding feature is activated. + use span::Span; macro_rules! tokens { @@ -150,18 +157,19 @@ tokens! { } syms: { (pub struct As => "as"), - (pub struct Box => "box"), + (pub struct Box_ => "box"), (pub struct Break => "break"), + (pub struct CapSelf => "Self"), (pub struct Catch => "catch"), (pub struct Const => "const"), (pub struct Continue => "continue"), (pub struct Crate => "crate"), - (pub struct Default => "default"), + (pub struct Default_ => "default"), (pub struct Do => "do"), (pub struct Else => "else"), (pub struct Enum => "enum"), (pub struct Extern => "extern"), - (pub struct Fn => "fn"), + (pub struct Fn_ => "fn"), (pub struct For => "for"), (pub struct If => "if"), (pub struct Impl => "impl"), @@ -178,6 +186,7 @@ tokens! { (pub struct Self_ => "self"), (pub struct Static => "static"), (pub struct Struct => "struct"), + (pub struct Super => "super"), (pub struct Trait => "trait"), (pub struct Type => "type"), (pub struct Union => "union"), @@ -190,7 +199,7 @@ tokens! { #[cfg(feature = "parsing")] mod parsing { - use proc_macro2::{TokenTree, TokenKind, Delimiter}; + use proc_macro2::{TokenTree, TokenKind, Delimiter, OpKind}; use IResult; use span::Span; @@ -224,19 +233,26 @@ mod parsing { where T: FromSpans, { let mut spans = [Span::default(); 3]; - assert!(s.chars().count() <= spans.len()); + assert!(s.len() <= spans.len()); let chars = s.chars(); let mut it = tokens.iter(); - for (ch, slot) in chars.zip(&mut spans) { - let (span, c) = match it.next() { - Some(&TokenTree { span, kind: TokenKind::Op(c, _) }) => (span, c), + for (i, (ch, slot)) in chars.zip(&mut spans).enumerate() { + let tok = match it.next() { + Some(tok) => tok, + _ => return IResult::Error + }; + let kind = match tok.kind { + TokenKind::Op(c, kind) if c == ch => kind, _ => return IResult::Error }; - if c != ch { - return IResult::Error + if i != s.len() - 1 { + match kind { + OpKind::Joint => {} + OpKind::Alone => return IResult::Error, + } } - *slot = Span(span); + *slot = Span(tok.span); } IResult::Done(it.as_slice(), new(T::from_spans(&spans))) } @@ -290,7 +306,7 @@ mod parsing { let rest = others.clone().into_iter().collect::>(); match f(&rest) { IResult::Done(remaining, ret) => { - if remaining.len() == 0 { + if remaining.is_empty() { IResult::Done(tokens.as_slice(), (ret, new(Span(span)))) } else { IResult::Error diff --git a/tests/test_expr.rs b/tests/test_expr.rs index 6216191565..749d50f5dd 100644 --- a/tests/test_expr.rs +++ b/tests/test_expr.rs @@ -86,7 +86,7 @@ fn test_catch_expr() { } "#; - let actual = parse_crate(raw).unwrap(); + let actual = raw.parse::().unwrap(); assert_eq!(&actual.items[0].ident, "catch"); diff --git a/tests/test_generics.rs b/tests/test_generics.rs index e18d2ef6af..6c370358b1 100644 --- a/tests/test_generics.rs +++ b/tests/test_generics.rs @@ -93,7 +93,7 @@ fn test_split_for_impl() { fn test_ty_param_bound() { let tokens = quote!('a); let expected = TyParamBound::Region(Lifetime::new("'a")); - assert_eq!(expected, parse_ty_param_bound(&tokens.to_string()).unwrap()); + assert_eq!(expected, tokens.to_string().parse().unwrap()); let tokens = quote!(Debug); let expected = TyParamBound::Trait( @@ -102,7 +102,7 @@ fn test_ty_param_bound() { trait_ref: "Debug".into(), }, TraitBoundModifier::None); - assert_eq!(expected, parse_ty_param_bound(&tokens.to_string()).unwrap()); + assert_eq!(expected, tokens.to_string().parse().unwrap()); let tokens = quote!(?Sized); let expected = TyParamBound::Trait( @@ -111,5 +111,5 @@ fn test_ty_param_bound() { trait_ref: "Sized".into(), }, TraitBoundModifier::Maybe(Default::default())); - assert_eq!(expected, parse_ty_param_bound(&tokens.to_string()).unwrap()); + assert_eq!(expected, tokens.to_string().parse().unwrap()); } diff --git a/tests/test_macro_input.rs b/tests/test_macro_input.rs index 9f80f6c649..bddf52e547 100644 --- a/tests/test_macro_input.rs +++ b/tests/test_macro_input.rs @@ -55,7 +55,7 @@ fn test_unit() { }), }; - assert_eq!(expected, parse_macro_input(raw.parse().unwrap()).unwrap()); + assert_eq!(expected, raw.parse().unwrap()); } #[test] @@ -140,7 +140,7 @@ fn test_struct() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); @@ -310,7 +310,7 @@ fn test_enum() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); @@ -380,7 +380,7 @@ fn test_attr_with_path() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); @@ -423,7 +423,7 @@ fn test_attr_with_non_mod_style_path() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); @@ -463,7 +463,7 @@ fn test_attr_with_mod_style_path_with_self() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); @@ -515,7 +515,7 @@ fn test_pub_restricted() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); } @@ -542,7 +542,7 @@ fn test_pub_restricted_crate() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); } @@ -570,7 +570,7 @@ fn test_pub_restricted_super() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); } @@ -598,7 +598,7 @@ fn test_pub_restricted_in_super() { }), }; - let actual = parse_macro_input(raw.parse().unwrap()).unwrap(); + let actual = raw.parse().unwrap(); assert_eq!(expected, actual); } diff --git a/tests/test_meta_item.rs b/tests/test_meta_item.rs index acf1710af2..cbdbb0221d 100644 --- a/tests/test_meta_item.rs +++ b/tests/test_meta_item.rs @@ -1,10 +1,12 @@ #![cfg(feature = "extra-traits")] extern crate syn; +extern crate synom; extern crate proc_macro2; +use proc_macro2::{Literal, TokenStream}; use syn::*; -use proc_macro2::Literal; +use synom::IResult; fn lit>(t: T) -> Lit { Lit { @@ -93,6 +95,14 @@ fn test_meta_item_multiple() { } fn run_test>(input: &str, expected: T) { - let attr = parse_outer_attr(input).unwrap(); + let tokens = input.parse::().unwrap(); + let tokens = tokens.into_iter().collect::>(); + let attr = match Attribute::parse_outer(&tokens) { + IResult::Done(rest, e) => { + assert!(rest.is_empty()); + e + } + IResult::Error => panic!("failed to parse"), + }; assert_eq!(expected.into(), attr.meta_item().unwrap()); } diff --git a/tests/test_round_trip.rs b/tests/test_round_trip.rs index 9db4f6977a..21bc51bb30 100644 --- a/tests/test_round_trip.rs +++ b/tests/test_round_trip.rs @@ -122,7 +122,7 @@ fn test_round_trip() { file.read_to_string(&mut content).unwrap(); let start = Instant::now(); - let (krate, elapsed) = match syn::parse_crate(&content) { + let (krate, elapsed) = match content.parse::() { Ok(krate) => (krate, start.elapsed()), Err(msg) => { errorf!("syn failed to parse\n{:?}\n", msg); diff --git a/tests/test_token_trees.rs b/tests/test_token_trees.rs index cacd90e39d..499581d439 100644 --- a/tests/test_token_trees.rs +++ b/tests/test_token_trees.rs @@ -4,7 +4,7 @@ extern crate syn; extern crate proc_macro2; use syn::TokenTree; -use proc_macro2::{TokenKind, OpKind, Delimiter}; +use proc_macro2::{TokenKind, OpKind, Delimiter, TokenStream}; use proc_macro2::Delimiter::*; fn op(c: char) -> TokenTree { @@ -69,7 +69,10 @@ fn test_struct() { ], )]; - let result = syn::parse_token_trees(raw.parse().unwrap()).unwrap(); + let result = raw.parse::().unwrap() + .into_iter() + .map(TokenTree) + .collect::>(); if result != expected { panic!("{:#?}\n!=\n{:#?}", result, expected); }