Fix #35829 (syntax extension's quote_expr!() does not handle b"…")

* Handles `b"…"`, `br#"…"#` and `...` for `quote_expr!()`.
* Refactored the match statement to allow it to complain loudly on any
  unhandled token.
* Similarly, proc_macro's `quote!()` did not handle `br#"…"#` or `r#"…"#`,
  so this commit fixes it too.

src/libproc_macro_plugin/quote.rs

@@ -133,6 +133,14 @@ impl<'a> Quote for &'a str {
     }
 }
 
+impl Quote for usize {
+    fn quote(&self) -> TokenStream {
+        let integer_symbol = Symbol::intern(&self.to_string());
+        TokenTree::Token(DUMMY_SP, Token::Literal(token::Lit::Integer(integer_symbol), None))
+            .into()
+    }
+}
+
 impl Quote for Ident {
     fn quote(&self) -> TokenStream {
         // FIXME(jseyfried) quote hygiene
@@ -193,15 +201,17 @@ impl Quote for token::BinOpToken {
 impl Quote for Lit {
     fn quote(&self) -> TokenStream {
         macro_rules! gen_match {
-            ($($i:ident),*) => {
+            ($($i:ident),*; $($raw:ident),*) => {
                 match *self {
                     $( Lit::$i(lit) => quote!(::syntax::parse::token::Lit::$i((quote lit))), )*
-                    _ => panic!("Unsupported literal"),
+                    $( Lit::$raw(lit, n) => {
+                        quote!(::syntax::parse::token::Lit::$raw((quote lit), (quote n)))
+                    })*
                 }
             }
         }
 
-        gen_match!(Byte, Char, Float, Str_, Integer, ByteStr)
+        gen_match!(Byte, Char, Float, Str_, Integer, ByteStr; StrRaw, ByteStrRaw)
     }
 }
 

src/libsyntax/ext/quote.rs

@@ -612,16 +612,20 @@ fn mk_delim(cx: &ExtCtxt, sp: Span, delim: token::DelimToken) -> P<ast::Expr> {
 #[allow(non_upper_case_globals)]
 fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
     macro_rules! mk_lit {
-        ($name: expr, $suffix: expr, $($args: expr),*) => {{
-            let inner = cx.expr_call(sp, mk_token_path(cx, sp, $name), vec![$($args),*]);
+        ($name: expr, $suffix: expr, $content: expr $(, $count: expr)*) => {{
+            let name = mk_name(cx, sp, ast::Ident::with_empty_ctxt($content));
+            let inner = cx.expr_call(sp, mk_token_path(cx, sp, $name), vec![
+                name $(, cx.expr_usize(sp, $count))*
+            ]);
             let suffix = match $suffix {
                 Some(name) => cx.expr_some(sp, mk_name(cx, sp, ast::Ident::with_empty_ctxt(name))),
                 None => cx.expr_none(sp)
             };
             cx.expr_call(sp, mk_token_path(cx, sp, "Literal"), vec![inner, suffix])
         }}
     }
-    match *tok {
+
+    let name = match *tok {
         token::BinOp(binop) => {
             return cx.expr_call(sp, mk_token_path(cx, sp, "BinOp"), vec![mk_binop(cx, sp, binop)]);
         }
@@ -639,34 +643,14 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
                                 vec![mk_delim(cx, sp, delim)]);
         }
 
-        token::Literal(token::Byte(i), suf) => {
-            let e_byte = mk_name(cx, sp, ast::Ident::with_empty_ctxt(i));
-            return mk_lit!("Byte", suf, e_byte);
-        }
-
-        token::Literal(token::Char(i), suf) => {
-            let e_char = mk_name(cx, sp, ast::Ident::with_empty_ctxt(i));
-            return mk_lit!("Char", suf, e_char);
-        }
-
-        token::Literal(token::Integer(i), suf) => {
-            let e_int = mk_name(cx, sp, ast::Ident::with_empty_ctxt(i));
-            return mk_lit!("Integer", suf, e_int);
-        }
-
-        token::Literal(token::Float(fident), suf) => {
-            let e_fident = mk_name(cx, sp, ast::Ident::with_empty_ctxt(fident));
-            return mk_lit!("Float", suf, e_fident);
-        }
-
-        token::Literal(token::Str_(ident), suf) => {
-            return mk_lit!("Str_", suf, mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)))
-        }
-
-        token::Literal(token::StrRaw(ident, n), suf) => {
-            return mk_lit!("StrRaw", suf, mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)),
-                           cx.expr_usize(sp, n))
-        }
+        token::Literal(token::Byte(i), suf) => return mk_lit!("Byte", suf, i),
+        token::Literal(token::Char(i), suf) => return mk_lit!("Char", suf, i),
+        token::Literal(token::Integer(i), suf) => return mk_lit!("Integer", suf, i),
+        token::Literal(token::Float(i), suf) => return mk_lit!("Float", suf, i),
+        token::Literal(token::Str_(i), suf) => return mk_lit!("Str_", suf, i),
+        token::Literal(token::StrRaw(i, n), suf) => return mk_lit!("StrRaw", suf, i, n),
+        token::Literal(token::ByteStr(i), suf) => return mk_lit!("ByteStr", suf, i),
+        token::Literal(token::ByteStrRaw(i, n), suf) => return mk_lit!("ByteStrRaw", suf, i, n),
 
         token::Ident(ident) => {
             return cx.expr_call(sp,
@@ -688,10 +672,6 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
 
         token::Interpolated(_) => panic!("quote! with interpolated token"),
 
-        _ => ()
-    }
-
-    let name = match *tok {
         token::Eq           => "Eq",
         token::Lt           => "Lt",
         token::Le           => "Le",
@@ -706,6 +686,7 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
         token::At           => "At",
         token::Dot          => "Dot",
         token::DotDot       => "DotDot",
+        token::DotDotDot    => "DotDotDot",
         token::Comma        => "Comma",
         token::Semi         => "Semi",
         token::Colon        => "Colon",
@@ -718,7 +699,10 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
         token::Question     => "Question",
         token::Underscore   => "Underscore",
         token::Eof          => "Eof",
-        _                   => panic!("unhandled token in quote!"),
+
+        token::Whitespace | token::SubstNt(_) | token::Comment | token::Shebang(_) => {
+            panic!("unhandled token in quote!");
+        }
     };
     mk_token_path(cx, sp, name)
 }

src/test/run-pass-fulldeps/issue-35829.rs

@@ -0,0 +1,55 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// ignore-stage1
+// ignore-cross-compile
+#![feature(quote, rustc_private)]
+
+extern crate syntax;
+
+use syntax::ext::base::{ExtCtxt, DummyResolver};
+use syntax::ext::expand::ExpansionConfig;
+use syntax::parse::ParseSess;
+use syntax::codemap::{FilePathMapping, dummy_spanned};
+use syntax::print::pprust::expr_to_string;
+use syntax::ast::{Expr, ExprKind, LitKind, StrStyle, RangeLimits};
+use syntax::symbol::Symbol;
+use syntax::ptr::P;
+
+use std::rc::Rc;
+
+fn main() {
+    let parse_sess = ParseSess::new(FilePathMapping::empty());
+    let exp_cfg = ExpansionConfig::default("issue_35829".to_owned());
+    let mut resolver = DummyResolver;
+    let cx = ExtCtxt::new(&parse_sess, exp_cfg, &mut resolver);
+
+    // check byte string
+    let byte_string = quote_expr!(&cx, b"one");
+    let byte_string_lit_kind = LitKind::ByteStr(Rc::new(b"one".to_vec()));
+    assert_eq!(byte_string.node, ExprKind::Lit(P(dummy_spanned(byte_string_lit_kind))));
+
+    // check raw byte string
+    let raw_byte_string = quote_expr!(&cx, br###"#"two"#"###);
+    let raw_byte_string_lit_kind = LitKind::ByteStr(Rc::new(b"#\"two\"#".to_vec()));
+    assert_eq!(raw_byte_string.node, ExprKind::Lit(P(dummy_spanned(raw_byte_string_lit_kind))));
+
+    // check dotdotdot
+    let closed_range = quote_expr!(&cx, 0 ... 1);
+    assert_eq!(closed_range.node, ExprKind::Range(
+        Some(quote_expr!(&cx, 0)),
+        Some(quote_expr!(&cx, 1)),
+        RangeLimits::Closed
+    ));
+
+    // test case from 35829
+    let expr_35829 = quote_expr!(&cx, std::io::stdout().write(b"one"));
+    assert_eq!(expr_to_string(&expr_35829), r#"std::io::stdout().write(b"one")"#);
+}

src/test/run-pass-fulldeps/macro-quote-1.rs

@@ -17,11 +17,24 @@
 extern crate syntax;
 extern crate syntax_pos;
 
-use syntax::ast::Ident;
-use syntax::parse::token;
+use syntax::ast::{Ident, Name};
+use syntax::parse::token::{self, Token, Lit};
 use syntax::tokenstream::TokenTree;
 
 fn main() {
     let true_tok = token::Ident(Ident::from_str("true"));
     assert!(quote!(true).eq_unspanned(&true_tok.into()));
+
+    // issue #35829, extended check to proc_macro.
+    let triple_dot_tok = Token::DotDotDot;
+    assert!(quote!(...).eq_unspanned(&triple_dot_tok.into()));
+
+    let byte_str_tok = Token::Literal(Lit::ByteStr(Name::intern("one")), None);
+    assert!(quote!(b"one").eq_unspanned(&byte_str_tok.into()));
+
+    let byte_str_raw_tok = Token::Literal(Lit::ByteStrRaw(Name::intern("#\"two\"#"), 3), None);
+    assert!(quote!(br###"#"two"#"###).eq_unspanned(&byte_str_raw_tok.into()));
+
+    let str_raw_tok = Token::Literal(Lit::StrRaw(Name::intern("#\"three\"#"), 2), None);
+    assert!(quote!(r##"#"three"#"##).eq_unspanned(&str_raw_tok.into()));
 }