diff --git a/src/grammar/verify.rs b/src/grammar/verify.rs index 48be58f731cd7..de0995e42753c 100644 --- a/src/grammar/verify.rs +++ b/src/grammar/verify.rs @@ -75,7 +75,7 @@ fn parse_token_list(file: &str) -> HashMap { "ANDAND" => Token::AndAnd, "AT" => Token::At, "LBRACKET" => Token::OpenDelim(DelimToken::Bracket), - "LIT_STR_RAW" => Token::Literal(Lit::StrRaw(Name(0), 0), None), + "LIT_STR_RAW" => Token::Literal(0, Lit::StrRaw(Name(0)), None), "RPAREN" => Token::CloseDelim(DelimToken::Paren), "SLASH" => Token::BinOp(BinOpToken::Slash), "COMMA" => Token::Comma, @@ -112,7 +112,7 @@ fn parse_token_list(file: &str) -> HashMap { "GT" => Token::Gt, "LE" => Token::Le, "LIT_BINARY" => Token::Literal(Lit::ByteStr(Name(0)), None), - "LIT_BINARY_RAW" => Token::Literal(Lit::ByteStrRaw(Name(0), 0), None), + "LIT_BINARY_RAW" => Token::Literal(0, Lit::ByteStrRaw(Name(0)), None), "QUESTION" => Token::Question, "SHEBANG" => Token::Shebang(Name(0)), _ => continue, @@ -204,16 +204,16 @@ fn parse_antlr_token(s: &str, tokens: &HashMap, surrogate_ Token::BinOp(..) => Token::BinOp(str_to_binop(content)), Token::BinOpEq(..) => Token::BinOpEq(str_to_binop(&content[..content.len() - 1])), Token::Literal(Lit::Str_(..), n) => Token::Literal(Lit::Str_(fix(content)), n), - Token::Literal(Lit::StrRaw(..), n) => Token::Literal(Lit::StrRaw(fix(content), - count(content)), n), + Token::Literal(Lit::StrRaw(..), n) => Token::Literal(Lit::StrRaw(count(content), + fix(content)), n), Token::Literal(Lit::Char(..), n) => Token::Literal(Lit::Char(fixchar(content)), n), Token::Literal(Lit::Byte(..), n) => Token::Literal(Lit::Byte(fixchar(content)), n), Token::DocComment(..) => Token::DocComment(nm), Token::Literal(Lit::Integer(..), n) => Token::Literal(Lit::Integer(nm), n), Token::Literal(Lit::Float(..), n) => Token::Literal(Lit::Float(nm), n), Token::Literal(Lit::ByteStr(..), n) => Token::Literal(Lit::ByteStr(nm), n), - Token::Literal(Lit::ByteStrRaw(..), n) => Token::Literal(Lit::ByteStrRaw(fix(content), - count(content)), n), + Token::Literal(Lit::ByteStrRaw(..), n) => Token::Literal(Lit::ByteStrRaw(count(content), + fix(content)), n), Token::Ident(..) => Token::Ident(ast::Ident::with_empty_ctxt(nm)), Token::Lifetime(..) => Token::Lifetime(ast::Ident::with_empty_ctxt(nm)), ref t => t.clone() diff --git a/src/librustc_incremental/calculate_svh/svh_visitor.rs b/src/librustc_incremental/calculate_svh/svh_visitor.rs index fd0856393fc1b..8cee7c131908f 100644 --- a/src/librustc_incremental/calculate_svh/svh_visitor.rs +++ b/src/librustc_incremental/calculate_svh/svh_visitor.rs @@ -1042,8 +1042,8 @@ impl<'a, 'hash, 'tcx> StrictVersionHashVisitor<'a, 'hash, 'tcx> { token::Lit::Float(val) | token::Lit::Str_(val) | token::Lit::ByteStr(val) => val.as_str().hash(self.st), - token::Lit::StrRaw(val, n) | - token::Lit::ByteStrRaw(val, n) => { + token::Lit::StrRaw(n, val) | + token::Lit::ByteStrRaw(n, val) => { val.as_str().hash(self.st); n.hash(self.st); } diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 2a911aceb9d94..906a3b496683a 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -1079,7 +1079,7 @@ pub enum StrStyle { /// A raw string, like `r##"foo"##` /// /// The uint is the number of `#` symbols used - Raw(usize) + Raw(u16) } /// A literal diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index fe5cb87ad59b5..66530476feaf9 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -101,7 +101,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt, }, (3, Some(&TokenTree::Token(_, token::Ident(ref code))), Some(&TokenTree::Token(_, token::Comma)), - Some(&TokenTree::Token(_, token::Literal(token::StrRaw(description, _), None)))) => { + Some(&TokenTree::Token(_, token::Literal(token::StrRaw(_, description), None)))) => { (code, Some(description)) } _ => unreachable!() diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index a208b934d79e1..b902e63ff00ba 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -143,6 +143,7 @@ pub trait AstBuilder { fn expr_usize(&self, span: Span, i: usize) -> P; fn expr_isize(&self, sp: Span, i: isize) -> P; fn expr_u8(&self, sp: Span, u: u8) -> P; + fn expr_u16(&self, sp: Span, u: u16) -> P; fn expr_u32(&self, sp: Span, u: u32) -> P; fn expr_bool(&self, sp: Span, value: bool) -> P; @@ -738,6 +739,9 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn expr_u32(&self, sp: Span, u: u32) -> P { self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U32))) } + fn expr_u16(&self, sp: Span, u: u16) -> P { + self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U16))) + } fn expr_u8(&self, sp: Span, u: u8) -> P { self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U8))) } diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index aa777a19a9bcb..656e56096ca2c 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -649,9 +649,9 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { return mk_lit!("Str_", suf, mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident))) } - token::Literal(token::StrRaw(ident, n), suf) => { - return mk_lit!("StrRaw", suf, mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)), - cx.expr_usize(sp, n)) + token::Literal(token::StrRaw(n, ident), suf) => { + return mk_lit!("StrRaw", suf, cx.expr_u16(sp, n), + mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident))); } token::Ident(ident) => { diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 681dec0ab564d..493209afd6a63 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1428,7 +1428,7 @@ impl<'a> StringReader<'a> { Symbol::intern("??") }; let suffix = self.scan_optional_raw_name(); - return Ok(token::Literal(token::StrRaw(id, hash_count), suffix)); + return Ok(token::Literal(token::StrRaw(hash_count, id), suffix)); } '-' => { if self.nextch_is('>') { @@ -1597,6 +1597,7 @@ impl<'a> StringReader<'a> { let mut hash_count = 0; while self.ch_is('#') { self.bump(); + // njn: check for u16 overflow? hash_count += 1; } @@ -1641,8 +1642,8 @@ impl<'a> StringReader<'a> { self.bump(); } self.bump(); - return token::ByteStrRaw(self.name_from_to(content_start_bpos, content_end_bpos), - hash_count); + return token::ByteStrRaw(hash_count, + self.name_from_to(content_start_bpos, content_end_bpos)); } } @@ -1849,7 +1850,7 @@ mod tests { assert_eq!(setup(&cm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()) .next_token() .tok, - token::Literal(token::StrRaw(Symbol::intern("\"#a\\b\x00c\""), 3), None)); + token::Literal(token::StrRaw(3, Symbol::intern("\"#a\\b\x00c\"")), None)); } #[test] @@ -1882,10 +1883,10 @@ mod tests { token::Literal(token::Integer(Symbol::intern("2")), Some(Symbol::intern("us")))); assert_eq!(setup(&cm, &sh, "r###\"raw\"###suffix".to_string()).next_token().tok, - token::Literal(token::StrRaw(Symbol::intern("raw"), 3), + token::Literal(token::StrRaw(3, Symbol::intern("raw")), Some(Symbol::intern("suffix")))); assert_eq!(setup(&cm, &sh, "br###\"raw\"###suffix".to_string()).next_token().tok, - token::Literal(token::ByteStrRaw(Symbol::intern("raw"), 3), + token::Literal(token::ByteStrRaw(3, Symbol::intern("raw")), Some(Symbol::intern("suffix")))); } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 49226be4147d7..c26ba1d717973 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -1528,14 +1528,14 @@ impl<'a> Parser<'a> { let s = Symbol::intern(&parse::str_lit(&s.as_str())); (true, LitKind::Str(s, ast::StrStyle::Cooked)) } - token::StrRaw(s, n) => { + token::StrRaw(n, s) => { let s = Symbol::intern(&parse::raw_str_lit(&s.as_str())); (true, LitKind::Str(s, ast::StrStyle::Raw(n))) } token::ByteStr(i) => { (true, LitKind::ByteStr(parse::byte_str_lit(&i.as_str()))) } - token::ByteStrRaw(i, _) => { + token::ByteStrRaw(_, i) => { (true, LitKind::ByteStr(Rc::new(i.to_string().into_bytes()))) } }; @@ -5654,7 +5654,7 @@ impl<'a> Parser<'a> { /// the `extern` keyword, if one is found. fn parse_opt_abi(&mut self) -> PResult<'a, Option> { match self.token { - token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(s, _), suf) => { + token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(_, s), suf) => { let sp = self.span; self.expect_no_suffix(sp, "ABI spec", suf); self.bump(); @@ -6151,7 +6151,7 @@ impl<'a> Parser<'a> { pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option)> { let ret = match self.token { token::Literal(token::Str_(s), suf) => (s, ast::StrStyle::Cooked, suf), - token::Literal(token::StrRaw(s, n), suf) => (s, ast::StrStyle::Raw(n), suf), + token::Literal(token::StrRaw(n, s), suf) => (s, ast::StrStyle::Raw(n), suf), _ => return None }; self.bump(); diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 8ac39dd462e7c..ff24cb4e70cc4 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -56,9 +56,9 @@ pub enum Lit { Integer(ast::Name), Float(ast::Name), Str_(ast::Name), - StrRaw(ast::Name, usize), /* raw str delimited by n hash symbols */ + StrRaw(u16, ast::Name), /* raw str delimited by n hash symbols */ ByteStr(ast::Name), - ByteStrRaw(ast::Name, usize), /* raw byte str delimited by n hash symbols */ + ByteStrRaw(u16, ast::Name), /* raw byte str delimited by n hash symbols */ } impl Lit { diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index c28b9d00501b7..27c54c64459c4 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -255,12 +255,13 @@ pub fn token_to_string(tok: &Token) -> String { token::Float(c) => c.to_string(), token::Integer(c) => c.to_string(), token::Str_(s) => format!("\"{}\"", s), - token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}", - delim=repeat("#", n), + token::StrRaw(n, s) => format!("r{delim}\"{string}\"{delim}", + delim=repeat("#", n as usize), string=s), token::ByteStr(v) => format!("b\"{}\"", v), - token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}", - delim=repeat("#", n), + token::ByteStrRaw(n, s) => format!("br{delim}\"{string}\"{delim}", + delim=repeat("#", n as + usize), string=s), }; @@ -687,7 +688,7 @@ pub trait PrintState<'a> { } ast::StrStyle::Raw(n) => { (format!("r{delim}\"{string}\"{delim}", - delim=repeat("#", n), + delim=repeat("#", n as usize), string=st)) } }; diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 0d5dcaf339feb..e06e4827cb897 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -157,6 +157,7 @@ impl TokenTree { // Searches for the occurrences of `"#*` and returns the minimum number of `#`s // required to wrap the text. + // njn: check for u16 overflow? let num_of_hashes = stripped.chars() .scan(0, |cnt, x| { *cnt = if x == '"' { @@ -177,7 +178,7 @@ impl TokenTree { tts: vec![TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"))), TokenTree::Token(sp, token::Eq), TokenTree::Token(sp, token::Literal( - token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))], + token::StrRaw(num_of_hashes, Symbol::intern(&stripped)), None))], close_span: sp, })) } @@ -303,7 +304,7 @@ impl TokenTree { span: sp, }) } - TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => { + TokenTree::Token(sp, Token::Literal(Lit::StrRaw(n, s), _)) => { let l = LitKind::Str(Symbol::intern(&parse::raw_str_lit(&s.as_str())), ast::StrStyle::Raw(n)); Some(Spanned {