Rollup merge of rust-lang#53289 - ljedrz:improve_lexer, r=michaelwoer…

…ister

A few cleanups and minor improvements for the lexer

- improve readability by adjusting the formatting of some function signatures and adding some newlines
- reorder some functions for easier reading
- remove redundant `'static` in `const`s
- remove some explicit `return`s
- read directly to a `String` in `gather_comments_and_literals`
- change `unwrap_or!` (macro) to `unwrap_or` (function)
- move an `assert!`ion from `try_next_token` (called in a loop) to `try_real_token` after all calls to `try_next_token`
- `#[inline]` some one-liner functions
- assign directly from an `if-else` expression
- refactor a `match` to `map_or`
- add a `token::is_irrelevant` function to detect tokens that are not "`real`"

src/libsyntax/parse/lexer/comments.rs

@@ -63,6 +63,7 @@ pub fn strip_doc_comment_decoration(comment: &str) -> String {
         if !lines.is_empty() && lines[0].chars().all(|c| c == '*') {
             i += 1;
         }
+
         while i < j && lines[i].trim().is_empty() {
             i += 1;
         }
@@ -74,9 +75,11 @@ pub fn strip_doc_comment_decoration(comment: &str) -> String {
                .all(|c| c == '*') {
             j -= 1;
         }
+
         while j > i && lines[j - 1].trim().is_empty() {
             j -= 1;
         }
+
         lines[i..j].to_vec()
     }
 
@@ -85,6 +88,7 @@ pub fn strip_doc_comment_decoration(comment: &str) -> String {
         let mut i = usize::MAX;
         let mut can_trim = true;
         let mut first = true;
+
         for line in &lines {
             for (j, c) in line.chars().enumerate() {
                 if j > i || !"* \t".contains(c) {
@@ -119,7 +123,8 @@ pub fn strip_doc_comment_decoration(comment: &str) -> String {
     }
 
     // one-line comments lose their prefix
-    const ONELINERS: &'static [&'static str] = &["///!", "///", "//!", "//"];
+    const ONELINERS: &[&str] = &["///!", "///", "//!", "//"];
+
     for prefix in ONELINERS {
         if comment.starts_with(*prefix) {
             return (&comment[prefix.len()..]).to_string();
@@ -205,6 +210,7 @@ fn all_whitespace(s: &str, col: CharPos) -> Option<usize> {
     let len = s.len();
     let mut col = col.to_usize();
     let mut cursor: usize = 0;
+
     while col > 0 && cursor < len {
         let ch = char_at(s, cursor);
         if !ch.is_whitespace() {
@@ -213,7 +219,8 @@ fn all_whitespace(s: &str, col: CharPos) -> Option<usize> {
         cursor += ch.len_utf8();
         col -= 1;
     }
-    return Some(cursor);
+
+    Some(cursor)
 }
 
 fn trim_whitespace_prefix_and_push_line(lines: &mut Vec<String>, s: String, col: CharPos) {
@@ -246,11 +253,13 @@ fn read_block_comment(rdr: &mut StringReader,
         "src_index={}, end_src_index={}, line_begin_pos={}",
         src_index, end_src_index, rdr.filemap.line_begin_pos(rdr.pos).to_u32());
     let mut n = 0;
+
     while src_index < end_src_index {
         let c = char_at(&rdr.src, src_index);
         src_index += c.len_utf8();
         n += 1;
     }
+
     let col = CharPos(n);
 
     rdr.bump();
@@ -358,10 +367,10 @@ pub struct Literal {
 // it appears this function is called only from pprust... that's
 // probably not a good thing.
 pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut dyn Read)
-                                    -> (Vec<Comment>, Vec<Literal>) {
-    let mut src = Vec::new();
-    srdr.read_to_end(&mut src).unwrap();
-    let src = String::from_utf8(src).unwrap();
+    -> (Vec<Comment>, Vec<Literal>)
+{
+    let mut src = String::new();
+    srdr.read_to_string(&mut src).unwrap();
     let cm = CodeMap::new(sess.codemap().path_mapping().clone());
     let filemap = cm.new_filemap(path, src);
     let mut rdr = lexer::StringReader::new_raw(sess, filemap, None);
@@ -370,6 +379,7 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut
     let mut literals: Vec<Literal> = Vec::new();
     let mut code_to_the_left = false; // Only code
     let mut anything_to_the_left = false; // Code or comments
+
     while !rdr.is_eof() {
         loop {
             // Eat all the whitespace and count blank lines.