Skip to content

Commit 5d771cd

Browse files
committed
Auto merge of #31455 - tmiasko:expected-tokens, r=alexcrichton
Previously when breaking tokens into smaller pieces, the replace_token function have been used. It replaced current token and updated span information, but it did not clear the list of expected tokens, neither did it update remaining info about last token. This could lead to incorrect error message, like one described in the issue #24780: expected one of ... `>` ... found `>`
2 parents b5da60d + cecf83f commit 5d771cd

File tree

2 files changed

+39
-11
lines changed

2 files changed

+39
-11
lines changed

src/libsyntax/parse/parser.rs

+19-11
Original file line numberDiff line numberDiff line change
@@ -682,7 +682,7 @@ impl<'a> Parser<'a> {
682682
token::AndAnd => {
683683
let span = self.span;
684684
let lo = span.lo + BytePos(1);
685-
Ok(self.replace_token(token::BinOp(token::And), lo, span.hi))
685+
Ok(self.bump_with(token::BinOp(token::And), lo, span.hi))
686686
}
687687
_ => self.unexpected()
688688
}
@@ -717,7 +717,7 @@ impl<'a> Parser<'a> {
717717
token::BinOp(token::Shl) => {
718718
let span = self.span;
719719
let lo = span.lo + BytePos(1);
720-
self.replace_token(token::Lt, lo, span.hi);
720+
self.bump_with(token::Lt, lo, span.hi);
721721
true
722722
}
723723
_ => false,
@@ -745,17 +745,17 @@ impl<'a> Parser<'a> {
745745
token::BinOp(token::Shr) => {
746746
let span = self.span;
747747
let lo = span.lo + BytePos(1);
748-
Ok(self.replace_token(token::Gt, lo, span.hi))
748+
Ok(self.bump_with(token::Gt, lo, span.hi))
749749
}
750750
token::BinOpEq(token::Shr) => {
751751
let span = self.span;
752752
let lo = span.lo + BytePos(1);
753-
Ok(self.replace_token(token::Ge, lo, span.hi))
753+
Ok(self.bump_with(token::Ge, lo, span.hi))
754754
}
755755
token::Ge => {
756756
let span = self.span;
757757
let lo = span.lo + BytePos(1);
758-
Ok(self.replace_token(token::Eq, lo, span.hi))
758+
Ok(self.bump_with(token::Eq, lo, span.hi))
759759
}
760760
_ => {
761761
let gt_str = Parser::token_to_string(&token::Gt);
@@ -977,15 +977,23 @@ impl<'a> Parser<'a> {
977977
old_token
978978
}
979979

980-
/// EFFECT: replace the current token and span with the given one
981-
pub fn replace_token(&mut self,
982-
next: token::Token,
983-
lo: BytePos,
984-
hi: BytePos) {
980+
/// Advance the parser using provided token as a next one. Use this when
981+
/// consuming a part of a token. For example a single `<` from `<<`.
982+
pub fn bump_with(&mut self,
983+
next: token::Token,
984+
lo: BytePos,
985+
hi: BytePos) {
985986
self.last_span = mk_sp(self.span.lo, lo);
986-
self.token = next;
987+
// It would be incorrect to just stash current token, but fortunately
988+
// for tokens currently using `bump_with`, last_token will be of no
989+
// use anyway.
990+
self.last_token = None;
991+
self.last_token_interpolated = false;
987992
self.span = mk_sp(lo, hi);
993+
self.token = next;
994+
self.expected_tokens.clear();
988995
}
996+
989997
pub fn buffer_length(&mut self) -> isize {
990998
if self.buffer_start <= self.buffer_end {
991999
return self.buffer_end - self.buffer_start;

src/test/parse-fail/issue-24780.rs

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
// Verify that '>' is not both expected and found at the same time, as it used
12+
// to happen in #24780. For example, following should be an error:
13+
// expected one of ..., `>`, ... found `>`
14+
//
15+
// compile-flags: -Z parse-only
16+
17+
fn foo() -> Vec<usize>> {
18+
//~^ ERROR expected one of `!`, `::`, `where`, or `{`, found `>`
19+
Vec::new()
20+
}

0 commit comments

Comments
 (0)