From 2f78ebe2a8772e10a142a7b473dc73286c3a1985 Mon Sep 17 00:00:00 2001 From: Iban Eguia Moraza Date: Wed, 10 Jun 2020 19:02:23 +0200 Subject: [PATCH 001/289] Started with the new lexer implementation --- boa/src/syntax/ast/token.rs | 6 +- boa/src/syntax/lexer/comment.rs | 25 + boa/src/syntax/lexer/cursor.rs | 209 +++++++ boa/src/syntax/lexer/error.rs | 41 ++ boa/src/syntax/lexer/lexer_old.rs | 872 ++++++++++++++++++++++++++++ boa/src/syntax/lexer/mod.rs | 928 ++++-------------------------- boa/src/syntax/lexer/string.rs | 189 ++++++ 7 files changed, 1435 insertions(+), 835 deletions(-) create mode 100644 boa/src/syntax/lexer/comment.rs create mode 100644 boa/src/syntax/lexer/cursor.rs create mode 100644 boa/src/syntax/lexer/error.rs create mode 100644 boa/src/syntax/lexer/lexer_old.rs create mode 100644 boa/src/syntax/lexer/string.rs diff --git a/boa/src/syntax/ast/token.rs b/boa/src/syntax/ast/token.rs index f29e501a68d..d6398c57759 100644 --- a/boa/src/syntax/ast/token.rs +++ b/boa/src/syntax/ast/token.rs @@ -8,7 +8,7 @@ use crate::builtins::BigInt; use crate::syntax::{ ast::{Keyword, Punctuator, Span}, - lexer::LexerError, + lexer::Error as LexerError, }; use bitflags::bitflags; use std::{ @@ -114,7 +114,7 @@ impl FromStr for RegExpFlags { b'u' => Self::UNICODE, b'y' => Self::STICKY, _ => { - return Err(LexerError::new(format!( + return Err(LexerError::syntax(format!( "invalid regular expression flag {}", char::from(c) ))) @@ -124,7 +124,7 @@ impl FromStr for RegExpFlags { if !flags.contains(new_flag) { flags.insert(new_flag); } else { - return Err(LexerError::new(format!( + return Err(LexerError::syntax(format!( "invalid regular expression flag {}", char::from(c) ))); diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs new file mode 100644 index 00000000000..500852fb73c --- /dev/null +++ b/boa/src/syntax/lexer/comment.rs @@ -0,0 +1,25 @@ +//! Coments lexing. + +use super::{Cursor, Error, Tokenizer}; +use crate::syntax::ast::bigint::BigInt; +use crate::syntax::ast::{ + token::{NumericLiteral, Token, TokenKind}, + Position, Punctuator, Span, +}; +use std::{ + char::{decode_utf16, from_u32}, + fmt, + io::{self, BufRead, Bytes, Read, Seek}, + iter::Peekable, + str::{Chars, FromStr}, +}; + +/// Lexes single line comments, starting with `//`. +#[derive(Debug, Clone, Copy)] +pub(super) struct SingleLineComment; + +impl Tokenizer for SingleLineComment { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result { + unimplemented!() + } +} diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs new file mode 100644 index 00000000000..66649524fde --- /dev/null +++ b/boa/src/syntax/lexer/cursor.rs @@ -0,0 +1,209 @@ +use crate::syntax::ast::bigint::BigInt; +use crate::syntax::ast::{ + token::{NumericLiteral, Token, TokenKind}, + Position, Punctuator, Span, +}; +use std::{ + char::{decode_utf16, from_u32}, + error, fmt, + io::{self, BufRead, Bytes, Read, Seek}, + iter::Peekable, + str::{Chars, FromStr}, +}; + +/// Cursor over the source code. +#[derive(Debug)] +pub(super) struct Cursor { + iter: InnerIter, + peeked: Option>>, + pos: Position, +} + +impl Cursor { + /// Gets the current position of the cursor in the source code. + #[inline] + pub(super) fn pos(&self) -> Position { + self.pos + } + + /// Advances the position to the next line. + #[inline] + fn next_column(&mut self) { + let current_line = self.pos.line_number(); + let next_column = self.pos.column_number() + 1; + self.pos = Position::new(current_line, next_column); + } + + /// Advances the position to the next column. + #[inline] + fn next_line(&mut self) { + let next_line = self.pos.line_number() + 1; + self.pos = Position::new(next_line, 1); + } + + /// Performs a carriage return to modify the position in the source. + #[inline] + fn carriage_return(&mut self) { + let current_line = self.pos.line_number(); + self.pos = Position::new(current_line, 1); + } +} + +impl Cursor +where + R: Read, +{ + /// Creates a new Lexer cursor. + #[inline] + pub(super) fn new(inner: R) -> Self { + Self { + iter: InnerIter::new(inner.bytes()), + peeked: None, + pos: Position::new(1, 1), + } + } + + /// Peeks the next character. + #[inline] + pub(super) fn peek(&mut self) -> Option<&io::Result> { + let iter = &mut self.iter; + self.peeked.get_or_insert_with(|| iter.next()).as_ref() + } + + /// Compares the character passed in to the next character, if they match true is returned and the buffer is incremented + #[inline] + pub(super) fn next_is(&mut self, peek: char) -> io::Result { + Ok(match self.peek() { + None => false, + Some(&Ok(next)) if next == peek => { + let _ = self.peeked.take(); + true + } + _ => false, + Some(&Err(_)) => return self.peeked.take().unwrap().unwrap().map(|_| false), + }) + } + + /// Fills the buffer with all characters until the stop character is found. + /// + /// Note: It will not add the stop character to the buffer. + pub(super) fn take_until(&mut self, stop: char, buf: &mut String) -> io::Result<()> { + unimplemented!() + } + + /// Retrieves the given number of characters and adds them to the buffer. + pub(super) fn take(&mut self, count: usize, buf: &mut String) -> io::Result<()> { + unimplemented!() + } + + /// It will fill the buffer with checked ASCII bytes. + pub(super) fn fill_bytes(&mut self, buf: &[u8]) -> io::Result<()> { + unimplemented!() + } + + /// Retrieves the next character as an ASCII character. + /// + /// It will make sure that the next character is an ASCII byte, or return an error otherwise. + pub(super) fn next_as_byte(&mut self) -> Option> { + unimplemented!() + } +} + +impl Iterator for Cursor +where + R: Read, +{ + type Item = io::Result; + + #[inline] + fn next(&mut self) -> Option { + let chr = match self.peeked.take() { + Some(v) => v, + None => self.iter.next(), + }; + + match chr { + Some(Ok('\r')) => self.carriage_return(), + Some(Ok('\r')) | Some(Ok('\u{2028}')) | Some(Ok('\u{2029}')) => self.next_line(), + Some(Ok(_)) => self.next_column(), + _ => {} + } + + chr + } +} + +/// Inner iterator for a cursor. +#[derive(Debug)] +struct InnerIter { + iter: Bytes, +} + +impl InnerIter { + /// Creates a new inner iterator. + fn new(iter: Bytes) -> Self { + Self { iter } + } +} + +impl Iterator for InnerIter +where + R: Read, +{ + type Item = io::Result; + + fn next(&mut self) -> Option { + use std::convert::TryFrom; + + let first_byte = match self.iter.next()? { + Ok(b) => b, + Err(e) => return Some(Err(e)), + }; + + let chr: char = if first_byte < 0x80 { + // 0b0xxx_xxxx + first_byte.into() + } else { + let mut buf = [first_byte, 0u8, 0u8, 0u8]; + let num_bytes = if first_byte < 0xE0 { + // 0b110x_xxxx + 2 + } else if first_byte < 0xF0 { + // 0b1110_xxxx + 3 + } else { + // 0b1111_0xxx + 4 + }; + + for i in 1..num_bytes { + let next = match self.iter.next() { + Some(Ok(b)) => b, + Some(Err(e)) => return Some(Err(e)), + None => { + return Some(Err(io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ))) + } + }; + + buf[i] = next; + } + + let int = u32::from_le_bytes(buf); + + match char::try_from(int).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ) + }) { + Ok(chr) => chr, + Err(e) => return Some(Err(e)), + } + }; + + Some(Ok(chr)) + } +} diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs new file mode 100644 index 00000000000..525861f2e18 --- /dev/null +++ b/boa/src/syntax/lexer/error.rs @@ -0,0 +1,41 @@ +use std::{error::Error as StdError, fmt, io}; + +#[derive(Debug)] +pub enum Error { + IO(io::Error), + Syntax(Box), +} + +impl From for Error { + fn from(err: io::Error) -> Self { + Self::IO(err) + } +} + +impl Error { + /// Creates a new syntax error. + pub(super) fn syntax(err: M) -> Self + where + M: Into>, + { + Self::Syntax(err.into()) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Self::IO(e) => write!(f, "I/O error: {}", e), + Self::Syntax(e) => write!(f, "Syntax Error: {}", e), + } + } +} + +impl StdError for Error { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + match *self { + Self::IO(err) => Some(&err), + Self::Syntax(_) => None, + } + } +} diff --git a/boa/src/syntax/lexer/lexer_old.rs b/boa/src/syntax/lexer/lexer_old.rs new file mode 100644 index 00000000000..568b54d526d --- /dev/null +++ b/boa/src/syntax/lexer/lexer_old.rs @@ -0,0 +1,872 @@ +//! A lexical analyzer for JavaScript source code. +//! +//! The Lexer splits its input source code into a sequence of input elements called tokens, represented by the [Token](../ast/token/struct.Token.html) structure. +//! It also removes whitespace and comments and attaches them to the next token. + +#[cfg(test)] +mod tests; + +use crate::builtins::BigInt; +use crate::{ + syntax::ast::{ + token::{NumericLiteral, Token, TokenKind}, + Position, Punctuator, Span, + }, + BoaProfiler, +}; +use std::{ + char::{decode_utf16, from_u32}, + error, fmt, + iter::Peekable, + str::{Chars, FromStr}, +}; + +/// `vop` tests the next token to see if we're on an assign operation of just a plain binary operation. +/// +/// If the next value is not an assignment operation it will pattern match the provided values and return the corresponding token. +macro_rules! vop { + ($this:ident, $assign_op:expr, $op:expr) => ({ + let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; + match preview { + '=' => { + $this.next(); + $this.next_column(); + $assign_op + } + _ => $op, + } + }); + ($this:ident, $assign_op:expr, $op:expr, {$($case:pat => $block:expr), +}) => ({ + let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; + match preview { + '=' => { + $this.next(); + $this.next_column(); + $assign_op + }, + $($case => { + $this.next(); + $this.next_column(); + $block + })+, + _ => $op + } + }); + ($this:ident, $op:expr, {$($case:pat => $block:expr),+}) => { + let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; + match preview { + $($case => { + $this.next()?; + $this.next_column(); + $block + })+, + _ => $op + } + } +} + +/// The `op` macro handles binary operations or assignment operations and converts them into tokens. +macro_rules! op { + ($this:ident, $start_pos:expr, $assign_op:expr, $op:expr) => ({ + let punc = vop!($this, $assign_op, $op); + $this.push_punc(punc, $start_pos); + }); + ($this:ident, $start_pos:expr, $assign_op:expr, $op:expr, {$($case:pat => $block:expr),+}) => ({ + let punc = vop!($this, $assign_op, $op, {$($case => $block),+}); + $this.push_punc(punc, $start_pos); + }); +} + +/// An error that occurred during lexing or compiling of the source input. +/// +/// [LexerError] implements [fmt::Display] so you just display this value as an error +#[derive(Debug, Clone)] +pub struct LexerError { + /// details will be displayed when a LexerError occurs. + details: String, +} + +impl LexerError { + /// Create a new LexerError struct + /// + /// * `msg` - The message to show when LexerError is displayed + pub(crate) fn new(msg: M) -> Self + where + M: Into, + { + Self { + details: msg.into(), + } + } +} + +impl fmt::Display for LexerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.details) + } +} + +impl error::Error for LexerError { + fn description(&self) -> &str { + &self.details + } + + fn cause(&self) -> Option<&dyn error::Error> { + // Generic error, underlying cause isn't tracked. + None + } +} + +/// A lexical analyzer for JavaScript source code. +#[derive(Debug)] +pub struct Lexer<'a> { + /// The list of tokens generated so far. + /// + /// This field is public so you can use them once lexing has finished. + pub tokens: Vec, + /// The current position in the source code. + position: Position, + /// The full Peekable buffer, an array of [Char]s + buffer: Peekable>, +} + +impl<'a> Lexer<'a> { + /// Returns a Lexer with a buffer inside + /// + /// The buffer needs to have a lifetime as long as the Lexer instance itself + pub fn new(buffer: &'a str) -> Lexer<'a> { + Lexer { + tokens: Vec::new(), + position: Position::new(1, 1), + buffer: buffer.chars().peekable(), + } + } + + /// Push a token onto the token queue. + fn push_token(&mut self, tk: TokenKind, start: Position) { + let end = if let TokenKind::LineTerminator = tk { + self.position + } else { + Position::new( + self.position.line_number(), + self.position.column_number() - 1, + ) + }; + self.tokens.push(Token::new(tk, Span::new(start, end))) + } + + /// Push a punctuation token + fn push_punc(&mut self, punc: Punctuator, start: Position) { + self.push_token(TokenKind::Punctuator(punc), start); + } + + /// Changes the current position by advancing to the next column. + fn next_column(&mut self) { + let pos = Position::new( + self.position.line_number(), + self.position.column_number() + 1, + ); + self.position = pos; + } + + /// Changes the current position by advancing the given number of columns. + fn move_columns(&mut self, columns: u32) { + let pos = Position::new( + self.position.line_number(), + self.position.column_number() + columns, + ); + self.position = pos; + } + + fn carriage_return(&mut self) { + let pos = Position::new(self.position.line_number(), 1); + self.position = pos; + } + + /// Changes the current position by advancing to the next line. + fn next_line(&mut self) { + let pos = Position::new(self.position.line_number() + 1, 1); + self.position = pos; + } + + /// Changes the current position by advancing the given number of lines. + fn move_lines(&mut self, lines: u32) { + let pos = Position::new(self.position.line_number() + lines, 1); + self.position = pos; + } + + /// next fetches the next token and return it, or a LexerError if there are no more. + fn next(&mut self) -> char { + self.buffer.next().expect( + "No more more characters to consume from input stream, \ + use preview_next() first to check before calling next()", + ) + } + + /// Preview the next character but don't actually increment + fn preview_next(&mut self) -> Option { + self.buffer.peek().copied() + } + + /// Preview a char x indexes further in buf, without incrementing + fn preview_multiple_next(&mut self, nb_next: usize) -> Option { + let mut next_peek = None; + + for (i, x) in self.buffer.clone().enumerate() { + if i >= nb_next { + break; + } + + next_peek = Some(x); + } + + next_peek + } + + /// Utility Function, while ``f(char)`` is true, read chars and move curser. + /// All chars are returned as a string + fn take_char_while(&mut self, mut f: F) -> Result + where + F: FnMut(char) -> bool, + { + let mut s = String::new(); + while self.buffer.peek().is_some() + && f(self.preview_next().expect("Could not preview next value")) + { + s.push(self.next()); + } + + Ok(s) + } + + /// Compares the character passed in to the next character, if they match true is returned and the buffer is incremented + fn next_is(&mut self, peek: char) -> bool { + let result = self.preview_next() == Some(peek); + if result { + self.next_column(); + self.buffer.next(); + } + result + } + + /// Utility function for checkint the NumericLiteral is not followed by an `IdentifierStart` or `DecimalDigit` character. + /// + /// More information: + /// - [ECMAScript Specification][spec] + /// + /// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals + fn check_after_numeric_literal(&mut self) -> Result<(), LexerError> { + match self.preview_next() { + Some(ch) + if ch.is_ascii_alphabetic() || ch == '$' || ch == '_' || ch.is_ascii_digit() => + { + Err(LexerError::new("NumericLiteral token must not be followed by IdentifierStart nor DecimalDigit characters")) + } + Some(_) => Ok(()), + None => Ok(()) + } + } + + /// Lexes a numerical literal. + /// + /// More information: + /// - [ECMAScript Specification][spec] + /// + /// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals + fn reed_numerical_literal(&mut self, ch: char) -> Result<(), LexerError> { + /// This is a helper structure + /// + /// This structure helps with identifying what numerical type it is and what base is it. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum NumericKind { + Rational, + Integer(u8), + BigInt(u8), + } + + impl NumericKind { + /// Get the base of the number kind. + fn base(self) -> u32 { + match self { + Self::Rational => 10, + Self::Integer(base) => base as u32, + Self::BigInt(base) => base as u32, + } + } + + /// Converts `self` to BigInt kind. + fn to_bigint(self) -> Self { + match self { + Self::Rational => unreachable!("can not convert rational number to BigInt"), + Self::Integer(base) => Self::BigInt(base), + Self::BigInt(base) => Self::BigInt(base), + } + } + } + + // TODO: Setup strict mode. + let strict_mode = false; + + let mut buf = ch.to_string(); + let mut kind = NumericKind::Integer(10); + let start_pos = self.position; + if ch == '0' { + match self.preview_next() { + None => { + self.next_column(); + self.push_token( + TokenKind::NumericLiteral(NumericLiteral::Integer(0)), + start_pos, + ); + return Ok(()); + } + Some('x') | Some('X') => { + self.next(); + self.next_column(); + kind = NumericKind::Integer(16); + } + Some('o') | Some('O') => { + self.next(); + self.next_column(); + kind = NumericKind::Integer(8); + } + Some('b') | Some('B') => { + self.next(); + self.next_column(); + kind = NumericKind::Integer(2); + } + Some(ch) if ch.is_ascii_digit() => { + let mut is_implicit_octal = true; + while let Some(ch) = self.preview_next() { + if !ch.is_ascii_digit() { + break; + } else if !ch.is_digit(8) { + is_implicit_octal = false; + } + buf.push(self.next()); + } + if !strict_mode { + if is_implicit_octal { + kind = NumericKind::Integer(8); + } + } else { + return Err(if is_implicit_octal { + LexerError::new( + "Implicit octal literals are not allowed in strict mode.", + ) + } else { + LexerError::new( + "Decimals with leading zeros are not allowed in strict mode.", + ) + }); + } + } + Some(_) => {} + } + } + + while let Some(ch) = self.preview_next() { + if !ch.is_digit(kind.base()) { + break; + } + buf.push(self.next()); + } + + if self.next_is('n') { + kind = kind.to_bigint(); + } + + if let NumericKind::Integer(10) = kind { + 'digitloop: while let Some(ch) = self.preview_next() { + match ch { + '.' => loop { + kind = NumericKind::Rational; + buf.push(self.next()); + + let c = match self.preview_next() { + Some(ch) => ch, + None => break, + }; + + match c { + 'e' | 'E' => { + match self + .preview_multiple_next(2) + .unwrap_or_default() + .to_digit(10) + { + Some(0..=9) | None => { + buf.push(self.next()); + } + _ => { + break 'digitloop; + } + } + } + _ => { + if !c.is_digit(10) { + break 'digitloop; + } + } + } + }, + 'e' | 'E' => { + kind = NumericKind::Rational; + match self + .preview_multiple_next(2) + .unwrap_or_default() + .to_digit(10) + { + Some(0..=9) | None => { + buf.push(self.next()); + } + _ => { + break; + } + } + buf.push(self.next()); + } + '+' | '-' => { + break; + } + _ if ch.is_digit(10) => { + buf.push(self.next()); + } + _ => break, + } + } + } + + self.check_after_numeric_literal()?; + + let num = match kind { + NumericKind::BigInt(base) => { + NumericLiteral::BigInt( + BigInt::from_string_radix(&buf, base as u32).expect("Could not conver to BigInt") + ) + } + NumericKind::Rational /* base: 10 */ => { + NumericLiteral::Rational( + f64::from_str(&buf) + .map_err(|_| LexerError::new("Could not convert value to f64"))?, + ) + } + NumericKind::Integer(base) => { + if let Ok(num) = i32::from_str_radix(&buf, base as u32) { + NumericLiteral::Integer( + num + ) + } else { + let b = f64::from(base); + let mut result = 0.0_f64; + for c in buf.chars() { + let digit = f64::from(c.to_digit(base as u32).unwrap()); + result = result * b + digit; + } + + NumericLiteral::Rational(result) + } + + } + }; + + self.move_columns(buf.len() as u32); + self.push_token(TokenKind::NumericLiteral(num), start_pos); + + Ok(()) + } + + /// Runs the lexer until completion, returning a [LexerError] if there's a syntax issue, or an empty unit result + /// + /// # Example + /// + /// ``` + /// # use boa::syntax::lexer::{LexerError, Lexer}; + /// fn main() -> Result<(), LexerError> { + /// let buffer = String::from("Hello World"); + /// let mut lexer = Lexer::new(&buffer); + /// lexer.lex() + /// } + /// ``` + pub fn lex(&mut self) -> Result<(), LexerError> { + let _timer = BoaProfiler::global().start_event("lex", "lexing"); + loop { + // Check if we've reached the end + if self.preview_next().is_none() { + return Ok(()); + } + let start_pos = self.position; + self.next_column(); + let ch = self.next(); + match ch { + // StringLiteral + '"' | '\'' => { + let mut buf = String::new(); + loop { + if self.preview_next().is_none() { + return Err(LexerError::new("Unterminated String")); + } + match self.next() { + '\'' if ch == '\'' => { + break; + } + '"' if ch == '"' => { + break; + } + '\\' => { + if self.preview_next().is_none() { + return Err(LexerError::new("Unterminated String")); + } + let escape_pos = self.position; + let escape = self.next(); + if escape != '\n' { + let escaped_ch = match escape { + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'b' => '\x08', + 'f' => '\x0c', + '0' => '\0', + 'x' => { + let mut nums = String::with_capacity(2); + for _ in 0_u8..2 { + if self.preview_next().is_none() { + return Err(LexerError::new("Unterminated String")); + } + nums.push(self.next()); + } + self.move_columns(2); + let as_num = match u64::from_str_radix(&nums, 16) { + Ok(v) => v, + Err(_) => 0, + }; + match from_u32(as_num as u32) { + Some(v) => v, + None => panic!( + "{}: {} is not a valid unicode scalar value", + self.position, as_num + ), + } + } + 'u' => { + // There are 2 types of codepoints. Surragate codepoints and unicode codepoints. + // UTF-16 could be surrogate codepoints, "\uXXXX\uXXXX" which make up a single unicode codepoint. + // We will need to loop to make sure we catch all UTF-16 codepoints + // Example Test: https://github.com/tc39/test262/blob/ee3715ee56744ccc8aeb22a921f442e98090b3c1/implementation-contributed/v8/mjsunit/es6/unicode-escapes.js#L39-L44 + + // Support \u{X..X} (Unicode Codepoint) + if self.next_is('{') { + let s = self + .take_char_while(char::is_alphanumeric) + .expect("Could not read chars"); + + // We know this is a single unicode codepoint, convert to u32 + let as_num = match u32::from_str_radix(&s, 16) { + Ok(v) => v, + Err(_) => 0, + }; + let c = from_u32(as_num).ok_or_else(|| LexerError::new("Invalid Unicode escape sequence"))?; + + if self.preview_next().is_none() { + return Err(LexerError::new("Unterminated String")); + } + self.next(); // '}' + self.move_columns(s.len() as u32); + c + } else { + let mut codepoints: Vec = vec![]; + loop { + // Collect each character after \u e.g \uD83D will give "D83D" + let s = self + .take_char_while(char::is_alphanumeric) + .expect("Could not read chars"); + + // Convert to u16 + let as_num = match u16::from_str_radix(&s, 16) { + Ok(v) => v, + Err(_) => 0, + }; + + codepoints.push(as_num); + self.move_columns(s.len() as u32); + + // Check for another UTF-16 codepoint + if self.next_is('\\') && self.next_is('u') { + continue; + } + break; + } + + // codepoints length should either be 1 (unicode codepoint) or 2 (surrogate codepoint). + // Rust's decode_utf16 will deal with it regardless + decode_utf16(codepoints.iter().cloned()) + .next() + .expect("Could not get next codepoint") + .expect("Could not get next codepoint") + } + } + '\'' | '"' | '\\' => escape, + ch => { + let details = format!("invalid escape sequence `{}` at line {}, column {}", escape_pos.line_number(), escape_pos.column_number(), ch); + return Err(LexerError { details }); + } + }; + buf.push(escaped_ch); + } + } + next_ch => buf.push(next_ch), + } + } + let str_length = buf.len() as u32; + // Why +1? Quotation marks are not included, + // So technically it would be +2, (for both " ") but we want to be 1 less + // to compensate for the incrementing at the top + self.move_columns( str_length.wrapping_add(1)); + self.push_token(TokenKind::string_literal(buf), start_pos); + } + // TemplateLiteral + '`' => { + let mut buf = String::new(); + loop { + if self.preview_next().is_none() { + return Err(LexerError::new("Unterminated template literal")); + } + match self.next() { + '`' => { + break; + } + next_ch => buf.push(next_ch), + // TODO when there is an expression inside the literal + } + } + let str_length = buf.len() as u32; + // Why +1? Quotation marks are not included, + // So technically it would be +2, (for both " ") but we want to be 1 less + // to compensate for the incrementing at the top + self.move_columns( str_length.wrapping_add(1)); + self.push_token(TokenKind::template_literal(buf), start_pos); + } + _ if ch.is_digit(10) => self.reed_numerical_literal(ch)?, + _ if ch.is_alphabetic() || ch == '$' || ch == '_' => { + let mut buf = ch.to_string(); + while let Some(ch) = self.preview_next() { + if ch.is_alphabetic() || ch.is_digit(10) || ch == '_' { + buf.push(self.next()); + } else { + break; + } + } + let tk = match buf.as_str() { + "true" => TokenKind::BooleanLiteral(true), + "false" => TokenKind::BooleanLiteral(false), + "null" => TokenKind::NullLiteral, + "NaN" => TokenKind::NumericLiteral(NumericLiteral::Rational(f64::NAN)), + slice => { + if let Ok(keyword) = FromStr::from_str(slice) { + TokenKind::Keyword(keyword) + } else { + TokenKind::identifier(slice) + } + } + }; + + // Move position forward the length of the token + self.move_columns( (buf.len().wrapping_sub(1)) as u32); + + self.push_token(tk, start_pos); + } + ';' => self.push_punc(Punctuator::Semicolon, start_pos), + ':' => self.push_punc(Punctuator::Colon, start_pos), + '.' => { + // . or ... + if self.next_is('.') { + if self.next_is('.') { + self.push_punc(Punctuator::Spread, start_pos); + } else { + return Err(LexerError::new("Expecting Token .")); + } + } else { + self.push_punc(Punctuator::Dot, start_pos); + }; + } + '(' => self.push_punc(Punctuator::OpenParen, start_pos), + ')' => self.push_punc(Punctuator::CloseParen, start_pos), + ',' => self.push_punc(Punctuator::Comma, start_pos), + '{' => self.push_punc(Punctuator::OpenBlock, start_pos), + '}' => self.push_punc(Punctuator::CloseBlock, start_pos), + '[' => self.push_punc(Punctuator::OpenBracket, start_pos), + ']' => self.push_punc(Punctuator::CloseBracket, start_pos), + '?' => self.push_punc(Punctuator::Question, start_pos), + // Comments + '/' => { + if let Some(ch) = self.preview_next() { + match ch { + // line comment + '/' => { + while self.preview_next().is_some() { + if self.next() == '\n' { + break; + } + } + self.next_line() + } + // block comment + '*' => { + let mut lines = 0; + loop { + if self.preview_next().is_none() { + return Err(LexerError::new("unterminated multiline comment")); + } + match self.next() { + '*' => { + if self.next_is('/') { + break; + } + } + next_ch => { + if next_ch == '\n' { + lines += 1; + } + }, + } + } + self.move_lines(lines); + } + // division, assigndiv or regex literal + _ => { + // if we fail to parse a regex literal, store a copy of the current + // buffer to restore later on + let original_buffer = self.buffer.clone(); + let original_pos = self.position; + // first, try to parse a regex literal + let mut body = String::new(); + let mut regex = false; + loop { + self.next_column(); + match self.buffer.next() { + // end of body + Some('/') => { + regex = true; + break; + } + // newline/eof not allowed in regex literal + n @ Some('\n') | n @ Some('\r') | n @ Some('\u{2028}') + | n @ Some('\u{2029}') => { + self.carriage_return(); + if n != Some('\r') { + self.next_line(); + } + break + }, + None => { + self.position = Position::new(self.position.line_number(), self.position.column_number()-1); + break + } + // escape sequence + Some('\\') => { + body.push('\\'); + if self.preview_next().is_none() { + break; + } + match self.next() { + // newline not allowed in regex literal + '\n' | '\r' | '\u{2028}' | '\u{2029}' => break, + ch => body.push(ch), + } + } + Some(ch) => body.push(ch), + } + } + if regex { + // body was parsed, now look for flags + let flags = self.take_char_while(char::is_alphabetic)?; + self.move_columns(body.len() as u32 + 1 + flags.len() as u32); + self.push_token(TokenKind::regular_expression_literal( + body, flags.parse()?, + ), start_pos); + } else { + // failed to parse regex, restore original buffer position and + // parse either div or assigndiv + self.buffer = original_buffer; + self.position = original_pos; + if self.next_is('=') { + self.push_token(TokenKind::Punctuator( + Punctuator::AssignDiv, + ), start_pos); + } else { + self.push_token(TokenKind::Punctuator(Punctuator::Div), start_pos); + } + } + } + } + } else { + return Err(LexerError::new("Expecting Token /,*,= or regex")); + } + } + '*' => op!(self, start_pos, Punctuator::AssignMul, Punctuator::Mul, { + '*' => vop!(self, Punctuator::AssignPow, Punctuator::Exp) + }), + '+' => op!(self, start_pos, Punctuator::AssignAdd, Punctuator::Add, { + '+' => Punctuator::Inc + }), + '-' => op!(self, start_pos, Punctuator::AssignSub, Punctuator::Sub, { + '-' => { + Punctuator::Dec + } + }), + '%' => op!(self, start_pos, Punctuator::AssignMod, Punctuator::Mod), + '|' => op!(self, start_pos, Punctuator::AssignOr, Punctuator::Or, { + '|' => Punctuator::BoolOr + }), + '&' => op!(self, start_pos, Punctuator::AssignAnd, Punctuator::And, { + '&' => Punctuator::BoolAnd + }), + '^' => op!(self, start_pos, Punctuator::AssignXor, Punctuator::Xor), + '=' => op!(self, start_pos, if self.next_is('=') { + Punctuator::StrictEq + } else { + Punctuator::Eq + }, Punctuator::Assign, { + '>' => { + Punctuator::Arrow + } + }), + '<' => op!(self, start_pos, Punctuator::LessThanOrEq, Punctuator::LessThan, { + '<' => vop!(self, Punctuator::AssignLeftSh, Punctuator::LeftSh) + }), + '>' => op!(self, start_pos, Punctuator::GreaterThanOrEq, Punctuator::GreaterThan, { + '>' => vop!(self, Punctuator::AssignRightSh, Punctuator::RightSh, { + '>' => vop!(self, Punctuator::AssignURightSh, Punctuator::URightSh) + }) + }), + '!' => op!( + self, + start_pos, + vop!(self, Punctuator::StrictNotEq, Punctuator::NotEq), + Punctuator::Not + ), + '~' => self.push_punc(Punctuator::Neg, start_pos), + '\n' | '\u{2028}' | '\u{2029}' => { + self.next_line(); + self.push_token(TokenKind::LineTerminator, start_pos); + } + '\r' => { + self.carriage_return(); + } + // The rust char::is_whitespace function and the ecma standard use different sets + // of characters as whitespaces: + // * Rust uses \p{White_Space}, + // * ecma standard uses \{Space_Separator} + \u{0009}, \u{000B}, \u{000C}, \u{FEFF} + // + // Explicit whitespace: see https://tc39.es/ecma262/#table-32 + '\u{0020}' | '\u{0009}' | '\u{000B}' | '\u{000C}' | '\u{00A0}' | '\u{FEFF}' | + // Unicode Space_Seperator category (minus \u{0020} and \u{00A0} which are allready stated above) + '\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{202F}' | '\u{205F}' | '\u{3000}' => (), + _ => { + let details = format!("Unexpected '{}' at line {}, column {}", start_pos.line_number(), start_pos.column_number(), ch); + return Err(LexerError { details }); + }, + } + } + } +} diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 568b54d526d..ca320e6bed7 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -3,870 +3,134 @@ //! The Lexer splits its input source code into a sequence of input elements called tokens, represented by the [Token](../ast/token/struct.Token.html) structure. //! It also removes whitespace and comments and attaches them to the next token. +mod comment; +mod cursor; +pub mod error; +mod string; #[cfg(test)] mod tests; -use crate::builtins::BigInt; -use crate::{ - syntax::ast::{ - token::{NumericLiteral, Token, TokenKind}, - Position, Punctuator, Span, - }, - BoaProfiler, +pub use self::error::Error; +use self::{cursor::Cursor, string::StringLiteral}; +use crate::syntax::ast::bigint::BigInt; +use crate::syntax::ast::{ + token::{NumericLiteral, Token, TokenKind}, + Position, Punctuator, Span, }; use std::{ char::{decode_utf16, from_u32}, - error, fmt, + fmt, + io::{self, BufRead, Bytes, Read, Seek}, iter::Peekable, str::{Chars, FromStr}, }; -/// `vop` tests the next token to see if we're on an assign operation of just a plain binary operation. -/// -/// If the next value is not an assignment operation it will pattern match the provided values and return the corresponding token. -macro_rules! vop { - ($this:ident, $assign_op:expr, $op:expr) => ({ - let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; - match preview { - '=' => { - $this.next(); - $this.next_column(); - $assign_op - } - _ => $op, - } - }); - ($this:ident, $assign_op:expr, $op:expr, {$($case:pat => $block:expr), +}) => ({ - let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; - match preview { - '=' => { - $this.next(); - $this.next_column(); - $assign_op - }, - $($case => { - $this.next(); - $this.next_column(); - $block - })+, - _ => $op - } - }); - ($this:ident, $op:expr, {$($case:pat => $block:expr),+}) => { - let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; - match preview { - $($case => { - $this.next()?; - $this.next_column(); - $block - })+, - _ => $op - } - } -} - -/// The `op` macro handles binary operations or assignment operations and converts them into tokens. -macro_rules! op { - ($this:ident, $start_pos:expr, $assign_op:expr, $op:expr) => ({ - let punc = vop!($this, $assign_op, $op); - $this.push_punc(punc, $start_pos); - }); - ($this:ident, $start_pos:expr, $assign_op:expr, $op:expr, {$($case:pat => $block:expr),+}) => ({ - let punc = vop!($this, $assign_op, $op, {$($case => $block),+}); - $this.push_punc(punc, $start_pos); - }); -} - -/// An error that occurred during lexing or compiling of the source input. -/// -/// [LexerError] implements [fmt::Display] so you just display this value as an error -#[derive(Debug, Clone)] -pub struct LexerError { - /// details will be displayed when a LexerError occurs. - details: String, -} - -impl LexerError { - /// Create a new LexerError struct - /// - /// * `msg` - The message to show when LexerError is displayed - pub(crate) fn new(msg: M) -> Self +trait Tokenizer { + /// Lexes the next token. + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where - M: Into, - { - Self { - details: msg.into(), - } - } + R: Read; } -impl fmt::Display for LexerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.details) - } -} - -impl error::Error for LexerError { - fn description(&self) -> &str { - &self.details - } - - fn cause(&self) -> Option<&dyn error::Error> { - // Generic error, underlying cause isn't tracked. - None - } -} - -/// A lexical analyzer for JavaScript source code. +/// Lexer or tokenizer for the Boa JavaScript Engine. #[derive(Debug)] -pub struct Lexer<'a> { - /// The list of tokens generated so far. - /// - /// This field is public so you can use them once lexing has finished. - pub tokens: Vec, - /// The current position in the source code. - position: Position, - /// The full Peekable buffer, an array of [Char]s - buffer: Peekable>, +pub(crate) struct Lexer { + cursor: Cursor, + goal_symbol: InputElement, } -impl<'a> Lexer<'a> { - /// Returns a Lexer with a buffer inside +impl Lexer { + /// Checks if a character is whitespace as per ECMAScript standards. + /// + /// The Rust `char::is_whitespace` function and the ECMAScript standard use different sets of + /// characters as whitespaces: + /// * Rust uses `\p{White_Space}`, + /// * ECMAScript standard uses `\{Space_Separator}` + `\u{0009}`, `\u{000B}`, `\u{000C}`, `\u{FEFF}` /// - /// The buffer needs to have a lifetime as long as the Lexer instance itself - pub fn new(buffer: &'a str) -> Lexer<'a> { - Lexer { - tokens: Vec::new(), - position: Position::new(1, 1), - buffer: buffer.chars().peekable(), + /// [More information](https://tc39.es/ecma262/#table-32) + fn is_whitespace(ch: char) -> bool { + match ch { + '\u{0020}' | '\u{0009}' | '\u{000B}' | '\u{000C}' | '\u{00A0}' | '\u{FEFF}' | + // Unicode Space_Seperator category (minus \u{0020} and \u{00A0} which are allready stated above) + '\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{202F}' | '\u{205F}' | '\u{3000}' => true, + _ => false, } } - /// Push a token onto the token queue. - fn push_token(&mut self, tk: TokenKind, start: Position) { - let end = if let TokenKind::LineTerminator = tk { - self.position - } else { - Position::new( - self.position.line_number(), - self.position.column_number() - 1, - ) - }; - self.tokens.push(Token::new(tk, Span::new(start, end))) - } - - /// Push a punctuation token - fn push_punc(&mut self, punc: Punctuator, start: Position) { - self.push_token(TokenKind::Punctuator(punc), start); - } - - /// Changes the current position by advancing to the next column. - fn next_column(&mut self) { - let pos = Position::new( - self.position.line_number(), - self.position.column_number() + 1, - ); - self.position = pos; - } - - /// Changes the current position by advancing the given number of columns. - fn move_columns(&mut self, columns: u32) { - let pos = Position::new( - self.position.line_number(), - self.position.column_number() + columns, - ); - self.position = pos; - } - - fn carriage_return(&mut self) { - let pos = Position::new(self.position.line_number(), 1); - self.position = pos; - } - - /// Changes the current position by advancing to the next line. - fn next_line(&mut self) { - let pos = Position::new(self.position.line_number() + 1, 1); - self.position = pos; - } - - /// Changes the current position by advancing the given number of lines. - fn move_lines(&mut self, lines: u32) { - let pos = Position::new(self.position.line_number() + lines, 1); - self.position = pos; - } - - /// next fetches the next token and return it, or a LexerError if there are no more. - fn next(&mut self) -> char { - self.buffer.next().expect( - "No more more characters to consume from input stream, \ - use preview_next() first to check before calling next()", - ) - } - - /// Preview the next character but don't actually increment - fn preview_next(&mut self) -> Option { - self.buffer.peek().copied() - } - - /// Preview a char x indexes further in buf, without incrementing - fn preview_multiple_next(&mut self, nb_next: usize) -> Option { - let mut next_peek = None; - - for (i, x) in self.buffer.clone().enumerate() { - if i >= nb_next { - break; - } - - next_peek = Some(x); - } - - next_peek + /// Sets the goal symbol for the lexer. + pub(crate) fn set_goal(&mut self, elm: InputElement) { + self.goal_symbol = elm; } +} - /// Utility Function, while ``f(char)`` is true, read chars and move curser. - /// All chars are returned as a string - fn take_char_while(&mut self, mut f: F) -> Result - where - F: FnMut(char) -> bool, - { - let mut s = String::new(); - while self.buffer.peek().is_some() - && f(self.preview_next().expect("Could not preview next value")) - { - s.push(self.next()); +impl Lexer +where + R: Read, +{ + /// Creates a new lexer. + #[inline] + pub(crate) fn new(reader: R) -> Self { + Self { + cursor: Cursor::new(reader), + goal_symbol: Default::default(), } - - Ok(s) } +} - /// Compares the character passed in to the next character, if they match true is returned and the buffer is incremented - fn next_is(&mut self, peek: char) -> bool { - let result = self.preview_next() == Some(peek); - if result { - self.next_column(); - self.buffer.next(); - } - result - } +/// ECMAScript goal symbols. +/// +/// +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InputElement { + Div, + RegExp, + RegExpOrTemplateTail, + TemplateTail, +} - /// Utility function for checkint the NumericLiteral is not followed by an `IdentifierStart` or `DecimalDigit` character. - /// - /// More information: - /// - [ECMAScript Specification][spec] - /// - /// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals - fn check_after_numeric_literal(&mut self) -> Result<(), LexerError> { - match self.preview_next() { - Some(ch) - if ch.is_ascii_alphabetic() || ch == '$' || ch == '_' || ch.is_ascii_digit() => - { - Err(LexerError::new("NumericLiteral token must not be followed by IdentifierStart nor DecimalDigit characters")) - } - Some(_) => Ok(()), - None => Ok(()) - } +impl Default for InputElement { + fn default() -> Self { + todo!("what is the default input element?") } +} - /// Lexes a numerical literal. - /// - /// More information: - /// - [ECMAScript Specification][spec] - /// - /// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals - fn reed_numerical_literal(&mut self, ch: char) -> Result<(), LexerError> { - /// This is a helper structure - /// - /// This structure helps with identifying what numerical type it is and what base is it. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - enum NumericKind { - Rational, - Integer(u8), - BigInt(u8), - } - - impl NumericKind { - /// Get the base of the number kind. - fn base(self) -> u32 { - match self { - Self::Rational => 10, - Self::Integer(base) => base as u32, - Self::BigInt(base) => base as u32, - } - } - - /// Converts `self` to BigInt kind. - fn to_bigint(self) -> Self { - match self { - Self::Rational => unreachable!("can not convert rational number to BigInt"), - Self::Integer(base) => Self::BigInt(base), - Self::BigInt(base) => Self::BigInt(base), - } - } - } - - // TODO: Setup strict mode. - let strict_mode = false; - - let mut buf = ch.to_string(); - let mut kind = NumericKind::Integer(10); - let start_pos = self.position; - if ch == '0' { - match self.preview_next() { - None => { - self.next_column(); - self.push_token( - TokenKind::NumericLiteral(NumericLiteral::Integer(0)), - start_pos, - ); - return Ok(()); - } - Some('x') | Some('X') => { - self.next(); - self.next_column(); - kind = NumericKind::Integer(16); - } - Some('o') | Some('O') => { - self.next(); - self.next_column(); - kind = NumericKind::Integer(8); - } - Some('b') | Some('B') => { - self.next(); - self.next_column(); - kind = NumericKind::Integer(2); - } - Some(ch) if ch.is_ascii_digit() => { - let mut is_implicit_octal = true; - while let Some(ch) = self.preview_next() { - if !ch.is_ascii_digit() { - break; - } else if !ch.is_digit(8) { - is_implicit_octal = false; - } - buf.push(self.next()); - } - if !strict_mode { - if is_implicit_octal { - kind = NumericKind::Integer(8); - } - } else { - return Err(if is_implicit_octal { - LexerError::new( - "Implicit octal literals are not allowed in strict mode.", - ) - } else { - LexerError::new( - "Decimals with leading zeros are not allowed in strict mode.", - ) - }); - } - } - Some(_) => {} - } - } - - while let Some(ch) = self.preview_next() { - if !ch.is_digit(kind.base()) { - break; - } - buf.push(self.next()); - } - - if self.next_is('n') { - kind = kind.to_bigint(); - } - - if let NumericKind::Integer(10) = kind { - 'digitloop: while let Some(ch) = self.preview_next() { - match ch { - '.' => loop { - kind = NumericKind::Rational; - buf.push(self.next()); - - let c = match self.preview_next() { - Some(ch) => ch, - None => break, - }; - - match c { - 'e' | 'E' => { - match self - .preview_multiple_next(2) - .unwrap_or_default() - .to_digit(10) - { - Some(0..=9) | None => { - buf.push(self.next()); - } - _ => { - break 'digitloop; - } - } - } - _ => { - if !c.is_digit(10) { - break 'digitloop; - } - } - } - }, - 'e' | 'E' => { - kind = NumericKind::Rational; - match self - .preview_multiple_next(2) - .unwrap_or_default() - .to_digit(10) - { - Some(0..=9) | None => { - buf.push(self.next()); - } - _ => { - break; - } - } - buf.push(self.next()); - } - '+' | '-' => { - break; - } - _ if ch.is_digit(10) => { - buf.push(self.next()); - } - _ => break, - } - } - } - - self.check_after_numeric_literal()?; - - let num = match kind { - NumericKind::BigInt(base) => { - NumericLiteral::BigInt( - BigInt::from_string_radix(&buf, base as u32).expect("Could not conver to BigInt") - ) - } - NumericKind::Rational /* base: 10 */ => { - NumericLiteral::Rational( - f64::from_str(&buf) - .map_err(|_| LexerError::new("Could not convert value to f64"))?, - ) - } - NumericKind::Integer(base) => { - if let Ok(num) = i32::from_str_radix(&buf, base as u32) { - NumericLiteral::Integer( - num - ) - } else { - let b = f64::from(base); - let mut result = 0.0_f64; - for c in buf.chars() { - let digit = f64::from(c.to_digit(base as u32).unwrap()); - result = result * b + digit; - } - - NumericLiteral::Rational(result) - } - - } +impl Iterator for Lexer +where + R: Read, +{ + type Item = Result; + + fn next(&mut self) -> Option { + let (start, next_chr) = loop { + let start = self.cursor.pos(); + let next_chr = match self.cursor.next()? { + Ok(c) => c, + Err(e) => return Some(Err(e.into())), }; - self.move_columns(buf.len() as u32); - self.push_token(TokenKind::NumericLiteral(num), start_pos); - - Ok(()) - } - - /// Runs the lexer until completion, returning a [LexerError] if there's a syntax issue, or an empty unit result - /// - /// # Example - /// - /// ``` - /// # use boa::syntax::lexer::{LexerError, Lexer}; - /// fn main() -> Result<(), LexerError> { - /// let buffer = String::from("Hello World"); - /// let mut lexer = Lexer::new(&buffer); - /// lexer.lex() - /// } - /// ``` - pub fn lex(&mut self) -> Result<(), LexerError> { - let _timer = BoaProfiler::global().start_event("lex", "lexing"); - loop { - // Check if we've reached the end - if self.preview_next().is_none() { - return Ok(()); + // Ignore whitespace + if !Self::is_whitespace(next_chr) { + break (start, next_chr); } - let start_pos = self.position; - self.next_column(); - let ch = self.next(); - match ch { - // StringLiteral - '"' | '\'' => { - let mut buf = String::new(); - loop { - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated String")); - } - match self.next() { - '\'' if ch == '\'' => { - break; - } - '"' if ch == '"' => { - break; - } - '\\' => { - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated String")); - } - let escape_pos = self.position; - let escape = self.next(); - if escape != '\n' { - let escaped_ch = match escape { - 'n' => '\n', - 'r' => '\r', - 't' => '\t', - 'b' => '\x08', - 'f' => '\x0c', - '0' => '\0', - 'x' => { - let mut nums = String::with_capacity(2); - for _ in 0_u8..2 { - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated String")); - } - nums.push(self.next()); - } - self.move_columns(2); - let as_num = match u64::from_str_radix(&nums, 16) { - Ok(v) => v, - Err(_) => 0, - }; - match from_u32(as_num as u32) { - Some(v) => v, - None => panic!( - "{}: {} is not a valid unicode scalar value", - self.position, as_num - ), - } - } - 'u' => { - // There are 2 types of codepoints. Surragate codepoints and unicode codepoints. - // UTF-16 could be surrogate codepoints, "\uXXXX\uXXXX" which make up a single unicode codepoint. - // We will need to loop to make sure we catch all UTF-16 codepoints - // Example Test: https://github.com/tc39/test262/blob/ee3715ee56744ccc8aeb22a921f442e98090b3c1/implementation-contributed/v8/mjsunit/es6/unicode-escapes.js#L39-L44 - - // Support \u{X..X} (Unicode Codepoint) - if self.next_is('{') { - let s = self - .take_char_while(char::is_alphanumeric) - .expect("Could not read chars"); - - // We know this is a single unicode codepoint, convert to u32 - let as_num = match u32::from_str_radix(&s, 16) { - Ok(v) => v, - Err(_) => 0, - }; - let c = from_u32(as_num).ok_or_else(|| LexerError::new("Invalid Unicode escape sequence"))?; - - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated String")); - } - self.next(); // '}' - self.move_columns(s.len() as u32); - c - } else { - let mut codepoints: Vec = vec![]; - loop { - // Collect each character after \u e.g \uD83D will give "D83D" - let s = self - .take_char_while(char::is_alphanumeric) - .expect("Could not read chars"); - - // Convert to u16 - let as_num = match u16::from_str_radix(&s, 16) { - Ok(v) => v, - Err(_) => 0, - }; - - codepoints.push(as_num); - self.move_columns(s.len() as u32); - - // Check for another UTF-16 codepoint - if self.next_is('\\') && self.next_is('u') { - continue; - } - break; - } - - // codepoints length should either be 1 (unicode codepoint) or 2 (surrogate codepoint). - // Rust's decode_utf16 will deal with it regardless - decode_utf16(codepoints.iter().cloned()) - .next() - .expect("Could not get next codepoint") - .expect("Could not get next codepoint") - } - } - '\'' | '"' | '\\' => escape, - ch => { - let details = format!("invalid escape sequence `{}` at line {}, column {}", escape_pos.line_number(), escape_pos.column_number(), ch); - return Err(LexerError { details }); - } - }; - buf.push(escaped_ch); - } - } - next_ch => buf.push(next_ch), - } - } - let str_length = buf.len() as u32; - // Why +1? Quotation marks are not included, - // So technically it would be +2, (for both " ") but we want to be 1 less - // to compensate for the incrementing at the top - self.move_columns( str_length.wrapping_add(1)); - self.push_token(TokenKind::string_literal(buf), start_pos); - } - // TemplateLiteral - '`' => { - let mut buf = String::new(); - loop { - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated template literal")); - } - match self.next() { - '`' => { - break; - } - next_ch => buf.push(next_ch), - // TODO when there is an expression inside the literal - } - } - let str_length = buf.len() as u32; - // Why +1? Quotation marks are not included, - // So technically it would be +2, (for both " ") but we want to be 1 less - // to compensate for the incrementing at the top - self.move_columns( str_length.wrapping_add(1)); - self.push_token(TokenKind::template_literal(buf), start_pos); - } - _ if ch.is_digit(10) => self.reed_numerical_literal(ch)?, - _ if ch.is_alphabetic() || ch == '$' || ch == '_' => { - let mut buf = ch.to_string(); - while let Some(ch) = self.preview_next() { - if ch.is_alphabetic() || ch.is_digit(10) || ch == '_' { - buf.push(self.next()); - } else { - break; - } - } - let tk = match buf.as_str() { - "true" => TokenKind::BooleanLiteral(true), - "false" => TokenKind::BooleanLiteral(false), - "null" => TokenKind::NullLiteral, - "NaN" => TokenKind::NumericLiteral(NumericLiteral::Rational(f64::NAN)), - slice => { - if let Ok(keyword) = FromStr::from_str(slice) { - TokenKind::Keyword(keyword) - } else { - TokenKind::identifier(slice) - } - } - }; + }; - // Move position forward the length of the token - self.move_columns( (buf.len().wrapping_sub(1)) as u32); + let token = match next_chr { + '\r' | '\n' | '\u{2028}' | '\u{2029}' => Ok(Token::new( + TokenKind::LineTerminator, + Span::new(start, self.cursor.pos()), + )), + '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), + _ => unimplemented!(), + }; - self.push_token(tk, start_pos); - } - ';' => self.push_punc(Punctuator::Semicolon, start_pos), - ':' => self.push_punc(Punctuator::Colon, start_pos), - '.' => { - // . or ... - if self.next_is('.') { - if self.next_is('.') { - self.push_punc(Punctuator::Spread, start_pos); - } else { - return Err(LexerError::new("Expecting Token .")); - } - } else { - self.push_punc(Punctuator::Dot, start_pos); - }; - } - '(' => self.push_punc(Punctuator::OpenParen, start_pos), - ')' => self.push_punc(Punctuator::CloseParen, start_pos), - ',' => self.push_punc(Punctuator::Comma, start_pos), - '{' => self.push_punc(Punctuator::OpenBlock, start_pos), - '}' => self.push_punc(Punctuator::CloseBlock, start_pos), - '[' => self.push_punc(Punctuator::OpenBracket, start_pos), - ']' => self.push_punc(Punctuator::CloseBracket, start_pos), - '?' => self.push_punc(Punctuator::Question, start_pos), - // Comments - '/' => { - if let Some(ch) = self.preview_next() { - match ch { - // line comment - '/' => { - while self.preview_next().is_some() { - if self.next() == '\n' { - break; - } - } - self.next_line() - } - // block comment - '*' => { - let mut lines = 0; - loop { - if self.preview_next().is_none() { - return Err(LexerError::new("unterminated multiline comment")); - } - match self.next() { - '*' => { - if self.next_is('/') { - break; - } - } - next_ch => { - if next_ch == '\n' { - lines += 1; - } - }, - } - } - self.move_lines(lines); - } - // division, assigndiv or regex literal - _ => { - // if we fail to parse a regex literal, store a copy of the current - // buffer to restore later on - let original_buffer = self.buffer.clone(); - let original_pos = self.position; - // first, try to parse a regex literal - let mut body = String::new(); - let mut regex = false; - loop { - self.next_column(); - match self.buffer.next() { - // end of body - Some('/') => { - regex = true; - break; - } - // newline/eof not allowed in regex literal - n @ Some('\n') | n @ Some('\r') | n @ Some('\u{2028}') - | n @ Some('\u{2029}') => { - self.carriage_return(); - if n != Some('\r') { - self.next_line(); - } - break - }, - None => { - self.position = Position::new(self.position.line_number(), self.position.column_number()-1); - break - } - // escape sequence - Some('\\') => { - body.push('\\'); - if self.preview_next().is_none() { - break; - } - match self.next() { - // newline not allowed in regex literal - '\n' | '\r' | '\u{2028}' | '\u{2029}' => break, - ch => body.push(ch), - } - } - Some(ch) => body.push(ch), - } - } - if regex { - // body was parsed, now look for flags - let flags = self.take_char_while(char::is_alphabetic)?; - self.move_columns(body.len() as u32 + 1 + flags.len() as u32); - self.push_token(TokenKind::regular_expression_literal( - body, flags.parse()?, - ), start_pos); - } else { - // failed to parse regex, restore original buffer position and - // parse either div or assigndiv - self.buffer = original_buffer; - self.position = original_pos; - if self.next_is('=') { - self.push_token(TokenKind::Punctuator( - Punctuator::AssignDiv, - ), start_pos); - } else { - self.push_token(TokenKind::Punctuator(Punctuator::Div), start_pos); - } - } - } - } - } else { - return Err(LexerError::new("Expecting Token /,*,= or regex")); - } - } - '*' => op!(self, start_pos, Punctuator::AssignMul, Punctuator::Mul, { - '*' => vop!(self, Punctuator::AssignPow, Punctuator::Exp) - }), - '+' => op!(self, start_pos, Punctuator::AssignAdd, Punctuator::Add, { - '+' => Punctuator::Inc - }), - '-' => op!(self, start_pos, Punctuator::AssignSub, Punctuator::Sub, { - '-' => { - Punctuator::Dec - } - }), - '%' => op!(self, start_pos, Punctuator::AssignMod, Punctuator::Mod), - '|' => op!(self, start_pos, Punctuator::AssignOr, Punctuator::Or, { - '|' => Punctuator::BoolOr - }), - '&' => op!(self, start_pos, Punctuator::AssignAnd, Punctuator::And, { - '&' => Punctuator::BoolAnd - }), - '^' => op!(self, start_pos, Punctuator::AssignXor, Punctuator::Xor), - '=' => op!(self, start_pos, if self.next_is('=') { - Punctuator::StrictEq - } else { - Punctuator::Eq - }, Punctuator::Assign, { - '>' => { - Punctuator::Arrow - } - }), - '<' => op!(self, start_pos, Punctuator::LessThanOrEq, Punctuator::LessThan, { - '<' => vop!(self, Punctuator::AssignLeftSh, Punctuator::LeftSh) - }), - '>' => op!(self, start_pos, Punctuator::GreaterThanOrEq, Punctuator::GreaterThan, { - '>' => vop!(self, Punctuator::AssignRightSh, Punctuator::RightSh, { - '>' => vop!(self, Punctuator::AssignURightSh, Punctuator::URightSh) - }) - }), - '!' => op!( - self, - start_pos, - vop!(self, Punctuator::StrictNotEq, Punctuator::NotEq), - Punctuator::Not - ), - '~' => self.push_punc(Punctuator::Neg, start_pos), - '\n' | '\u{2028}' | '\u{2029}' => { - self.next_line(); - self.push_token(TokenKind::LineTerminator, start_pos); - } - '\r' => { - self.carriage_return(); - } - // The rust char::is_whitespace function and the ecma standard use different sets - // of characters as whitespaces: - // * Rust uses \p{White_Space}, - // * ecma standard uses \{Space_Separator} + \u{0009}, \u{000B}, \u{000C}, \u{FEFF} - // - // Explicit whitespace: see https://tc39.es/ecma262/#table-32 - '\u{0020}' | '\u{0009}' | '\u{000B}' | '\u{000C}' | '\u{00A0}' | '\u{FEFF}' | - // Unicode Space_Seperator category (minus \u{0020} and \u{00A0} which are allready stated above) - '\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{202F}' | '\u{205F}' | '\u{3000}' => (), - _ => { - let details = format!("Unexpected '{}' at line {}, column {}", start_pos.line_number(), start_pos.column_number(), ch); - return Err(LexerError { details }); - }, - } - } + Some(token) } } + +// impl Tokenizer for Lexer { +// fn lex(&mut self, cursor: &mut Cursor) -> io::Result +// where +// R: Read, +// { +// } +// } diff --git a/boa/src/syntax/lexer/string.rs b/boa/src/syntax/lexer/string.rs new file mode 100644 index 00000000000..492666c5a37 --- /dev/null +++ b/boa/src/syntax/lexer/string.rs @@ -0,0 +1,189 @@ +use super::{Cursor, Error, Tokenizer}; +use crate::syntax::ast::{ + token::{Token, TokenKind}, + Position, Span, +}; +use std::{ + char::{decode_utf16, from_u32}, + convert::TryFrom, + io::{self, ErrorKind, Read}, + str, +}; + +/// String literal lexing. +/// +/// Note: expects for the initializer `'` or `"` to already be consumed from the cursor. +#[derive(Debug, Clone, Copy)] +pub(super) struct StringLiteral { + terminator: StringTerminator, +} + +impl StringLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char) -> Self { + let terminator = match init { + '\'' => StringTerminator::SingleQuote, + '"' => StringTerminator::DoubleQuote, + _ => unreachable!(), + }; + + Self { terminator } + } +} + +/// Terminator for the string. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StringTerminator { + SingleQuote, + DoubleQuote, +} + +impl Tokenizer for StringLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + let mut buf = String::new(); + loop { + let next_chr_start = cursor.pos(); + let next_chr = cursor.next().ok_or_else(|| { + Error::from(io::Error::new( + ErrorKind::UnexpectedEof, + "unterminated string literal", + )) + })??; + + match next_chr { + '\'' if self.terminator == StringTerminator::SingleQuote => { + break; + } + '"' if self.terminator == StringTerminator::DoubleQuote => { + break; + } + '\\' => { + let escape = cursor.next().ok_or_else(|| { + Error::from(io::Error::new( + ErrorKind::UnexpectedEof, + "unterminated escape sequence in string literal", + )) + })??; + if escape != '\n' { + let escaped_ch = match escape { + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'b' => '\x08', + 'f' => '\x0c', + '0' => '\0', + 'x' => { + // TODO: optimize by getting just bytes + let mut nums = String::with_capacity(2); + for _ in 0_u8..2 { + let next = cursor.next().ok_or_else(|| { + Error::from(io::Error::new( + ErrorKind::UnexpectedEof, + "unterminated escape sequence in string literal", + )) + })??; + nums.push(next); + } + let as_num = match u64::from_str_radix(&nums, 16) { + Ok(v) => v, + Err(_) => 0, + }; + match from_u32(as_num as u32) { + Some(v) => v, + None => { + return Err(Error::syntax(format!( + "{}: {} is not a valid Unicode scalar value", + cursor.pos(), + as_num + ))) + } + } + } + 'u' => { + // There are 2 types of codepoints. Surragate codepoints and + // unicode codepoints. UTF-16 could be surrogate codepoints, + // "\uXXXX\uXXXX" which make up a single unicode codepoint. We will + // need to loop to make sure we catch all UTF-16 codepoints + + // Support \u{X..X} (Unicode Codepoint) + if cursor.next_is('{')? { + // The biggest code point is 0x10FFFF + let mut code_point = String::with_capacity(6); + cursor.take_until('}', &mut code_point)?; + + // We know this is a single unicode codepoint, convert to u32 + let as_num = + u32::from_str_radix(&code_point, 16).map_err(|_| { + Error::syntax( + "malformed Unicode character escape sequence", + ) + })?; + if as_num > 0x10_FFFF { + return Err(Error::syntax("Unicode codepoint must not be greater than 0x10FFFF in escape sequence")); + } + char::try_from(as_num).map_err(|_| { + Error::syntax("invalid Unicode escape sequence") + })? + } else { + let mut codepoints: Vec = vec![]; + loop { + // Collect each character after \u e.g \uD83D will give "D83D" + let mut code_point = [0u8; 4]; + cursor.fill_bytes(&mut code_point)?; + + // Convert to u16 + let as_num = match u16::from_str_radix( + str::from_utf8(&code_point) + .expect("the cursor returned invalid UTF-8"), + 16, + ) { + Ok(v) => v, + Err(_) => 0, + }; + + codepoints.push(as_num); + + // Check for another UTF-16 codepoint + if cursor.next_is('\\')? && cursor.next_is('u')? { + continue; + } + break; + } + + // codepoints length should either be 1 (unicode codepoint) or + // 2 (surrogate codepoint). Rust's decode_utf16 will deal with + // it regardless + // TODO: do not panic with invalid code points. + decode_utf16(codepoints.iter().copied()) + .next() + .expect("Could not get next codepoint") + .expect("Could not get next codepoint") + } + } + '\'' | '"' | '\\' => escape, + ch => { + let details = format!( + "invalid escape sequence `{}` at line {}, column {}", + next_chr_start.line_number(), + next_chr_start.column_number(), + ch + ); + return Err(Error::syntax(details)); + } + }; + buf.push(escaped_ch); + } + } + next_ch => buf.push(next_ch), + } + } + + Ok(Token::new( + TokenKind::string_literal(buf), + Span::new(start_pos, cursor.pos()), + )) + } +} From c1318ec21c8f19f7a18f9bda5d09a87abfd92a32 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Thu, 11 Jun 2020 00:35:44 +0100 Subject: [PATCH 002/289] Minimal amount to allow compiling --- boa/src/lib.rs | 12 ++++-- boa/src/syntax/lexer/comment.rs | 12 +----- boa/src/syntax/lexer/cursor.rs | 12 +----- boa/src/syntax/lexer/error.rs | 8 ++-- boa/src/syntax/lexer/mod.rs | 48 +++++++++++++++-------- boa/src/syntax/lexer/template.rs | 65 ++++++++++++++++++++++++++++++++ boa/src/syntax/lexer/tests.rs | 2 +- boa/src/syntax/parser/tests.rs | 24 +++++++++--- boa_cli/src/main.rs | 14 +++++-- boa_wasm/src/lib.rs | 13 ++++--- 10 files changed, 151 insertions(+), 59 deletions(-) create mode 100644 boa/src/syntax/lexer/template.rs diff --git a/boa/src/lib.rs b/boa/src/lib.rs index 4314d77f794..f43d27e1efe 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -50,9 +50,15 @@ pub use crate::{ }; fn parser_expr(src: &str) -> Result { - let mut lexer = Lexer::new(src); - lexer.lex().map_err(|e| format!("Syntax Error: {}", e))?; - let tokens = lexer.tokens; + let mut lexer = Lexer::new(src.as_bytes()); + + // Goes through and lexes entire given string before starting any parsing. + let mut tokens = Vec::new(); + + for token in lexer { + tokens.push(token.map_err(|e| format!("Lexing Error: {}", e))?); + } + Parser::new(&tokens) .parse_all() .map_err(|e| format!("Parsing Error: {}", e)) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 500852fb73c..fea54c7d372 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -1,17 +1,9 @@ //! Coments lexing. use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::bigint::BigInt; use crate::syntax::ast::{ - token::{NumericLiteral, Token, TokenKind}, - Position, Punctuator, Span, -}; -use std::{ - char::{decode_utf16, from_u32}, - fmt, - io::{self, BufRead, Bytes, Read, Seek}, - iter::Peekable, - str::{Chars, FromStr}, + token::Token, + Position, }; /// Lexes single line comments, starting with `//`. diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 66649524fde..9af3e826e2c 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -1,14 +1,6 @@ -use crate::syntax::ast::bigint::BigInt; -use crate::syntax::ast::{ - token::{NumericLiteral, Token, TokenKind}, - Position, Punctuator, Span, -}; +use crate::syntax::ast::Position; use std::{ - char::{decode_utf16, from_u32}, - error, fmt, - io::{self, BufRead, Bytes, Read, Seek}, - iter::Peekable, - str::{Chars, FromStr}, + io::{self, Bytes, Read} }; /// Cursor over the source code. diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 525861f2e18..0af65c633be 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -14,7 +14,7 @@ impl From for Error { impl Error { /// Creates a new syntax error. - pub(super) fn syntax(err: M) -> Self + pub(crate) fn syntax(err: M) -> Self where M: Into>, { @@ -24,7 +24,7 @@ impl Error { impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { + match self { Self::IO(e) => write!(f, "I/O error: {}", e), Self::Syntax(e) => write!(f, "Syntax Error: {}", e), } @@ -33,8 +33,8 @@ impl fmt::Display for Error { impl StdError for Error { fn source(&self) -> Option<&(dyn StdError + 'static)> { - match *self { - Self::IO(err) => Some(&err), + match self { + Self::IO(err) => Some(err), Self::Syntax(_) => None, } } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index ca320e6bed7..9d6b4d4de90 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -7,23 +7,22 @@ mod comment; mod cursor; pub mod error; mod string; -#[cfg(test)] -mod tests; + +#[macro_use] +mod template; + +// Temporary disabled while lexer in progress. +// #[cfg(test)] +// mod tests; pub use self::error::Error; -use self::{cursor::Cursor, string::StringLiteral}; -use crate::syntax::ast::bigint::BigInt; + +use self::{cursor::Cursor, string::StringLiteral, template::TemplateLiteral}; use crate::syntax::ast::{ - token::{NumericLiteral, Token, TokenKind}, - Position, Punctuator, Span, -}; -use std::{ - char::{decode_utf16, from_u32}, - fmt, - io::{self, BufRead, Bytes, Read, Seek}, - iter::Peekable, - str::{Chars, FromStr}, + token::{Token, TokenKind}, + Position, Span, }; +use std::io::Read; trait Tokenizer { /// Lexes the next token. @@ -34,7 +33,7 @@ trait Tokenizer { /// Lexer or tokenizer for the Boa JavaScript Engine. #[derive(Debug)] -pub(crate) struct Lexer { +pub struct Lexer { cursor: Cursor, goal_symbol: InputElement, } @@ -69,7 +68,7 @@ where { /// Creates a new lexer. #[inline] - pub(crate) fn new(reader: R) -> Self { + pub fn new(reader: R) -> Self { Self { cursor: Cursor::new(reader), goal_symbol: Default::default(), @@ -120,6 +119,7 @@ where Span::new(start, self.cursor.pos()), )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), + template_match!() => TemplateLiteral::new().lex(&mut self.cursor, start), _ => unimplemented!(), }; @@ -128,9 +128,25 @@ where } // impl Tokenizer for Lexer { -// fn lex(&mut self, cursor: &mut Cursor) -> io::Result +// fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> io::Result // where // R: Read, // { + // } // } + + +// Temporarily moved. +use crate::syntax::ast::Keyword; + +#[test] +fn check_single_line_comment() { + let s1 = "var \n//This is a comment\ntrue"; + let mut lexer = Lexer::new(s1.as_bytes()); + + assert_eq!(lexer.next().unwrap().unwrap().kind, TokenKind::Keyword(Keyword::Var)); + assert_eq!(lexer.next().unwrap().unwrap().kind, TokenKind::LineTerminator); + assert_eq!(lexer.next().unwrap().unwrap().kind, TokenKind::BooleanLiteral(true)); + assert!(lexer.next().is_none()); +} \ No newline at end of file diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs new file mode 100644 index 00000000000..ca2ca28a067 --- /dev/null +++ b/boa/src/syntax/lexer/template.rs @@ -0,0 +1,65 @@ +use super::{Cursor, Error, Tokenizer}; +use crate::syntax::ast::{ + token::{Token, TokenKind}, + Position, Span, +}; +use std::{ + io::{self, ErrorKind, Read} +}; + +macro_rules! template_match { + () => { + '`' + }; +} + +/// Template literal parsing. +/// +/// Expects: Initial ` to already be consumed by cursor. +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: +/// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals +#[derive(Debug, Clone, Copy)] +pub(super) struct TemplateLiteral { + +} + +impl TemplateLiteral { + /// Creates a new string literal lexer. + pub(super) fn new() -> Self { + Self { + + } + } +} + +impl Tokenizer for TemplateLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + let mut buf = String::new(); + loop { + match cursor.next() { + None => { + return Err(Error::from(io::Error::new( + ErrorKind::UnexpectedEof, + "Unterminated template literal", + ))); + } + Some(Err(e)) => return Err(Error::from(e)), + Some(Ok('`')) => break, // Template literal finished. + Some(Ok(next_ch)) => buf.push(next_ch), // TODO when there is an expression inside the literal + } + } + + Ok(Token::new( + TokenKind::template_literal(buf), + Span::new(start_pos, cursor.pos()), + )) + } +} \ No newline at end of file diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 4129bb3c179..30f3fa06348 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -543,4 +543,4 @@ fn addition_no_spaces_e_number() { lexer.tokens[2].kind, TokenKind::numeric_literal(100_000_000_000.0) ); -} +} */ \ No newline at end of file diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index c5e83d03d26..e77b9d046f3 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -20,11 +20,17 @@ pub(super) fn check_parser(js: &str, expr: L) where L: Into>, { - let mut lexer = Lexer::new(js); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new(js.as_bytes()); + + // Goes through and lexes entire given string. + let mut tokens = Vec::new(); + + for token in lexer { + tokens.push(token.expect("failed to lex")); + } assert_eq!( - Parser::new(&lexer.tokens) + Parser::new(&tokens) .parse_all() .expect("failed to parse"), StatementList::from(expr) @@ -34,10 +40,16 @@ where /// Checks that the given javascript string creates a parse error. // TODO: #[track_caller]: https://github.com/rust-lang/rust/issues/47809 pub(super) fn check_invalid(js: &str) { - let mut lexer = Lexer::new(js); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new(js.as_bytes()); + // lexer.lex().expect("failed to lex"); + + let mut tokens = Vec::new(); + + for token in lexer { + tokens.push(token.expect("failed to lex")); + } - assert!(Parser::new(&lexer.tokens).parse_all().is_err()); + assert!(Parser::new(&tokens).parse_all().is_err()); } /// Should be parsed as `new Class().method()` instead of `new (Class().method())` diff --git a/boa_cli/src/main.rs b/boa_cli/src/main.rs index a24c941a39c..10507bde2f1 100644 --- a/boa_cli/src/main.rs +++ b/boa_cli/src/main.rs @@ -31,6 +31,7 @@ use boa::{ forward_val, realm::Realm, syntax::ast::{node::StatementList, token::Token}, + syntax::lexer::Lexer, }; use std::{ fs::read_to_string, @@ -115,11 +116,16 @@ arg_enum! { /// Returns a error of type String with a message, /// if the source has a syntax error. fn lex_source(src: &str) -> Result, String> { - use boa::syntax::lexer::Lexer; + let mut lexer = Lexer::new(src.as_bytes()); - let mut lexer = Lexer::new(src); - lexer.lex().map_err(|e| format!("SyntaxError: {}", e))?; - Ok(lexer.tokens) + // Goes through and lexes entire given string. + let mut tokens = Vec::new(); + + for token in lexer { + tokens.push(token.map_err(|e| format!("Lexing Error: {}", e))?); + } + + Ok(tokens) } /// Parses the the token stream into a ast and returns it. diff --git a/boa_wasm/src/lib.rs b/boa_wasm/src/lib.rs index 97eab2e5c61..2d9d2e506fe 100644 --- a/boa_wasm/src/lib.rs +++ b/boa_wasm/src/lib.rs @@ -3,12 +3,15 @@ use wasm_bindgen::prelude::*; #[wasm_bindgen] pub fn evaluate(src: &str) -> Result { - let mut lexer = Lexer::new(src); - lexer - .lex() - .map_err(|e| JsValue::from(format!("Syntax Error: {}", e)))?; + let mut lexer = Lexer::new(src.as_bytes()); - let tokens = lexer.tokens; + // Goes through and lexes entire given string. + let mut tokens = Vec::new(); + + for token in lexer { + tokens.push(token.map_err(|e| format!("Lexing Error: {}", e))?); + } + let expr = Parser::new(&tokens) .parse_all() .map_err(|e| JsValue::from(format!("Parsing Error: {}", e)))?; From 119f26f46d07045319fd17f0e06d4b8144ca11d5 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Thu, 11 Jun 2020 11:47:53 +0100 Subject: [PATCH 003/289] Update boa_wasm/src/lib.rs Co-authored-by: Iban Eguia --- boa_wasm/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/boa_wasm/src/lib.rs b/boa_wasm/src/lib.rs index 2d9d2e506fe..d60cbb0c3d8 100644 --- a/boa_wasm/src/lib.rs +++ b/boa_wasm/src/lib.rs @@ -6,11 +6,7 @@ pub fn evaluate(src: &str) -> Result { let mut lexer = Lexer::new(src.as_bytes()); // Goes through and lexes entire given string. - let mut tokens = Vec::new(); - - for token in lexer { - tokens.push(token.map_err(|e| format!("Lexing Error: {}", e))?); - } + let tokens = lexer.collect::, _>().map_err(|e| format!("Lexing Error: {}", e))?; let expr = Parser::new(&tokens) .parse_all() From d4e3e347a5efe5b9cf396aa13cfa04d2621dde73 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Thu, 11 Jun 2020 11:48:08 +0100 Subject: [PATCH 004/289] Update boa/src/syntax/parser/tests.rs Co-authored-by: Iban Eguia --- boa/src/syntax/parser/tests.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index e77b9d046f3..59cdf221701 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -43,11 +43,7 @@ pub(super) fn check_invalid(js: &str) { let mut lexer = Lexer::new(js.as_bytes()); // lexer.lex().expect("failed to lex"); - let mut tokens = Vec::new(); - - for token in lexer { - tokens.push(token.expect("failed to lex")); - } + let tokens = lexer.collect::, _>().expect("failed to lex"); assert!(Parser::new(&tokens).parse_all().is_err()); } From 30c81fce74f2f61ff4a7bbb8db2d0f265f0a34bb Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Thu, 11 Jun 2020 11:48:15 +0100 Subject: [PATCH 005/289] Update boa_cli/src/main.rs Co-authored-by: Iban Eguia --- boa_cli/src/main.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/boa_cli/src/main.rs b/boa_cli/src/main.rs index 10507bde2f1..d7e2b3ec8c0 100644 --- a/boa_cli/src/main.rs +++ b/boa_cli/src/main.rs @@ -119,13 +119,7 @@ fn lex_source(src: &str) -> Result, String> { let mut lexer = Lexer::new(src.as_bytes()); // Goes through and lexes entire given string. - let mut tokens = Vec::new(); - - for token in lexer { - tokens.push(token.map_err(|e| format!("Lexing Error: {}", e))?); - } - - Ok(tokens) + lexer.collect::, _>().map_err(|e| format!("Lexing Error: {}", e)) } /// Parses the the token stream into a ast and returns it. From 42c583576c2775bfc23fe59b58f1a9a2e992e8d4 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Thu, 11 Jun 2020 11:49:00 +0100 Subject: [PATCH 006/289] Update boa/src/syntax/lexer/template.rs Co-authored-by: Iban Eguia --- boa/src/syntax/lexer/template.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs index ca2ca28a067..7d0a9dcc329 100644 --- a/boa/src/syntax/lexer/template.rs +++ b/boa/src/syntax/lexer/template.rs @@ -24,9 +24,7 @@ macro_rules! template_match { /// [spec]: /// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals #[derive(Debug, Clone, Copy)] -pub(super) struct TemplateLiteral { - -} +pub(super) struct TemplateLiteral; impl TemplateLiteral { /// Creates a new string literal lexer. @@ -62,4 +60,4 @@ impl Tokenizer for TemplateLiteral { Span::new(start_pos, cursor.pos()), )) } -} \ No newline at end of file +} From fe13f7a788ccef08a73946a1a07f53c27585ce79 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Thu, 11 Jun 2020 15:11:30 +0100 Subject: [PATCH 007/289] Moved token.rs file --- boa/src/syntax/ast/mod.rs | 2 -- boa/src/syntax/lexer/comment.rs | 6 ++--- boa/src/syntax/lexer/cursor.rs | 4 +-- boa/src/syntax/lexer/error.rs | 2 +- boa/src/syntax/lexer/mod.rs | 25 ++++++++++++------- boa/src/syntax/lexer/string.rs | 6 ++--- boa/src/syntax/lexer/template.rs | 24 ++++++------------ boa/src/syntax/{ast => lexer}/token.rs | 0 boa/src/syntax/parser/cursor.rs | 6 ++--- boa/src/syntax/parser/error.rs | 7 ++---- .../expression/assignment/arrow_function.rs | 3 ++- .../expression/assignment/conditional.rs | 3 ++- .../expression/assignment/exponentiation.rs | 3 ++- .../parser/expression/assignment/mod.rs | 3 ++- .../expression/left_hand_side/arguments.rs | 3 ++- .../parser/expression/left_hand_side/call.rs | 3 ++- .../expression/left_hand_side/member.rs | 3 ++- .../parser/expression/left_hand_side/mod.rs | 3 ++- boa/src/syntax/parser/expression/mod.rs | 3 ++- .../syntax/parser/expression/primary/mod.rs | 4 +-- .../primary/object_initializer/mod.rs | 3 +-- boa/src/syntax/parser/expression/unary.rs | 3 ++- boa/src/syntax/parser/expression/update.rs | 3 ++- boa/src/syntax/parser/function/mod.rs | 3 ++- boa/src/syntax/parser/mod.rs | 3 ++- boa/src/syntax/parser/statement/block/mod.rs | 4 ++- .../syntax/parser/statement/break_stm/mod.rs | 3 ++- .../parser/statement/continue_stm/mod.rs | 3 ++- .../parser/statement/declaration/lexical.rs | 3 ++- .../parser/statement/declaration/mod.rs | 3 ++- boa/src/syntax/parser/statement/if_stm/mod.rs | 3 ++- .../statement/iteration/do_while_statement.rs | 3 ++- .../statement/iteration/for_statement.rs | 3 ++- boa/src/syntax/parser/statement/mod.rs | 4 ++- .../syntax/parser/statement/return_stm/mod.rs | 3 ++- boa/src/syntax/parser/statement/throw/mod.rs | 3 ++- .../syntax/parser/statement/try_stm/mod.rs | 3 ++- boa/src/syntax/parser/statement/variable.rs | 3 ++- boa/src/syntax/parser/tests.rs | 4 +-- boa_cli/src/main.rs | 8 ++---- boa_wasm/src/lib.rs | 2 +- 41 files changed, 95 insertions(+), 88 deletions(-) rename boa/src/syntax/{ast => lexer}/token.rs (100%) diff --git a/boa/src/syntax/ast/mod.rs b/boa/src/syntax/ast/mod.rs index 6d96c9512db..b768c092460 100644 --- a/boa/src/syntax/ast/mod.rs +++ b/boa/src/syntax/ast/mod.rs @@ -6,7 +6,6 @@ pub mod node; pub mod op; pub mod position; pub mod punctuator; -pub mod token; pub use self::{ constant::Const, @@ -14,5 +13,4 @@ pub use self::{ node::Node, position::{Position, Span}, punctuator::Punctuator, - token::{Token, TokenKind}, }; diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index fea54c7d372..5bbe2a9524e 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -1,10 +1,8 @@ //! Coments lexing. use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::{ - token::Token, - Position, -}; +use crate::syntax::ast::Position; +use crate::syntax::lexer::{Token, TokenKind}; /// Lexes single line comments, starting with `//`. #[derive(Debug, Clone, Copy)] diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 9af3e826e2c..3510010953e 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -1,7 +1,5 @@ use crate::syntax::ast::Position; -use std::{ - io::{self, Bytes, Read} -}; +use std::io::{self, Bytes, Read}; /// Cursor over the source code. #[derive(Debug)] diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 0af65c633be..10a0c3a9249 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -14,7 +14,7 @@ impl From for Error { impl Error { /// Creates a new syntax error. - pub(crate) fn syntax(err: M) -> Self + pub(super) fn syntax(err: M) -> Self where M: Into>, { diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 9d6b4d4de90..f4904d5ce3e 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -7,6 +7,7 @@ mod comment; mod cursor; pub mod error; mod string; +pub mod token; #[macro_use] mod template; @@ -18,11 +19,9 @@ mod template; pub use self::error::Error; use self::{cursor::Cursor, string::StringLiteral, template::TemplateLiteral}; -use crate::syntax::ast::{ - token::{Token, TokenKind}, - Position, Span, -}; +use crate::syntax::ast::{Position, Span}; use std::io::Read; +pub use token::{Token, TokenKind}; trait Tokenizer { /// Lexes the next token. @@ -136,7 +135,6 @@ where // } // } - // Temporarily moved. use crate::syntax::ast::Keyword; @@ -145,8 +143,17 @@ fn check_single_line_comment() { let s1 = "var \n//This is a comment\ntrue"; let mut lexer = Lexer::new(s1.as_bytes()); - assert_eq!(lexer.next().unwrap().unwrap().kind, TokenKind::Keyword(Keyword::Var)); - assert_eq!(lexer.next().unwrap().unwrap().kind, TokenKind::LineTerminator); - assert_eq!(lexer.next().unwrap().unwrap().kind, TokenKind::BooleanLiteral(true)); + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::Keyword(Keyword::Var) + ); + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::LineTerminator + ); + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::BooleanLiteral(true) + ); assert!(lexer.next().is_none()); -} \ No newline at end of file +} diff --git a/boa/src/syntax/lexer/string.rs b/boa/src/syntax/lexer/string.rs index 492666c5a37..0711f55cb27 100644 --- a/boa/src/syntax/lexer/string.rs +++ b/boa/src/syntax/lexer/string.rs @@ -1,8 +1,6 @@ use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::{ - token::{Token, TokenKind}, - Position, Span, -}; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{Token, TokenKind}; use std::{ char::{decode_utf16, from_u32}, convert::TryFrom, diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs index ca2ca28a067..69c93045656 100644 --- a/boa/src/syntax/lexer/template.rs +++ b/boa/src/syntax/lexer/template.rs @@ -1,11 +1,7 @@ use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::{ - token::{Token, TokenKind}, - Position, Span, -}; -use std::{ - io::{self, ErrorKind, Read} -}; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{Token, TokenKind}; +use std::io::{self, ErrorKind, Read}; macro_rules! template_match { () => { @@ -16,24 +12,20 @@ macro_rules! template_match { /// Template literal parsing. /// /// Expects: Initial ` to already be consumed by cursor. -/// +/// /// More information: /// - [ECMAScript reference][spec] /// - [MDN documentation][mdn] /// -/// [spec]: +/// [spec]: /// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals #[derive(Debug, Clone, Copy)] -pub(super) struct TemplateLiteral { - -} +pub(super) struct TemplateLiteral {} impl TemplateLiteral { /// Creates a new string literal lexer. pub(super) fn new() -> Self { - Self { - - } + Self {} } } @@ -62,4 +54,4 @@ impl Tokenizer for TemplateLiteral { Span::new(start_pos, cursor.pos()), )) } -} \ No newline at end of file +} diff --git a/boa/src/syntax/ast/token.rs b/boa/src/syntax/lexer/token.rs similarity index 100% rename from boa/src/syntax/ast/token.rs rename to boa/src/syntax/lexer/token.rs diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 69b116ce3e4..8ffd93d0e0e 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -1,10 +1,8 @@ //! Cursor implementation for the parser. use super::ParseError; -use crate::syntax::ast::{ - token::{Token, TokenKind}, - Punctuator, -}; +use crate::syntax::ast::Punctuator; +use crate::syntax::lexer::{Token, TokenKind}; /// Token cursor. /// diff --git a/boa/src/syntax/parser/error.rs b/boa/src/syntax/parser/error.rs index 924010a3eef..27065b6600a 100644 --- a/boa/src/syntax/parser/error.rs +++ b/boa/src/syntax/parser/error.rs @@ -1,9 +1,6 @@ //! Error and result implementation for the parser. -use crate::syntax::ast::{ - position::Position, - token::{Token, TokenKind}, - Node, -}; +use crate::syntax::ast::{position::Position, Node}; +use crate::syntax::lexer::{Token, TokenKind}; use std::fmt; /// Result of a parsing operation. diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index a802ace594f..1ada0363c73 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -8,11 +8,12 @@ //! [spec]: https://tc39.es/ecma262/#sec-arrow-function-definitions use super::AssignmentExpression; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{ArrowFunctionDecl, FormalParameter, Node, Return, StatementList}, - Punctuator, TokenKind, + Punctuator, }, parser::{ error::{ErrorContext, ParseError, ParseResult}, diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index 0bc8b7d7277..e903f27400f 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -7,9 +7,10 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Conditional_Operator //! [spec]: https://tc39.es/ecma262/#sec-conditional-operator +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::ConditionalOp, Node, Punctuator, TokenKind}, + ast::{node::ConditionalOp, Node, Punctuator}, parser::{ expression::{AssignmentExpression, LogicalORExpression}, AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser, diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 811bde4537d..ef18aa772f8 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -7,12 +7,13 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Arithmetic_Operators#Exponentiation //! [spec]: https://tc39.es/ecma262/#sec-exp-operator +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{BinOp, Node}, op::NumOp, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::{unary::UnaryExpression, update::UpdateExpression}, diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 4f71f217361..99f5be42d32 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -12,11 +12,12 @@ mod conditional; mod exponentiation; use self::{arrow_function::ArrowFunction, conditional::ConditionalExpression}; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{Assign, BinOp, Node}, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{AllowAwait, AllowIn, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }, diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index a15c88077e6..829d7271b9c 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -7,9 +7,10 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Glossary/Argument //! [spec]: https://tc39.es/ecma262/#prod-Arguments +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Spread, Node, Punctuator, TokenKind}, + ast::{node::Spread, Node, Punctuator}, parser::{ expression::AssignmentExpression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index c1a5b958912..29113073bc8 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -8,6 +8,7 @@ //! [spec]: https://tc39.es/ecma262/#prod-CallExpression use super::arguments::Arguments; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ @@ -15,7 +16,7 @@ use crate::{ field::{GetConstField, GetField}, Call, Node, }, - Punctuator, TokenKind, + Punctuator, }, parser::{ expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, ParseResult, diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 91a6f04a43f..c3dff1939fa 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -6,6 +6,7 @@ //! [spec]: https://tc39.es/ecma262/#prod-MemberExpression use super::arguments::Arguments; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ @@ -13,7 +14,7 @@ use crate::{ field::{GetConstField, GetField}, Call, New, Node, }, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::{primary::PrimaryExpression, Expression}, diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 1be558b9ec9..c739cf2e1d2 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -12,9 +12,10 @@ mod call; mod member; use self::{call::CallExpression, member::MemberExpression}; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{Node, Punctuator, TokenKind}, + ast::{Node, Punctuator}, parser::{AllowAwait, AllowYield, Cursor, ParseResult, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 0b4137bc150..a2a8ad3acb2 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -18,11 +18,12 @@ mod update; use self::assignment::ExponentiationExpression; pub(super) use self::{assignment::AssignmentExpression, primary::Initializer}; use super::{AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser}; +use crate::syntax::lexer::TokenKind; use crate::{ profiler::BoaProfiler, syntax::ast::{ node::{BinOp, Node}, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, }; diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 0e31e581cc5..f17849034a7 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -18,11 +18,11 @@ use self::{ object_initializer::ObjectLiteral, }; use super::Expression; +use crate::syntax::lexer::{token::NumericLiteral, TokenKind}; use crate::syntax::{ ast::{ node::{Call, Identifier, New, Node}, - token::NumericLiteral, - Const, Keyword, Punctuator, TokenKind, + Const, Keyword, Punctuator, }, parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }; diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 3d2dc14ea45..f3560be63a4 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -9,12 +9,11 @@ #[cfg(test)] mod tests; - +use crate::syntax::lexer::{Token, TokenKind}; use crate::{ syntax::{ ast::{ node::{self, FunctionExpr, MethodDefinitionKind, Node, Object}, - token::{Token, TokenKind}, Punctuator, }, parser::{ diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 7a4449155e1..2da3ca06b31 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -7,11 +7,12 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Expressions_and_Operators#Unary //! [spec]: https://tc39.es/ecma262/#sec-unary-operators +use crate::syntax::lexer::TokenKind; use crate::syntax::{ ast::{ node::{self, Node}, op::UnaryOp, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::update::UpdateExpression, AllowAwait, AllowYield, Cursor, ParseError, diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index 59e3e233c4d..09e508c1a26 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -6,8 +6,9 @@ //! [spec]: https://tc39.es/ecma262/#sec-update-expressions use super::left_hand_side::LeftHandSideExpression; +use crate::syntax::lexer::TokenKind; use crate::syntax::{ - ast::{node, op::UnaryOp, Node, Punctuator, TokenKind}, + ast::{node, op::UnaryOp, Node, Punctuator}, parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }; diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 6005404ec4e..725bf0d90c1 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -10,10 +10,11 @@ #[cfg(test)] mod tests; +use crate::syntax::lexer::TokenKind; use crate::syntax::{ ast::{ node::{self}, - Punctuator, TokenKind, + Punctuator, }, parser::{ expression::Initializer, diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 79968cd7d3a..2ee067e5540 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -9,7 +9,8 @@ mod statement; mod tests; use self::error::{ParseError, ParseResult}; -use crate::syntax::ast::{node::StatementList, Token}; +use crate::syntax::ast::node::StatementList; +use crate::syntax::lexer::Token; use cursor::Cursor; /// Trait implemented by parsers. diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 843cb0341ea..04afc79fd00 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -11,10 +11,12 @@ mod tests; use super::StatementList; + +use crate::syntax::lexer::TokenKind; use crate::{ profiler::BoaProfiler, syntax::{ - ast::{node, Punctuator, TokenKind}, + ast::{node, Punctuator}, parser::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}, }, }; diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index e0a070de1f0..478b8b45973 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -12,9 +12,10 @@ mod tests; use super::LabelIdentifier; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Break, Keyword, Punctuator, TokenKind}, + ast::{node::Break, Keyword, Punctuator}, parser::{AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index 433f41dd0fc..88fe239c693 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -10,9 +10,10 @@ #[cfg(test)] mod tests; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Continue, Keyword, Punctuator, TokenKind}, + ast::{node::Continue, Keyword, Punctuator}, parser::{ statement::LabelIdentifier, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, }, diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 25dfec3a271..3fcff93ee33 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -7,11 +7,12 @@ //! //! [spec]: https://tc39.es/ecma262/#sec-let-and-const-declarations +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{ConstDecl, ConstDeclList, LetDecl, LetDeclList, Node}, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index c22aa676569..90ed00172ad 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -14,9 +14,10 @@ mod tests; use self::{hoistable::HoistableDeclaration, lexical::LexicalDeclaration}; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{Keyword, Node, TokenKind}, + ast::{Keyword, Node}, parser::{AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index d8e6b1c7294..7a21ef3465e 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -3,9 +3,10 @@ mod tests; use super::Statement; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::If, Keyword, Node, Punctuator, TokenKind}, + ast::{node::If, Keyword, Node, Punctuator}, parser::{ expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 91613fba1d6..716a03a996b 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -7,9 +7,10 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/do...while //! [spec]: https://tc39.es/ecma262/#sec-do-while-statement +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::DoWhileLoop, Keyword, Punctuator, TokenKind}, + ast::{node::DoWhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index bd1ed7c6a43..c6dba0a85e0 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -7,11 +7,12 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for //! [spec]: https://tc39.es/ecma262/#sec-for-statement +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{ForLoop, Node}, - Const, Keyword, Punctuator, TokenKind, + Const, Keyword, Punctuator, }, parser::{ expression::Expression, diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 3853ba843f7..aaf3802af80 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -36,8 +36,10 @@ use super::{ expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, ParseResult, TokenParser, }; + +use crate::syntax::lexer::TokenKind; use crate::{ - syntax::ast::{node, Keyword, Node, Punctuator, TokenKind}, + syntax::ast::{node, Keyword, Node, Punctuator}, BoaProfiler, }; diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index c1b0c6cf082..ccdb14a9cfb 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -1,9 +1,10 @@ #[cfg(test)] mod tests; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Return, Keyword, Node, Punctuator, TokenKind}, + ast::{node::Return, Keyword, Node, Punctuator}, parser::{expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index 1be0260b2ce..aa046be358e 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -1,9 +1,10 @@ #[cfg(test)] mod tests; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Throw, Keyword, Punctuator, TokenKind}, + ast::{node::Throw, Keyword, Punctuator}, parser::{expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index 041148f329b..afcb4519f99 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -7,9 +7,10 @@ mod tests; use self::catch::Catch; use self::finally::Finally; use super::block::Block; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Try, Keyword, TokenKind}, + ast::{node::Try, Keyword}, parser::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 904b012f0ce..ddc86c0c6d6 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -1,9 +1,10 @@ // use super::lexical_declaration_continuation; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{VarDecl, VarDeclList}, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index e77b9d046f3..7865c3d4790 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -30,9 +30,7 @@ where } assert_eq!( - Parser::new(&tokens) - .parse_all() - .expect("failed to parse"), + Parser::new(&tokens).parse_all().expect("failed to parse"), StatementList::from(expr) ); } diff --git a/boa_cli/src/main.rs b/boa_cli/src/main.rs index 10507bde2f1..f5928a25d19 100644 --- a/boa_cli/src/main.rs +++ b/boa_cli/src/main.rs @@ -26,12 +26,8 @@ )] use boa::{ - builtins::console::log, - exec::Interpreter, - forward_val, - realm::Realm, - syntax::ast::{node::StatementList, token::Token}, - syntax::lexer::Lexer, + builtins::console::log, exec::Interpreter, forward_val, realm::Realm, + syntax::ast::node::StatementList, syntax::lexer::Lexer, syntax::lexer::Token, }; use std::{ fs::read_to_string, diff --git a/boa_wasm/src/lib.rs b/boa_wasm/src/lib.rs index 2d9d2e506fe..a8dee2a2b4a 100644 --- a/boa_wasm/src/lib.rs +++ b/boa_wasm/src/lib.rs @@ -11,7 +11,7 @@ pub fn evaluate(src: &str) -> Result { for token in lexer { tokens.push(token.map_err(|e| format!("Lexing Error: {}", e))?); } - + let expr = Parser::new(&tokens) .parse_all() .map_err(|e| JsValue::from(format!("Parsing Error: {}", e)))?; From 688fff5678a22506eeaf3cba99bfa523938cb029 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Thu, 11 Jun 2020 17:20:00 +0100 Subject: [PATCH 008/289] New Lexer: Minimal amount to allow compiling (#477) Co-authored-by: Iban Eguia --- boa/src/lib.rs | 12 +++- boa/src/syntax/ast/mod.rs | 2 - boa/src/syntax/lexer/comment.rs | 14 +---- boa/src/syntax/lexer/cursor.rs | 14 +---- boa/src/syntax/lexer/error.rs | 6 +- boa/src/syntax/lexer/mod.rs | 59 +++++++++++++------ boa/src/syntax/lexer/string.rs | 6 +- boa/src/syntax/lexer/template.rs | 57 ++++++++++++++++++ boa/src/syntax/lexer/tests.rs | 2 +- boa/src/syntax/{ast => lexer}/token.rs | 0 boa/src/syntax/parser/cursor.rs | 6 +- boa/src/syntax/parser/error.rs | 7 +-- .../expression/assignment/arrow_function.rs | 3 +- .../expression/assignment/conditional.rs | 3 +- .../expression/assignment/exponentiation.rs | 3 +- .../parser/expression/assignment/mod.rs | 3 +- .../expression/left_hand_side/arguments.rs | 3 +- .../parser/expression/left_hand_side/call.rs | 3 +- .../expression/left_hand_side/member.rs | 3 +- .../parser/expression/left_hand_side/mod.rs | 3 +- boa/src/syntax/parser/expression/mod.rs | 3 +- .../syntax/parser/expression/primary/mod.rs | 4 +- .../primary/object_initializer/mod.rs | 3 +- boa/src/syntax/parser/expression/unary.rs | 3 +- boa/src/syntax/parser/expression/update.rs | 3 +- boa/src/syntax/parser/function/mod.rs | 3 +- boa/src/syntax/parser/mod.rs | 3 +- boa/src/syntax/parser/statement/block/mod.rs | 4 +- .../syntax/parser/statement/break_stm/mod.rs | 3 +- .../parser/statement/continue_stm/mod.rs | 3 +- .../parser/statement/declaration/lexical.rs | 3 +- .../parser/statement/declaration/mod.rs | 3 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 3 +- .../statement/iteration/do_while_statement.rs | 3 +- .../statement/iteration/for_statement.rs | 3 +- boa/src/syntax/parser/statement/mod.rs | 4 +- .../syntax/parser/statement/return_stm/mod.rs | 3 +- boa/src/syntax/parser/statement/throw/mod.rs | 3 +- .../syntax/parser/statement/try_stm/mod.rs | 3 +- boa/src/syntax/parser/statement/variable.rs | 3 +- boa/src/syntax/parser/tests.rs | 22 ++++--- boa_cli/src/main.rs | 16 +++-- boa_wasm/src/lib.rs | 11 ++-- 43 files changed, 205 insertions(+), 116 deletions(-) create mode 100644 boa/src/syntax/lexer/template.rs rename boa/src/syntax/{ast => lexer}/token.rs (100%) diff --git a/boa/src/lib.rs b/boa/src/lib.rs index 4314d77f794..f43d27e1efe 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -50,9 +50,15 @@ pub use crate::{ }; fn parser_expr(src: &str) -> Result { - let mut lexer = Lexer::new(src); - lexer.lex().map_err(|e| format!("Syntax Error: {}", e))?; - let tokens = lexer.tokens; + let mut lexer = Lexer::new(src.as_bytes()); + + // Goes through and lexes entire given string before starting any parsing. + let mut tokens = Vec::new(); + + for token in lexer { + tokens.push(token.map_err(|e| format!("Lexing Error: {}", e))?); + } + Parser::new(&tokens) .parse_all() .map_err(|e| format!("Parsing Error: {}", e)) diff --git a/boa/src/syntax/ast/mod.rs b/boa/src/syntax/ast/mod.rs index 6d96c9512db..b768c092460 100644 --- a/boa/src/syntax/ast/mod.rs +++ b/boa/src/syntax/ast/mod.rs @@ -6,7 +6,6 @@ pub mod node; pub mod op; pub mod position; pub mod punctuator; -pub mod token; pub use self::{ constant::Const, @@ -14,5 +13,4 @@ pub use self::{ node::Node, position::{Position, Span}, punctuator::Punctuator, - token::{Token, TokenKind}, }; diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 500852fb73c..5bbe2a9524e 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -1,18 +1,8 @@ //! Coments lexing. use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::bigint::BigInt; -use crate::syntax::ast::{ - token::{NumericLiteral, Token, TokenKind}, - Position, Punctuator, Span, -}; -use std::{ - char::{decode_utf16, from_u32}, - fmt, - io::{self, BufRead, Bytes, Read, Seek}, - iter::Peekable, - str::{Chars, FromStr}, -}; +use crate::syntax::ast::Position; +use crate::syntax::lexer::{Token, TokenKind}; /// Lexes single line comments, starting with `//`. #[derive(Debug, Clone, Copy)] diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 66649524fde..3510010953e 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -1,15 +1,5 @@ -use crate::syntax::ast::bigint::BigInt; -use crate::syntax::ast::{ - token::{NumericLiteral, Token, TokenKind}, - Position, Punctuator, Span, -}; -use std::{ - char::{decode_utf16, from_u32}, - error, fmt, - io::{self, BufRead, Bytes, Read, Seek}, - iter::Peekable, - str::{Chars, FromStr}, -}; +use crate::syntax::ast::Position; +use std::io::{self, Bytes, Read}; /// Cursor over the source code. #[derive(Debug)] diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 525861f2e18..10a0c3a9249 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -24,7 +24,7 @@ impl Error { impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { + match self { Self::IO(e) => write!(f, "I/O error: {}", e), Self::Syntax(e) => write!(f, "Syntax Error: {}", e), } @@ -33,8 +33,8 @@ impl fmt::Display for Error { impl StdError for Error { fn source(&self) -> Option<&(dyn StdError + 'static)> { - match *self { - Self::IO(err) => Some(&err), + match self { + Self::IO(err) => Some(err), Self::Syntax(_) => None, } } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index ca320e6bed7..f4904d5ce3e 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -7,23 +7,21 @@ mod comment; mod cursor; pub mod error; mod string; -#[cfg(test)] -mod tests; +pub mod token; + +#[macro_use] +mod template; + +// Temporary disabled while lexer in progress. +// #[cfg(test)] +// mod tests; pub use self::error::Error; -use self::{cursor::Cursor, string::StringLiteral}; -use crate::syntax::ast::bigint::BigInt; -use crate::syntax::ast::{ - token::{NumericLiteral, Token, TokenKind}, - Position, Punctuator, Span, -}; -use std::{ - char::{decode_utf16, from_u32}, - fmt, - io::{self, BufRead, Bytes, Read, Seek}, - iter::Peekable, - str::{Chars, FromStr}, -}; + +use self::{cursor::Cursor, string::StringLiteral, template::TemplateLiteral}; +use crate::syntax::ast::{Position, Span}; +use std::io::Read; +pub use token::{Token, TokenKind}; trait Tokenizer { /// Lexes the next token. @@ -34,7 +32,7 @@ trait Tokenizer { /// Lexer or tokenizer for the Boa JavaScript Engine. #[derive(Debug)] -pub(crate) struct Lexer { +pub struct Lexer { cursor: Cursor, goal_symbol: InputElement, } @@ -69,7 +67,7 @@ where { /// Creates a new lexer. #[inline] - pub(crate) fn new(reader: R) -> Self { + pub fn new(reader: R) -> Self { Self { cursor: Cursor::new(reader), goal_symbol: Default::default(), @@ -120,6 +118,7 @@ where Span::new(start, self.cursor.pos()), )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), + template_match!() => TemplateLiteral::new().lex(&mut self.cursor, start), _ => unimplemented!(), }; @@ -128,9 +127,33 @@ where } // impl Tokenizer for Lexer { -// fn lex(&mut self, cursor: &mut Cursor) -> io::Result +// fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> io::Result // where // R: Read, // { + // } // } + +// Temporarily moved. +use crate::syntax::ast::Keyword; + +#[test] +fn check_single_line_comment() { + let s1 = "var \n//This is a comment\ntrue"; + let mut lexer = Lexer::new(s1.as_bytes()); + + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::Keyword(Keyword::Var) + ); + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::LineTerminator + ); + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::BooleanLiteral(true) + ); + assert!(lexer.next().is_none()); +} diff --git a/boa/src/syntax/lexer/string.rs b/boa/src/syntax/lexer/string.rs index 492666c5a37..0711f55cb27 100644 --- a/boa/src/syntax/lexer/string.rs +++ b/boa/src/syntax/lexer/string.rs @@ -1,8 +1,6 @@ use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::{ - token::{Token, TokenKind}, - Position, Span, -}; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{Token, TokenKind}; use std::{ char::{decode_utf16, from_u32}, convert::TryFrom, diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs new file mode 100644 index 00000000000..e109f01c276 --- /dev/null +++ b/boa/src/syntax/lexer/template.rs @@ -0,0 +1,57 @@ +use super::{Cursor, Error, Tokenizer}; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{Token, TokenKind}; +use std::io::{self, ErrorKind, Read}; + +macro_rules! template_match { + () => { + '`' + }; +} + +/// Template literal parsing. +/// +/// Expects: Initial ` to already be consumed by cursor. +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: +/// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals +#[derive(Debug, Clone, Copy)] +pub(super) struct TemplateLiteral; + +impl TemplateLiteral { + /// Creates a new string literal lexer. + pub(super) fn new() -> Self { + Self {} + } +} + +impl Tokenizer for TemplateLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + let mut buf = String::new(); + loop { + match cursor.next() { + None => { + return Err(Error::from(io::Error::new( + ErrorKind::UnexpectedEof, + "Unterminated template literal", + ))); + } + Some(Err(e)) => return Err(Error::from(e)), + Some(Ok('`')) => break, // Template literal finished. + Some(Ok(next_ch)) => buf.push(next_ch), // TODO when there is an expression inside the literal + } + } + + Ok(Token::new( + TokenKind::template_literal(buf), + Span::new(start_pos, cursor.pos()), + )) + } +} diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 4129bb3c179..30f3fa06348 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -543,4 +543,4 @@ fn addition_no_spaces_e_number() { lexer.tokens[2].kind, TokenKind::numeric_literal(100_000_000_000.0) ); -} +} */ \ No newline at end of file diff --git a/boa/src/syntax/ast/token.rs b/boa/src/syntax/lexer/token.rs similarity index 100% rename from boa/src/syntax/ast/token.rs rename to boa/src/syntax/lexer/token.rs diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 69b116ce3e4..8ffd93d0e0e 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -1,10 +1,8 @@ //! Cursor implementation for the parser. use super::ParseError; -use crate::syntax::ast::{ - token::{Token, TokenKind}, - Punctuator, -}; +use crate::syntax::ast::Punctuator; +use crate::syntax::lexer::{Token, TokenKind}; /// Token cursor. /// diff --git a/boa/src/syntax/parser/error.rs b/boa/src/syntax/parser/error.rs index 924010a3eef..27065b6600a 100644 --- a/boa/src/syntax/parser/error.rs +++ b/boa/src/syntax/parser/error.rs @@ -1,9 +1,6 @@ //! Error and result implementation for the parser. -use crate::syntax::ast::{ - position::Position, - token::{Token, TokenKind}, - Node, -}; +use crate::syntax::ast::{position::Position, Node}; +use crate::syntax::lexer::{Token, TokenKind}; use std::fmt; /// Result of a parsing operation. diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index a802ace594f..1ada0363c73 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -8,11 +8,12 @@ //! [spec]: https://tc39.es/ecma262/#sec-arrow-function-definitions use super::AssignmentExpression; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{ArrowFunctionDecl, FormalParameter, Node, Return, StatementList}, - Punctuator, TokenKind, + Punctuator, }, parser::{ error::{ErrorContext, ParseError, ParseResult}, diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index 0bc8b7d7277..e903f27400f 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -7,9 +7,10 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Conditional_Operator //! [spec]: https://tc39.es/ecma262/#sec-conditional-operator +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::ConditionalOp, Node, Punctuator, TokenKind}, + ast::{node::ConditionalOp, Node, Punctuator}, parser::{ expression::{AssignmentExpression, LogicalORExpression}, AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser, diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 811bde4537d..ef18aa772f8 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -7,12 +7,13 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Arithmetic_Operators#Exponentiation //! [spec]: https://tc39.es/ecma262/#sec-exp-operator +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{BinOp, Node}, op::NumOp, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::{unary::UnaryExpression, update::UpdateExpression}, diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 4f71f217361..99f5be42d32 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -12,11 +12,12 @@ mod conditional; mod exponentiation; use self::{arrow_function::ArrowFunction, conditional::ConditionalExpression}; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{Assign, BinOp, Node}, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{AllowAwait, AllowIn, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }, diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index a15c88077e6..829d7271b9c 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -7,9 +7,10 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Glossary/Argument //! [spec]: https://tc39.es/ecma262/#prod-Arguments +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Spread, Node, Punctuator, TokenKind}, + ast::{node::Spread, Node, Punctuator}, parser::{ expression::AssignmentExpression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index c1a5b958912..29113073bc8 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -8,6 +8,7 @@ //! [spec]: https://tc39.es/ecma262/#prod-CallExpression use super::arguments::Arguments; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ @@ -15,7 +16,7 @@ use crate::{ field::{GetConstField, GetField}, Call, Node, }, - Punctuator, TokenKind, + Punctuator, }, parser::{ expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, ParseResult, diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 91a6f04a43f..c3dff1939fa 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -6,6 +6,7 @@ //! [spec]: https://tc39.es/ecma262/#prod-MemberExpression use super::arguments::Arguments; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ @@ -13,7 +14,7 @@ use crate::{ field::{GetConstField, GetField}, Call, New, Node, }, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::{primary::PrimaryExpression, Expression}, diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 1be558b9ec9..c739cf2e1d2 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -12,9 +12,10 @@ mod call; mod member; use self::{call::CallExpression, member::MemberExpression}; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{Node, Punctuator, TokenKind}, + ast::{Node, Punctuator}, parser::{AllowAwait, AllowYield, Cursor, ParseResult, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 0b4137bc150..a2a8ad3acb2 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -18,11 +18,12 @@ mod update; use self::assignment::ExponentiationExpression; pub(super) use self::{assignment::AssignmentExpression, primary::Initializer}; use super::{AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser}; +use crate::syntax::lexer::TokenKind; use crate::{ profiler::BoaProfiler, syntax::ast::{ node::{BinOp, Node}, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, }; diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 0e31e581cc5..f17849034a7 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -18,11 +18,11 @@ use self::{ object_initializer::ObjectLiteral, }; use super::Expression; +use crate::syntax::lexer::{token::NumericLiteral, TokenKind}; use crate::syntax::{ ast::{ node::{Call, Identifier, New, Node}, - token::NumericLiteral, - Const, Keyword, Punctuator, TokenKind, + Const, Keyword, Punctuator, }, parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }; diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 3d2dc14ea45..f3560be63a4 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -9,12 +9,11 @@ #[cfg(test)] mod tests; - +use crate::syntax::lexer::{Token, TokenKind}; use crate::{ syntax::{ ast::{ node::{self, FunctionExpr, MethodDefinitionKind, Node, Object}, - token::{Token, TokenKind}, Punctuator, }, parser::{ diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 7a4449155e1..2da3ca06b31 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -7,11 +7,12 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Expressions_and_Operators#Unary //! [spec]: https://tc39.es/ecma262/#sec-unary-operators +use crate::syntax::lexer::TokenKind; use crate::syntax::{ ast::{ node::{self, Node}, op::UnaryOp, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::update::UpdateExpression, AllowAwait, AllowYield, Cursor, ParseError, diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index 59e3e233c4d..09e508c1a26 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -6,8 +6,9 @@ //! [spec]: https://tc39.es/ecma262/#sec-update-expressions use super::left_hand_side::LeftHandSideExpression; +use crate::syntax::lexer::TokenKind; use crate::syntax::{ - ast::{node, op::UnaryOp, Node, Punctuator, TokenKind}, + ast::{node, op::UnaryOp, Node, Punctuator}, parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }; diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 6005404ec4e..725bf0d90c1 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -10,10 +10,11 @@ #[cfg(test)] mod tests; +use crate::syntax::lexer::TokenKind; use crate::syntax::{ ast::{ node::{self}, - Punctuator, TokenKind, + Punctuator, }, parser::{ expression::Initializer, diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 79968cd7d3a..2ee067e5540 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -9,7 +9,8 @@ mod statement; mod tests; use self::error::{ParseError, ParseResult}; -use crate::syntax::ast::{node::StatementList, Token}; +use crate::syntax::ast::node::StatementList; +use crate::syntax::lexer::Token; use cursor::Cursor; /// Trait implemented by parsers. diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 843cb0341ea..04afc79fd00 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -11,10 +11,12 @@ mod tests; use super::StatementList; + +use crate::syntax::lexer::TokenKind; use crate::{ profiler::BoaProfiler, syntax::{ - ast::{node, Punctuator, TokenKind}, + ast::{node, Punctuator}, parser::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}, }, }; diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index e0a070de1f0..478b8b45973 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -12,9 +12,10 @@ mod tests; use super::LabelIdentifier; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Break, Keyword, Punctuator, TokenKind}, + ast::{node::Break, Keyword, Punctuator}, parser::{AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index 433f41dd0fc..88fe239c693 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -10,9 +10,10 @@ #[cfg(test)] mod tests; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Continue, Keyword, Punctuator, TokenKind}, + ast::{node::Continue, Keyword, Punctuator}, parser::{ statement::LabelIdentifier, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, }, diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 25dfec3a271..3fcff93ee33 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -7,11 +7,12 @@ //! //! [spec]: https://tc39.es/ecma262/#sec-let-and-const-declarations +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{ConstDecl, ConstDeclList, LetDecl, LetDeclList, Node}, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index c22aa676569..90ed00172ad 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -14,9 +14,10 @@ mod tests; use self::{hoistable::HoistableDeclaration, lexical::LexicalDeclaration}; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{Keyword, Node, TokenKind}, + ast::{Keyword, Node}, parser::{AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index d8e6b1c7294..7a21ef3465e 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -3,9 +3,10 @@ mod tests; use super::Statement; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::If, Keyword, Node, Punctuator, TokenKind}, + ast::{node::If, Keyword, Node, Punctuator}, parser::{ expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 91613fba1d6..716a03a996b 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -7,9 +7,10 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/do...while //! [spec]: https://tc39.es/ecma262/#sec-do-while-statement +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::DoWhileLoop, Keyword, Punctuator, TokenKind}, + ast::{node::DoWhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index bd1ed7c6a43..c6dba0a85e0 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -7,11 +7,12 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for //! [spec]: https://tc39.es/ecma262/#sec-for-statement +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{ForLoop, Node}, - Const, Keyword, Punctuator, TokenKind, + Const, Keyword, Punctuator, }, parser::{ expression::Expression, diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 3853ba843f7..aaf3802af80 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -36,8 +36,10 @@ use super::{ expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, ParseResult, TokenParser, }; + +use crate::syntax::lexer::TokenKind; use crate::{ - syntax::ast::{node, Keyword, Node, Punctuator, TokenKind}, + syntax::ast::{node, Keyword, Node, Punctuator}, BoaProfiler, }; diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index c1b0c6cf082..ccdb14a9cfb 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -1,9 +1,10 @@ #[cfg(test)] mod tests; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Return, Keyword, Node, Punctuator, TokenKind}, + ast::{node::Return, Keyword, Node, Punctuator}, parser::{expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index 1be0260b2ce..aa046be358e 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -1,9 +1,10 @@ #[cfg(test)] mod tests; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Throw, Keyword, Punctuator, TokenKind}, + ast::{node::Throw, Keyword, Punctuator}, parser::{expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index 041148f329b..afcb4519f99 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -7,9 +7,10 @@ mod tests; use self::catch::Catch; use self::finally::Finally; use super::block::Block; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ - ast::{node::Try, Keyword, TokenKind}, + ast::{node::Try, Keyword}, parser::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 904b012f0ce..ddc86c0c6d6 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -1,9 +1,10 @@ // use super::lexical_declaration_continuation; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ node::{VarDecl, VarDeclList}, - Keyword, Punctuator, TokenKind, + Keyword, Punctuator, }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index c5e83d03d26..30c8e4443c1 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -20,13 +20,17 @@ pub(super) fn check_parser(js: &str, expr: L) where L: Into>, { - let mut lexer = Lexer::new(js); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new(js.as_bytes()); + + // Goes through and lexes entire given string. + let mut tokens = Vec::new(); + + for token in lexer { + tokens.push(token.expect("failed to lex")); + } assert_eq!( - Parser::new(&lexer.tokens) - .parse_all() - .expect("failed to parse"), + Parser::new(&tokens).parse_all().expect("failed to parse"), StatementList::from(expr) ); } @@ -34,10 +38,12 @@ where /// Checks that the given javascript string creates a parse error. // TODO: #[track_caller]: https://github.com/rust-lang/rust/issues/47809 pub(super) fn check_invalid(js: &str) { - let mut lexer = Lexer::new(js); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new(js.as_bytes()); + // lexer.lex().expect("failed to lex"); + + let tokens = lexer.collect::, _>>().expect("failed to lex"); - assert!(Parser::new(&lexer.tokens).parse_all().is_err()); + assert!(Parser::new(&tokens).parse_all().is_err()); } /// Should be parsed as `new Class().method()` instead of `new (Class().method())` diff --git a/boa_cli/src/main.rs b/boa_cli/src/main.rs index a24c941a39c..aa1c20f9a0b 100644 --- a/boa_cli/src/main.rs +++ b/boa_cli/src/main.rs @@ -26,11 +26,8 @@ )] use boa::{ - builtins::console::log, - exec::Interpreter, - forward_val, - realm::Realm, - syntax::ast::{node::StatementList, token::Token}, + builtins::console::log, exec::Interpreter, forward_val, realm::Realm, + syntax::ast::node::StatementList, syntax::lexer::Lexer, syntax::lexer::Token, }; use std::{ fs::read_to_string, @@ -115,11 +112,12 @@ arg_enum! { /// Returns a error of type String with a message, /// if the source has a syntax error. fn lex_source(src: &str) -> Result, String> { - use boa::syntax::lexer::Lexer; + let mut lexer = Lexer::new(src.as_bytes()); - let mut lexer = Lexer::new(src); - lexer.lex().map_err(|e| format!("SyntaxError: {}", e))?; - Ok(lexer.tokens) + // Goes through and lexes entire given string. + lexer + .collect::, _>>() + .map_err(|e| format!("Lexing Error: {}", e)) } /// Parses the the token stream into a ast and returns it. diff --git a/boa_wasm/src/lib.rs b/boa_wasm/src/lib.rs index 97eab2e5c61..81b48606e32 100644 --- a/boa_wasm/src/lib.rs +++ b/boa_wasm/src/lib.rs @@ -3,12 +3,13 @@ use wasm_bindgen::prelude::*; #[wasm_bindgen] pub fn evaluate(src: &str) -> Result { - let mut lexer = Lexer::new(src); - lexer - .lex() - .map_err(|e| JsValue::from(format!("Syntax Error: {}", e)))?; + let mut lexer = Lexer::new(src.as_bytes()); + + // Goes through and lexes entire given string. + let tokens = lexer + .collect::, _>>() + .map_err(|e| format!("Lexing Error: {}", e))?; - let tokens = lexer.tokens; let expr = Parser::new(&tokens) .parse_all() .map_err(|e| JsValue::from(format!("Parsing Error: {}", e)))?; From 204ef0ad90cf129dde4c71eee3af75c109f1a45b Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 11:11:05 +0100 Subject: [PATCH 009/289] Moved identifier/keyword lexing into new lexer --- boa/src/syntax/lexer/comment.rs | 6 + boa/src/syntax/lexer/cursor.rs | 6 +- boa/src/syntax/lexer/error.rs | 15 ++ boa/src/syntax/lexer/identifier.rs | 72 ++++++++ boa/src/syntax/lexer/lexer_old.rs | 2 +- boa/src/syntax/lexer/mod.rs | 20 +- boa/src/syntax/lexer/number.rs | 285 +++++++++++++++++++++++++++++ 7 files changed, 400 insertions(+), 6 deletions(-) create mode 100644 boa/src/syntax/lexer/identifier.rs create mode 100644 boa/src/syntax/lexer/number.rs diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 5bbe2a9524e..18b551723e6 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -4,6 +4,12 @@ use super::{Cursor, Error, Tokenizer}; use crate::syntax::ast::Position; use crate::syntax::lexer::{Token, TokenKind}; +macro_rules! comment_match { + () => {{ + '/' + }}; +} + /// Lexes single line comments, starting with `//`. #[derive(Debug, Clone, Copy)] pub(super) struct SingleLineComment; diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 3510010953e..833bf91c641 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -16,15 +16,15 @@ impl Cursor { self.pos } - /// Advances the position to the next line. + /// Advances the position to the next column. #[inline] - fn next_column(&mut self) { + pub(super) fn next_column(&mut self) { let current_line = self.pos.line_number(); let next_column = self.pos.column_number() + 1; self.pos = Position::new(current_line, next_column); } - /// Advances the position to the next column. + /// Advances the position to the next line. #[inline] fn next_line(&mut self) { let next_line = self.pos.line_number() + 1; diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 10a0c3a9249..3c1b65e2a3b 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -4,6 +4,7 @@ use std::{error::Error as StdError, fmt, io}; pub enum Error { IO(io::Error), Syntax(Box), + StrictMode(Box), // Not 100% decided on this name. } impl From for Error { @@ -20,6 +21,18 @@ impl Error { { Self::Syntax(err.into()) } + + /// Creates a new StrictMode error. + /// + /// This error is used to represent the case where a piece of javascript + /// cannot be lexed/parsed because it is in invalid when strict mdoe is + /// enabled. + pub(super) fn strict(err: M) -> Self + where + M: Into>, + { + Self::StrictMode(err.into()) + } } impl fmt::Display for Error { @@ -27,6 +40,7 @@ impl fmt::Display for Error { match self { Self::IO(e) => write!(f, "I/O error: {}", e), Self::Syntax(e) => write!(f, "Syntax Error: {}", e), + Self::StrictMode(e) => write!(f, "Strict Mode Error: {}", e), } } } @@ -36,6 +50,7 @@ impl StdError for Error { match self { Self::IO(err) => Some(err), Self::Syntax(_) => None, + Self::StrictMode(_) => None, } } } diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs new file mode 100644 index 00000000000..f5c95c370ee --- /dev/null +++ b/boa/src/syntax/lexer/identifier.rs @@ -0,0 +1,72 @@ +use super::{Cursor, Error, Tokenizer}; +use crate::builtins::BigInt; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::token::NumericLiteral; +use crate::syntax::lexer::{Token, TokenKind}; +use std::io::{self, ErrorKind, Read}; +use std::str::FromStr; + +/// Identifier or keyword lexing. +/// +/// This currently includes boolean/NaN lexing. +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: +/// [mdn]: +#[derive(Debug, Clone, Copy)] +pub(super) struct Identifier { + init: char, +} + +impl Identifier { + /// Creates a new identifier/keyword lexer. + pub(super) fn new(init: char) -> Self { + Self { init: init } + } +} + +impl Tokenizer for Identifier { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + let mut buf = self.init.to_string(); + + loop { + match cursor.peek() { + None => { + break; + } + Some(Ok(c)) => { + if c.is_alphabetic() || c.is_digit(10) || *c == '_' { + let ch = cursor.next().unwrap()?; + buf.push(ch); + } else { + break; + } + } + Some(Err(_e)) => { + // TODO handle error. + } + } + } + let tk = match buf.as_str() { + "true" => TokenKind::BooleanLiteral(true), + "false" => TokenKind::BooleanLiteral(false), + "null" => TokenKind::NullLiteral, + "NaN" => TokenKind::NumericLiteral(NumericLiteral::Rational(f64::NAN)), + slice => { + if let Ok(keyword) = FromStr::from_str(slice) { + TokenKind::Keyword(keyword) + } else { + TokenKind::identifier(slice) + } + } + }; + + Ok(Token::new(tk, Span::new(start_pos, cursor.pos()))) + } +} diff --git a/boa/src/syntax/lexer/lexer_old.rs b/boa/src/syntax/lexer/lexer_old.rs index 568b54d526d..9e426e1db3c 100644 --- a/boa/src/syntax/lexer/lexer_old.rs +++ b/boa/src/syntax/lexer/lexer_old.rs @@ -477,7 +477,7 @@ impl<'a> Lexer<'a> { } /// Runs the lexer until completion, returning a [LexerError] if there's a syntax issue, or an empty unit result - /// + // /// # Example /// /// ``` diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index f4904d5ce3e..0e74ff52f83 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -3,22 +3,32 @@ //! The Lexer splits its input source code into a sequence of input elements called tokens, represented by the [Token](../ast/token/struct.Token.html) structure. //! It also removes whitespace and comments and attaches them to the next token. +#[macro_use] mod comment; mod cursor; pub mod error; + +#[macro_use] mod string; pub mod token; #[macro_use] mod template; +mod number; + +mod identifier; + // Temporary disabled while lexer in progress. // #[cfg(test)] // mod tests; pub use self::error::Error; -use self::{cursor::Cursor, string::StringLiteral, template::TemplateLiteral}; +use self::{ + cursor::Cursor, identifier::Identifier, number::NumberLiteral, string::StringLiteral, + template::TemplateLiteral, +}; use crate::syntax::ast::{Position, Span}; use std::io::Read; pub use token::{Token, TokenKind}; @@ -88,7 +98,9 @@ pub(crate) enum InputElement { impl Default for InputElement { fn default() -> Self { - todo!("what is the default input element?") + InputElement::Div + // Decided on InputElementDiv as default for now based on documentation from + // } } @@ -119,6 +131,10 @@ where )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), template_match!() => TemplateLiteral::new().lex(&mut self.cursor, start), + _ if next_chr.is_digit(10) => NumberLiteral::new(next_chr).lex(&mut self.cursor, start), + _ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => { + Identifier::new(next_chr).lex(&mut self.cursor, start) + } _ => unimplemented!(), }; diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs new file mode 100644 index 00000000000..bf30a7b1a8b --- /dev/null +++ b/boa/src/syntax/lexer/number.rs @@ -0,0 +1,285 @@ +use super::{Cursor, Error, Tokenizer}; +use crate::builtins::BigInt; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::token::NumericLiteral; +use crate::syntax::lexer::{Token, TokenKind}; +use std::io::{self, ErrorKind, Read}; +use std::str::FromStr; + +/// Number literal parsing. +/// +/// +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals +/// [mdn]: +#[derive(Debug, Clone, Copy)] +pub(super) struct NumberLiteral { + init: char, +} + +impl NumberLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char) -> Self { + Self { init: init } + } +} + +impl Tokenizer for NumberLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + unimplemented!("Number literal lexing"); + } +} + +/* +impl Tokenizer for NumberLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + /// This is a helper structure + /// + /// This structure helps with identifying what numerical type it is and what base is it. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum NumericKind { + Rational, + Integer(u8), + BigInt(u8), + } + + impl NumericKind { + /// Get the base of the number kind. + fn base(self) -> u32 { + match self { + Self::Rational => 10, + Self::Integer(base) => base as u32, + Self::BigInt(base) => base as u32, + } + } + + /// Converts `self` to BigInt kind. + fn to_bigint(self) -> Self { + match self { + Self::Rational => unreachable!("can not convert rational number to BigInt"), + Self::Integer(base) => Self::BigInt(base), + Self::BigInt(base) => Self::BigInt(base), + } + } + } + + // TODO: Setup strict mode. + let strict_mode = false; + + let mut buf = self.init.to_string(); + + let mut kind = NumericKind::Integer(10); + + if self.init == '0' { + match cursor.peek() { + None => { + cursor.next_column(); + return Ok(Token::new( + TokenKind::NumericLiteral(NumericLiteral::Integer(0)), + Span::new(start_pos, cursor.pos()), + )); + } + Some(r) => { + match r.map_err(|e| Error::IO(e))? { + 'x' | 'X' => { + cursor.next(); + cursor.next_column(); + kind = NumericKind::Integer(16); + } + 'o' | 'O' => { + cursor.next(); + cursor.next_column(); + kind = NumericKind::Integer(8); + } + 'b' | 'B' => { + cursor.next(); + cursor.next_column(); + kind = NumericKind::Integer(2); + } + ch if ch.is_ascii_digit() => { + let mut is_implicit_octal = true; + while let Some(ch) = cursor.peek(){ + let c = ch.map_err(|e| Error::IO(e))?; + if !c.is_ascii_digit() { + break; + } else if !c.is_digit(8) { + is_implicit_octal = false; + } + cursor.next(); + buf.push(c); + } + if !strict_mode { + if is_implicit_octal { + kind = NumericKind::Integer(8); + } + } else { + return Err(if is_implicit_octal { + Error::strict( + "Implicit octal literals are not allowed in strict mode.", + ) + } else { + Error::strict( + "Decimals with leading zeros are not allowed in strict mode.", + ) + }); + } + } + _ => {} + } + } + } + } + + while let Some(ch) = cursor.peek() { + let c = ch.map_err(|e| Error::IO(e))?; + if !c.is_digit(kind.base()) { + break; + } + cursor.next(); + buf.push(c); + } + + if cursor.next_is('n')? { + kind = kind.to_bigint(); + } + + if let NumericKind::Integer(10) = kind { + 'digitloop: while let Some(cx) = cursor.peek() { + match cx.map_err(|e| Error::IO(e))? { + '.' => loop { + kind = NumericKind::Rational; + cursor.next(); + buf.push('.'); + + let c = match cursor.peek() { + Some(ch) => ch.map_err(|e| Error::IO(e))?, + None => break, + }; + + match c { + 'e' | 'E' => { + cursor.next(); // Consume 'e' or 'E' + + match cursor.peek() { + None => { + cursor.next(); + } + Some(x) => { + let val = x.map_err(|e| Error::IO(e))?; + match val.to_digit(10) { + Some(0..=9) => { + cursor.next(); // Consume digit. + buf.push(val); + } + _ => { + break 'digitloop; + } + } + } + _ => { + break 'digitloop; + } + } + } + _ => { + if !c.is_digit(10) { + break 'digitloop; + } + } + } + }, + 'e' | 'E' => { + cursor.next(); // Consume 'e' or 'E' + kind = NumericKind::Rational; + match cursor.peek() { + None => { + cursor.next(); + } + Some(x) => { + let val = x.map_err(|e| Error::IO(e))?; + match val.to_digit(10) { + Some(0..=9) => { + cursor.next(); // Consume digit. + buf.push(val); + } + _ => { + break; + } + } + } + _ => { + break; + } + } + } + '+' | '-' => { + break; + } + x if x.is_digit(10) => { + buf.push(x); + } + _ => break, + } + } + } + + // Check the NumericLiteral is not followed by an `IdentifierStart` or `DecimalDigit` character. + match cursor.peek() { + Some(r) => { + let c = r.map_err(|e| Error::IO(e))?; + if c.is_ascii_alphabetic() || c == '$' || c == '_' || c.is_ascii_digit() { + return Err(Error::syntax("NumericLiteral token must not be followed by IdentifierStart nor DecimalDigit characters")); + } + }, + _ => {} + } + + let num = match kind { + NumericKind::BigInt(base) => { + NumericLiteral::BigInt( + BigInt::from_string_radix(&buf, base as u32).expect("Could not conver to BigInt") + ) + } + NumericKind::Rational /* base: 10 */ => { + NumericLiteral::Rational( + f64::from_str(&buf) + .map_err(|_| Error::syntax("Could not convert value to f64"))?, + ) + } + NumericKind::Integer(base) => { + if let Ok(num) = i32::from_str_radix(&buf, base as u32) { + NumericLiteral::Integer( + num + ) + } else { + let b = f64::from(base); + let mut result = 0.0_f64; + for c in buf.chars() { + let digit = f64::from(c.to_digit(base as u32).unwrap()); + result = result * b + digit; + } + + NumericLiteral::Rational(result) + } + + } + }; + + Ok(Token::new( + TokenKind::NumericLiteral(num), + Span::new(start_pos, cursor.pos()), + )) + } +} + +*/ From f47b44aac0245d2aed39bf5c08401a51fcf746d6 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 15:05:52 +0100 Subject: [PATCH 010/289] Lexing punctuation --- boa/src/syntax/lexer/comment.rs | 36 +++++++++++++++++++--- boa/src/syntax/lexer/cursor.rs | 2 +- boa/src/syntax/lexer/mod.rs | 22 +++++++++++-- boa/src/syntax/lexer/operator.rs | 31 +++++++++++++++++++ boa/src/syntax/lexer/spread.rs | 53 ++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 boa/src/syntax/lexer/operator.rs create mode 100644 boa/src/syntax/lexer/spread.rs diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 18b551723e6..a085371cd0c 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -3,6 +3,12 @@ use super::{Cursor, Error, Tokenizer}; use crate::syntax::ast::Position; use crate::syntax::lexer::{Token, TokenKind}; +use std::{ + char::{decode_utf16, from_u32}, + convert::TryFrom, + io::{self, ErrorKind, Read}, + str, +}; macro_rules! comment_match { () => {{ @@ -10,12 +16,32 @@ macro_rules! comment_match { }}; } -/// Lexes single line comments, starting with `//`. -#[derive(Debug, Clone, Copy)] -pub(super) struct SingleLineComment; +/// Skips comments. +/// +/// Assumes that the '/' char is already consumed. +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: +/// [mdn]: +pub(super) struct Comment; -impl Tokenizer for SingleLineComment { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result { +impl Comment { + /// Creates a new comment lexer. + pub(super) fn new() -> Self { + Self {} + } +} + + +impl Tokenizer for Comment { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { unimplemented!() } } + diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 833bf91c641..d4e44d3ba01 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -26,7 +26,7 @@ impl Cursor { /// Advances the position to the next line. #[inline] - fn next_line(&mut self) { + pub(super) fn next_line(&mut self) { let next_line = self.pos.line_number() + 1; self.pos = Position::new(next_line, 1); } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 0e74ff52f83..27e98230bec 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -17,6 +17,10 @@ mod template; mod number; +mod operator; + +mod spread; + mod identifier; // Temporary disabled while lexer in progress. @@ -27,9 +31,9 @@ pub use self::error::Error; use self::{ cursor::Cursor, identifier::Identifier, number::NumberLiteral, string::StringLiteral, - template::TemplateLiteral, + template::TemplateLiteral, spread::SpreadLiteral, comment::Comment, operator::Operator, }; -use crate::syntax::ast::{Position, Span}; +use crate::syntax::ast::{Position, Span, Punctuator}; use std::io::Read; pub use token::{Token, TokenKind}; @@ -135,6 +139,20 @@ where _ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => { Identifier::new(next_chr).lex(&mut self.cursor, start) } + ';' => Ok(Token::new(Punctuator::Semicolon.into(), Span::new(start, self.cursor.pos()))), + ':' => Ok(Token::new(Punctuator::Colon.into(), Span::new(start, self.cursor.pos()))), + '.' => SpreadLiteral::new().lex(&mut self.cursor, start), + '(' => Ok(Token::new(Punctuator::OpenParen.into(), Span::new(start, self.cursor.pos()))), + ')' => Ok(Token::new(Punctuator::CloseParen.into(), Span::new(start, self.cursor.pos()))), + ',' => Ok(Token::new(Punctuator::Comma.into(), Span::new(start, self.cursor.pos()))), + '{' => Ok(Token::new(Punctuator::OpenBlock.into(), Span::new(start, self.cursor.pos()))), + '}' => Ok(Token::new(Punctuator::CloseBlock.into(), Span::new(start, self.cursor.pos()))), + '[' => Ok(Token::new(Punctuator::OpenBracket.into(), Span::new(start, self.cursor.pos()))), + ']' => Ok(Token::new(Punctuator::CloseBracket.into(), Span::new(start, self.cursor.pos()))), + '?' => Ok(Token::new(Punctuator::Question.into(), Span::new(start, self.cursor.pos()))), + '/' => Comment::new().lex(&mut self.cursor, start), + '*' | '+' | '-' | '%' | '|' | '&' | '^' | '=' | '<' | '>' | '!' | '~' => Operator::new(next_chr).lex(&mut self.cursor, start), + _ => unimplemented!(), }; diff --git a/boa/src/syntax/lexer/operator.rs b/boa/src/syntax/lexer/operator.rs new file mode 100644 index 00000000000..4aff13b55ca --- /dev/null +++ b/boa/src/syntax/lexer/operator.rs @@ -0,0 +1,31 @@ +use super::{Cursor, Error, Tokenizer}; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{Token, TokenKind}; +use std::{ + char::{decode_utf16, from_u32}, + convert::TryFrom, + io::{self, ErrorKind, Read}, + str, +}; + + +#[derive(Debug, Clone, Copy)] +pub(super) struct Operator { + init: char +} + +impl Operator { + /// Creates a new string literal lexer. + pub(super) fn new(init: char) -> Self { + Self {init} + } +} + +impl Tokenizer for Operator { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + unimplemented!() + } +} \ No newline at end of file diff --git a/boa/src/syntax/lexer/spread.rs b/boa/src/syntax/lexer/spread.rs new file mode 100644 index 00000000000..75fed2ef9e2 --- /dev/null +++ b/boa/src/syntax/lexer/spread.rs @@ -0,0 +1,53 @@ +use super::{Cursor, Error, Tokenizer}; +use crate::syntax::ast::{Position, Span, Punctuator}; +use crate::syntax::lexer::{Token, TokenKind}; +use std::{ + char::{decode_utf16, from_u32}, + convert::TryFrom, + io::{self, ErrorKind, Read}, + str, +}; + +/// String literal lexing. +/// +/// Note: expects for the initializer `'` or `"` to already be consumed from the cursor. +#[derive(Debug, Clone, Copy)] +pub(super) struct SpreadLiteral; + +impl SpreadLiteral { + /// Creates a new string literal lexer. + pub(super) fn new() -> Self { + Self {} + } +} + +impl Tokenizer for SpreadLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + // . or ... + match cursor.next_is('.') { + Err(e) => { + Err(e.into()) + }, + Ok(true) => { + match cursor.next_is('.') { + Err(e) => { + Err(e.into()) + }, + Ok(true) => { + Ok(Token::new(Punctuator::Spread.into(), Span::new(start_pos, cursor.pos()))) + }, + Ok(false) => { + Err(Error::syntax("Expecting Token .")) + } + } + }, + Ok(false) => { + Ok(Token::new(Punctuator::Dot.into(), Span::new(start_pos, cursor.pos()))) + } + } + } +} + From 51c7425d2e9d6d97243df1f8de6c17e30de800fd Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 15:30:07 +0100 Subject: [PATCH 011/289] Branches of high level lexer created --- boa/src/syntax/lexer/comment.rs | 6 +-- boa/src/syntax/lexer/cursor.rs | 2 +- boa/src/syntax/lexer/mod.rs | 71 +++++++++++++++++++++++++------- boa/src/syntax/lexer/operator.rs | 7 ++-- boa/src/syntax/lexer/spread.rs | 33 ++++++--------- boa/src/syntax/lexer/token.rs | 9 ++++ 6 files changed, 83 insertions(+), 45 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index a085371cd0c..0a22517f309 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -17,14 +17,14 @@ macro_rules! comment_match { } /// Skips comments. -/// +/// /// Assumes that the '/' char is already consumed. /// /// More information: /// - [ECMAScript reference][spec] /// - [MDN documentation][mdn] /// -/// [spec]: +/// [spec]: /// [mdn]: pub(super) struct Comment; @@ -35,7 +35,6 @@ impl Comment { } } - impl Tokenizer for Comment { fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where @@ -44,4 +43,3 @@ impl Tokenizer for Comment { unimplemented!() } } - diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index d4e44d3ba01..c583c891c57 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -33,7 +33,7 @@ impl Cursor { /// Performs a carriage return to modify the position in the source. #[inline] - fn carriage_return(&mut self) { + pub(super) fn carriage_return(&mut self) { let current_line = self.pos.line_number(); self.pos = Position::new(current_line, 1); } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 27e98230bec..8e526b41c9c 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -30,10 +30,10 @@ mod identifier; pub use self::error::Error; use self::{ - cursor::Cursor, identifier::Identifier, number::NumberLiteral, string::StringLiteral, - template::TemplateLiteral, spread::SpreadLiteral, comment::Comment, operator::Operator, + comment::Comment, cursor::Cursor, identifier::Identifier, number::NumberLiteral, + operator::Operator, spread::SpreadLiteral, string::StringLiteral, template::TemplateLiteral, }; -use crate::syntax::ast::{Position, Span, Punctuator}; +use crate::syntax::ast::{Position, Punctuator, Span}; use std::io::Read; pub use token::{Token, TokenKind}; @@ -139,21 +139,60 @@ where _ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => { Identifier::new(next_chr).lex(&mut self.cursor, start) } - ';' => Ok(Token::new(Punctuator::Semicolon.into(), Span::new(start, self.cursor.pos()))), - ':' => Ok(Token::new(Punctuator::Colon.into(), Span::new(start, self.cursor.pos()))), + ';' => Ok(Token::new( + Punctuator::Semicolon.into(), + Span::new(start, self.cursor.pos()), + )), + ':' => Ok(Token::new( + Punctuator::Colon.into(), + Span::new(start, self.cursor.pos()), + )), '.' => SpreadLiteral::new().lex(&mut self.cursor, start), - '(' => Ok(Token::new(Punctuator::OpenParen.into(), Span::new(start, self.cursor.pos()))), - ')' => Ok(Token::new(Punctuator::CloseParen.into(), Span::new(start, self.cursor.pos()))), - ',' => Ok(Token::new(Punctuator::Comma.into(), Span::new(start, self.cursor.pos()))), - '{' => Ok(Token::new(Punctuator::OpenBlock.into(), Span::new(start, self.cursor.pos()))), - '}' => Ok(Token::new(Punctuator::CloseBlock.into(), Span::new(start, self.cursor.pos()))), - '[' => Ok(Token::new(Punctuator::OpenBracket.into(), Span::new(start, self.cursor.pos()))), - ']' => Ok(Token::new(Punctuator::CloseBracket.into(), Span::new(start, self.cursor.pos()))), - '?' => Ok(Token::new(Punctuator::Question.into(), Span::new(start, self.cursor.pos()))), + '(' => Ok(Token::new( + Punctuator::OpenParen.into(), + Span::new(start, self.cursor.pos()), + )), + ')' => Ok(Token::new( + Punctuator::CloseParen.into(), + Span::new(start, self.cursor.pos()), + )), + ',' => Ok(Token::new( + Punctuator::Comma.into(), + Span::new(start, self.cursor.pos()), + )), + '{' => Ok(Token::new( + Punctuator::OpenBlock.into(), + Span::new(start, self.cursor.pos()), + )), + '}' => Ok(Token::new( + Punctuator::CloseBlock.into(), + Span::new(start, self.cursor.pos()), + )), + '[' => Ok(Token::new( + Punctuator::OpenBracket.into(), + Span::new(start, self.cursor.pos()), + )), + ']' => Ok(Token::new( + Punctuator::CloseBracket.into(), + Span::new(start, self.cursor.pos()), + )), + '?' => Ok(Token::new( + Punctuator::Question.into(), + Span::new(start, self.cursor.pos()), + )), '/' => Comment::new().lex(&mut self.cursor, start), - '*' | '+' | '-' | '%' | '|' | '&' | '^' | '=' | '<' | '>' | '!' | '~' => Operator::new(next_chr).lex(&mut self.cursor, start), - - _ => unimplemented!(), + '*' | '+' | '-' | '%' | '|' | '&' | '^' | '=' | '<' | '>' | '!' | '~' => { + Operator::new(next_chr).lex(&mut self.cursor, start) + } + _ => { + let details = format!( + "Unexpected '{}' at line {}, column {}", + next_chr, + start.line_number(), + start.column_number() + ); + Err(Error::syntax(details)) + } }; Some(token) diff --git a/boa/src/syntax/lexer/operator.rs b/boa/src/syntax/lexer/operator.rs index 4aff13b55ca..63256ad2d52 100644 --- a/boa/src/syntax/lexer/operator.rs +++ b/boa/src/syntax/lexer/operator.rs @@ -8,16 +8,15 @@ use std::{ str, }; - #[derive(Debug, Clone, Copy)] pub(super) struct Operator { - init: char + init: char, } impl Operator { /// Creates a new string literal lexer. pub(super) fn new(init: char) -> Self { - Self {init} + Self { init } } } @@ -28,4 +27,4 @@ impl Tokenizer for Operator { { unimplemented!() } -} \ No newline at end of file +} diff --git a/boa/src/syntax/lexer/spread.rs b/boa/src/syntax/lexer/spread.rs index 75fed2ef9e2..f4503b0e870 100644 --- a/boa/src/syntax/lexer/spread.rs +++ b/boa/src/syntax/lexer/spread.rs @@ -1,5 +1,5 @@ use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::{Position, Span, Punctuator}; +use crate::syntax::ast::{Position, Punctuator, Span}; use crate::syntax::lexer::{Token, TokenKind}; use std::{ char::{decode_utf16, from_u32}, @@ -28,26 +28,19 @@ impl Tokenizer for SpreadLiteral { { // . or ... match cursor.next_is('.') { - Err(e) => { - Err(e.into()) + Err(e) => Err(e.into()), + Ok(true) => match cursor.next_is('.') { + Err(e) => Err(e.into()), + Ok(true) => Ok(Token::new( + Punctuator::Spread.into(), + Span::new(start_pos, cursor.pos()), + )), + Ok(false) => Err(Error::syntax("Expecting Token .")), }, - Ok(true) => { - match cursor.next_is('.') { - Err(e) => { - Err(e.into()) - }, - Ok(true) => { - Ok(Token::new(Punctuator::Spread.into(), Span::new(start_pos, cursor.pos()))) - }, - Ok(false) => { - Err(Error::syntax("Expecting Token .")) - } - } - }, - Ok(false) => { - Ok(Token::new(Punctuator::Dot.into(), Span::new(start_pos, cursor.pos()))) - } + Ok(false) => Ok(Token::new( + Punctuator::Dot.into(), + Span::new(start_pos, cursor.pos()), + )), } } } - diff --git a/boa/src/syntax/lexer/token.rs b/boa/src/syntax/lexer/token.rs index d6398c57759..9a8b1aabf0a 100644 --- a/boa/src/syntax/lexer/token.rs +++ b/boa/src/syntax/lexer/token.rs @@ -247,6 +247,9 @@ pub enum TokenKind { /// Indicates the end of a line (`\n`). LineTerminator, + + /// Indicates a comment, the content isn't stored. + Comment, } impl From for TokenKind { @@ -332,6 +335,11 @@ impl TokenKind { pub fn line_terminator() -> Self { Self::LineTerminator } + + /// Creates a 'Comment' token kind. + pub fn comment() -> Self { + Self::Comment + } } impl Display for TokenKind { @@ -350,6 +358,7 @@ impl Display for TokenKind { Self::TemplateLiteral(ref lit) => write!(f, "{}", lit), Self::RegularExpressionLiteral(ref body, ref flags) => write!(f, "/{}/{}", body, flags), Self::LineTerminator => write!(f, "line terminator"), + Self::Comment => write!(f, "comment"), } } } From 3bcce8a8b31d124b6a10828e7c209690dbaa5da2 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 15:46:54 +0100 Subject: [PATCH 012/289] Fixed clippy warnings (some temporarily) --- boa/src/lib.rs | 2 +- boa/src/syntax/lexer/comment.rs | 15 +++++-------- boa/src/syntax/lexer/cursor.rs | 17 +++++++++------ boa/src/syntax/lexer/error.rs | 2 +- boa/src/syntax/lexer/identifier.rs | 3 +-- boa/src/syntax/lexer/mod.rs | 34 ++++++------------------------ boa/src/syntax/lexer/number.rs | 11 ++++------ boa/src/syntax/lexer/operator.rs | 13 ++++-------- boa/src/syntax/lexer/spread.rs | 9 ++------ boa/src/syntax/lexer/tests.rs | 20 ++++++++++++++++++ boa/src/syntax/parser/tests.rs | 5 ++--- boa_cli/src/main.rs | 2 +- boa_wasm/src/lib.rs | 2 +- 13 files changed, 58 insertions(+), 77 deletions(-) diff --git a/boa/src/lib.rs b/boa/src/lib.rs index f43d27e1efe..accd21f3581 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -50,7 +50,7 @@ pub use crate::{ }; fn parser_expr(src: &str) -> Result { - let mut lexer = Lexer::new(src.as_bytes()); + let lexer = Lexer::new(src.as_bytes()); // Goes through and lexes entire given string before starting any parsing. let mut tokens = Vec::new(); diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 0a22517f309..3b524d29f43 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -2,18 +2,13 @@ use super::{Cursor, Error, Tokenizer}; use crate::syntax::ast::Position; -use crate::syntax::lexer::{Token, TokenKind}; -use std::{ - char::{decode_utf16, from_u32}, - convert::TryFrom, - io::{self, ErrorKind, Read}, - str, -}; +use crate::syntax::lexer::Token; +use std::io::Read; macro_rules! comment_match { - () => {{ + () => { '/' - }}; + }; } /// Skips comments. @@ -36,7 +31,7 @@ impl Comment { } impl Tokenizer for Comment { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + fn lex(&mut self, _cursor: &mut Cursor, _start_pos: Position) -> Result where R: Read, { diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index c583c891c57..410672b41ad 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -70,31 +70,31 @@ where true } _ => false, - Some(&Err(_)) => return self.peeked.take().unwrap().unwrap().map(|_| false), + // Some(&Err(_)) => return self.peeked.take().unwrap().unwrap().map(|_| false), }) } /// Fills the buffer with all characters until the stop character is found. /// /// Note: It will not add the stop character to the buffer. - pub(super) fn take_until(&mut self, stop: char, buf: &mut String) -> io::Result<()> { + pub(super) fn take_until(&mut self, _stop: char, _buf: &mut String) -> io::Result<()> { unimplemented!() } /// Retrieves the given number of characters and adds them to the buffer. - pub(super) fn take(&mut self, count: usize, buf: &mut String) -> io::Result<()> { + pub(super) fn _take(&mut self, _count: usize, _buf: &mut String) -> io::Result<()> { unimplemented!() } /// It will fill the buffer with checked ASCII bytes. - pub(super) fn fill_bytes(&mut self, buf: &[u8]) -> io::Result<()> { + pub(super) fn fill_bytes(&mut self, _buf: &mut [u8]) -> io::Result<()> { unimplemented!() } /// Retrieves the next character as an ASCII character. /// /// It will make sure that the next character is an ASCII byte, or return an error otherwise. - pub(super) fn next_as_byte(&mut self) -> Option> { + pub(super) fn _next_as_byte(&mut self) -> Option> { unimplemented!() } } @@ -113,8 +113,11 @@ where }; match chr { - Some(Ok('\r')) => self.carriage_return(), - Some(Ok('\r')) | Some(Ok('\u{2028}')) | Some(Ok('\u{2029}')) => self.next_line(), + Some(Ok('\r')) => { + self.carriage_return(); + self.next_line() + } + Some(Ok('\u{2028}')) | Some(Ok('\u{2029}')) => self.next_line(), Some(Ok(_)) => self.next_column(), _ => {} } diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 3c1b65e2a3b..3f9ac0be4e0 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -27,7 +27,7 @@ impl Error { /// This error is used to represent the case where a piece of javascript /// cannot be lexed/parsed because it is in invalid when strict mdoe is /// enabled. - pub(super) fn strict(err: M) -> Self + pub(super) fn _strict(err: M) -> Self where M: Into>, { diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index f5c95c370ee..ae350f51334 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -1,9 +1,8 @@ use super::{Cursor, Error, Tokenizer}; -use crate::builtins::BigInt; use crate::syntax::ast::{Position, Span}; use crate::syntax::lexer::token::NumericLiteral; use crate::syntax::lexer::{Token, TokenKind}; -use std::io::{self, ErrorKind, Read}; +use std::io::Read; use std::str::FromStr; /// Identifier or keyword lexing. diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 8e526b41c9c..df4413e202c 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -5,6 +5,7 @@ #[macro_use] mod comment; + mod cursor; pub mod error; @@ -70,7 +71,7 @@ impl Lexer { } /// Sets the goal symbol for the lexer. - pub(crate) fn set_goal(&mut self, elm: InputElement) { + pub(crate) fn _set_goal(&mut self, elm: InputElement) { self.goal_symbol = elm; } } @@ -95,9 +96,9 @@ where #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum InputElement { Div, - RegExp, - RegExpOrTemplateTail, - TemplateTail, + _RegExp, + _RegExpOrTemplateTail, + _TemplateTail, } impl Default for InputElement { @@ -180,7 +181,7 @@ where Punctuator::Question.into(), Span::new(start, self.cursor.pos()), )), - '/' => Comment::new().lex(&mut self.cursor, start), + comment_match!() => Comment::new().lex(&mut self.cursor, start), '*' | '+' | '-' | '%' | '|' | '&' | '^' | '=' | '<' | '>' | '!' | '~' => { Operator::new(next_chr).lex(&mut self.cursor, start) } @@ -207,26 +208,3 @@ where // } // } - -// Temporarily moved. -use crate::syntax::ast::Keyword; - -#[test] -fn check_single_line_comment() { - let s1 = "var \n//This is a comment\ntrue"; - let mut lexer = Lexer::new(s1.as_bytes()); - - assert_eq!( - lexer.next().unwrap().unwrap().kind, - TokenKind::Keyword(Keyword::Var) - ); - assert_eq!( - lexer.next().unwrap().unwrap().kind, - TokenKind::LineTerminator - ); - assert_eq!( - lexer.next().unwrap().unwrap().kind, - TokenKind::BooleanLiteral(true) - ); - assert!(lexer.next().is_none()); -} diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index bf30a7b1a8b..d396567d81c 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -1,10 +1,7 @@ use super::{Cursor, Error, Tokenizer}; -use crate::builtins::BigInt; -use crate::syntax::ast::{Position, Span}; -use crate::syntax::lexer::token::NumericLiteral; -use crate::syntax::lexer::{Token, TokenKind}; -use std::io::{self, ErrorKind, Read}; -use std::str::FromStr; +use crate::syntax::ast::Position; +use crate::syntax::lexer::Token; +use std::io::Read; /// Number literal parsing. /// @@ -29,7 +26,7 @@ impl NumberLiteral { } impl Tokenizer for NumberLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + fn lex(&mut self, _cursor: &mut Cursor, _start_pos: Position) -> Result where R: Read, { diff --git a/boa/src/syntax/lexer/operator.rs b/boa/src/syntax/lexer/operator.rs index 63256ad2d52..b106c7d8caa 100644 --- a/boa/src/syntax/lexer/operator.rs +++ b/boa/src/syntax/lexer/operator.rs @@ -1,12 +1,7 @@ use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::{Position, Span}; -use crate::syntax::lexer::{Token, TokenKind}; -use std::{ - char::{decode_utf16, from_u32}, - convert::TryFrom, - io::{self, ErrorKind, Read}, - str, -}; +use crate::syntax::ast::Position; +use crate::syntax::lexer::Token; +use std::io::Read; #[derive(Debug, Clone, Copy)] pub(super) struct Operator { @@ -21,7 +16,7 @@ impl Operator { } impl Tokenizer for Operator { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + fn lex(&mut self, _cursor: &mut Cursor, _start_pos: Position) -> Result where R: Read, { diff --git a/boa/src/syntax/lexer/spread.rs b/boa/src/syntax/lexer/spread.rs index f4503b0e870..482f8480898 100644 --- a/boa/src/syntax/lexer/spread.rs +++ b/boa/src/syntax/lexer/spread.rs @@ -1,12 +1,7 @@ use super::{Cursor, Error, Tokenizer}; use crate::syntax::ast::{Position, Punctuator, Span}; -use crate::syntax::lexer::{Token, TokenKind}; -use std::{ - char::{decode_utf16, from_u32}, - convert::TryFrom, - io::{self, ErrorKind, Read}, - str, -}; +use crate::syntax::lexer::Token; +use std::io::Read; /// String literal lexing. /// diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 30f3fa06348..695fa69c703 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -8,6 +8,26 @@ fn span(start: (u32, u32), end: (u32, u32)) -> Span { Span::new(Position::new(start.0, start.1), Position::new(end.0, end.1)) } +#[test] +fn check_single_line_comment() { + let s1 = "var \n//This is a comment\ntrue"; + let mut lexer = Lexer::new(s1.as_bytes()); + + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::Keyword(Keyword::Var) + ); + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::LineTerminator + ); + assert_eq!( + lexer.next().unwrap().unwrap().kind, + TokenKind::BooleanLiteral(true) + ); + assert!(lexer.next().is_none()); +} + #[test] fn check_single_line_comment() { let s1 = "var \n//This is a comment\ntrue"; diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 30c8e4443c1..f1af394521f 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -20,7 +20,7 @@ pub(super) fn check_parser(js: &str, expr: L) where L: Into>, { - let mut lexer = Lexer::new(js.as_bytes()); + let lexer = Lexer::new(js.as_bytes()); // Goes through and lexes entire given string. let mut tokens = Vec::new(); @@ -38,8 +38,7 @@ where /// Checks that the given javascript string creates a parse error. // TODO: #[track_caller]: https://github.com/rust-lang/rust/issues/47809 pub(super) fn check_invalid(js: &str) { - let mut lexer = Lexer::new(js.as_bytes()); - // lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(js.as_bytes()); let tokens = lexer.collect::, _>>().expect("failed to lex"); diff --git a/boa_cli/src/main.rs b/boa_cli/src/main.rs index aa1c20f9a0b..f2445372a5a 100644 --- a/boa_cli/src/main.rs +++ b/boa_cli/src/main.rs @@ -112,7 +112,7 @@ arg_enum! { /// Returns a error of type String with a message, /// if the source has a syntax error. fn lex_source(src: &str) -> Result, String> { - let mut lexer = Lexer::new(src.as_bytes()); + let lexer = Lexer::new(src.as_bytes()); // Goes through and lexes entire given string. lexer diff --git a/boa_wasm/src/lib.rs b/boa_wasm/src/lib.rs index 81b48606e32..fa5b5a09d5c 100644 --- a/boa_wasm/src/lib.rs +++ b/boa_wasm/src/lib.rs @@ -3,7 +3,7 @@ use wasm_bindgen::prelude::*; #[wasm_bindgen] pub fn evaluate(src: &str) -> Result { - let mut lexer = Lexer::new(src.as_bytes()); + let lexer = Lexer::new(src.as_bytes()); // Goes through and lexes entire given string. let tokens = lexer From 9e73a6bbc0b08424e8f8550c64c6d28328ee209e Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 15:50:07 +0100 Subject: [PATCH 013/289] Small explaination comment modification --- boa/src/syntax/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/src/syntax/mod.rs b/boa/src/syntax/mod.rs index 739c4a069b3..6605bf23af7 100644 --- a/boa/src/syntax/mod.rs +++ b/boa/src/syntax/mod.rs @@ -1,4 +1,4 @@ -//! Syntactical analysis, such as AST, Parsing and Lexing +//! Syntactical analysis, such as Abstract Syntax Tree (AST), Parsing and Lexing pub mod ast; pub mod lexer; From 514680d1fa2deaeb71e92ce997920b9de447d7c3 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 15:56:37 +0100 Subject: [PATCH 014/289] Fixed warning in cursor --- boa/src/syntax/lexer/cursor.rs | 4 ++-- boa/src/syntax/lexer/identifier.rs | 2 +- boa/src/syntax/lexer/number.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 410672b41ad..1a81bc5dda6 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -169,7 +169,7 @@ where 4 }; - for i in 1..num_bytes { + for b in buf.iter_mut().take(num_bytes).skip(1) { let next = match self.iter.next() { Some(Ok(b)) => b, Some(Err(e)) => return Some(Err(e)), @@ -181,7 +181,7 @@ where } }; - buf[i] = next; + *b = next; } let int = u32::from_le_bytes(buf); diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index ae350f51334..dbd03640321 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -23,7 +23,7 @@ pub(super) struct Identifier { impl Identifier { /// Creates a new identifier/keyword lexer. pub(super) fn new(init: char) -> Self { - Self { init: init } + Self { init } } } diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index d396567d81c..1ece8d345b0 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -21,7 +21,7 @@ pub(super) struct NumberLiteral { impl NumberLiteral { /// Creates a new string literal lexer. pub(super) fn new(init: char) -> Self { - Self { init: init } + Self { init } } } From 81d847f26f4975cea5d7f5a752579a6a9d94633f Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 16:05:16 +0100 Subject: [PATCH 015/289] Fixed benchmarks lexer usage --- boa/benches/exec.rs | 80 +++++++++++++++++++++------------- boa/benches/lexer.rs | 15 ++++--- boa/benches/parser.rs | 31 ++++++++----- boa/src/syntax/parser/tests.rs | 7 +-- 4 files changed, 80 insertions(+), 53 deletions(-) diff --git a/boa/benches/exec.rs b/boa/benches/exec.rs index 1b84ed1c4ae..74d97ff6148 100644 --- a/boa/benches/exec.rs +++ b/boa/benches/exec.rs @@ -26,11 +26,13 @@ fn symbol_creation(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(SYMBOL_CREATION)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(SYMBOL_CREATION.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Symbols (Execution)", move |b| { @@ -57,11 +59,13 @@ fn for_loop_execution(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(FOR_LOOP)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(FOR_LOOP.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("For loop (Execution)", move |b| { @@ -88,11 +92,13 @@ fn fibonacci(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(FIBONACCI)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(FIBONACCI.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Fibonacci (Execution)", move |b| { @@ -117,11 +123,13 @@ fn object_creation(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(OBJECT_CREATION)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(OBJECT_CREATION.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Object Creation (Execution)", move |b| { @@ -146,11 +154,13 @@ fn object_prop_access_const(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(OBJECT_PROP_ACCESS_CONST)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(OBJECT_PROP_ACCESS_CONST.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Static Object Property Access (Execution)", move |b| { @@ -175,11 +185,13 @@ fn object_prop_access_dyn(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(OBJECT_PROP_ACCESS_DYN)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(OBJECT_PROP_ACCESS_DYN.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Dynamic Object Property Access (Execution)", move |b| { @@ -201,11 +213,13 @@ fn regexp_literal_creation(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(REGEXP_LITERAL_CREATION)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(REGEXP_LITERAL_CREATION.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("RegExp Literal Creation (Execution)", move |b| { @@ -227,11 +241,13 @@ fn regexp_creation(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(REGEXP_CREATION)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(REGEXP_CREATION.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("RegExp (Execution)", move |b| { @@ -253,11 +269,13 @@ fn regexp_literal(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(REGEXP_LITERAL)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(REGEXP_LITERAL.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("RegExp Literal (Execution)", move |b| { @@ -279,11 +297,13 @@ fn regexp(c: &mut Criterion) { let mut engine = Interpreter::new(realm); // Lex all the tokens. - let mut lexer = Lexer::new(black_box(REGEXP)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(REGEXP.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); // Parse the AST nodes. - let nodes = Parser::new(&black_box(lexer.tokens)).parse_all().unwrap(); + let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("RegExp (Execution)", move |b| { diff --git a/boa/benches/lexer.rs b/boa/benches/lexer.rs index bd441e4905c..905de9ed198 100644 --- a/boa/benches/lexer.rs +++ b/boa/benches/lexer.rs @@ -17,9 +17,10 @@ static EXPRESSION: &str = r#" fn expression_lexer(c: &mut Criterion) { c.bench_function("Expression (Lexer)", move |b| { b.iter(|| { - let mut lexer = Lexer::new(black_box(EXPRESSION)); + let lexer = Lexer::new(black_box(EXPRESSION.as_bytes())); - lexer.lex() + // Goes through and lexes entire given string. + lexer.collect::, _>>().expect("failed to lex"); }) }); } @@ -29,10 +30,11 @@ static HELLO_WORLD: &str = "let foo = 'hello world!'; foo;"; fn hello_world_lexer(c: &mut Criterion) { c.bench_function("Hello World (Lexer)", move |b| { b.iter(|| { - let mut lexer = Lexer::new(black_box(HELLO_WORLD)); + let lexer = Lexer::new(black_box(HELLO_WORLD.as_bytes())); // return the value into the blackbox so its not optimized away // https://gist.github.com/jasonwilliams/5325da61a794d8211dcab846d466c4fd - lexer.lex() + // Goes through and lexes entire given string. + lexer.collect::, _>>().expect("failed to lex"); }) }); } @@ -52,9 +54,10 @@ for (let a = 10; a < 100; a++) { fn for_loop_lexer(c: &mut Criterion) { c.bench_function("For loop (Lexer)", move |b| { b.iter(|| { - let mut lexer = Lexer::new(black_box(FOR_LOOP)); + let lexer = Lexer::new(black_box(FOR_LOOP.as_bytes())); - lexer.lex() + // Goes through and lexes entire given string. + lexer.collect::, _>>().expect("failed to lex"); }) }); } diff --git a/boa/benches/parser.rs b/boa/benches/parser.rs index 9a58aff5442..057b9a135e9 100644 --- a/boa/benches/parser.rs +++ b/boa/benches/parser.rs @@ -19,10 +19,12 @@ fn expression_parser(c: &mut Criterion) { c.bench_function("Expression (Parser)", move |b| { b.iter(|| { - let mut lexer = Lexer::new(black_box(EXPRESSION)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(EXPRESSION.as_bytes())); - Parser::new(&black_box(lexer.tokens)).parse_all() + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); + + Parser::new(&black_box(tokens)).parse_all() }) }); } @@ -34,10 +36,11 @@ fn hello_world_parser(c: &mut Criterion) { c.bench_function("Hello World (Parser)", move |b| { b.iter(|| { - let mut lexer = Lexer::new(black_box(HELLO_WORLD)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(HELLO_WORLD.as_bytes())); + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); - Parser::new(&black_box(lexer.tokens)).parse_all() + Parser::new(&black_box(tokens)).parse_all() }) }); } @@ -59,10 +62,12 @@ fn for_loop_parser(c: &mut Criterion) { c.bench_function("For loop (Parser)", move |b| { b.iter(|| { - let mut lexer = Lexer::new(black_box(FOR_LOOP)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(FOR_LOOP.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); - Parser::new(&black_box(lexer.tokens)).parse_all() + Parser::new(&black_box(tokens)).parse_all() }) }); } @@ -101,10 +106,12 @@ fn long_file_parser(c: &mut Criterion) { let file_str = fs::read_to_string(FILE_NAME) .unwrap_or_else(|_| panic!("could not read {}", FILE_NAME)); - let mut lexer = Lexer::new(black_box(&file_str)); - lexer.lex().expect("failed to lex"); + let lexer = Lexer::new(black_box(file_str.as_bytes())); + + // Goes through and lexes entire given string. + let tokens = lexer.collect::, _>>().expect("failed to lex"); - Parser::new(&black_box(lexer.tokens)).parse_all() + Parser::new(&black_box(tokens)).parse_all() }) }); diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index f1af394521f..47fba3fdd10 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -23,11 +23,7 @@ where let lexer = Lexer::new(js.as_bytes()); // Goes through and lexes entire given string. - let mut tokens = Vec::new(); - - for token in lexer { - tokens.push(token.expect("failed to lex")); - } + let tokens = lexer.collect::, _>>().expect("failed to lex"); assert_eq!( Parser::new(&tokens).parse_all().expect("failed to parse"), @@ -40,6 +36,7 @@ where pub(super) fn check_invalid(js: &str) { let lexer = Lexer::new(js.as_bytes()); + // Goes through and lexes entire given string. let tokens = lexer.collect::, _>>().expect("failed to lex"); assert!(Parser::new(&tokens).parse_all().is_err()); From 8846f5e801a630d0c573e2d5a28fd99a9a86a380 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 18:27:41 +0100 Subject: [PATCH 016/289] Operator lexing reimplemented --- boa/src/syntax/lexer/number.rs | 4 +- boa/src/syntax/lexer/operator.rs | 148 +++++++++++++++++++++++++++++-- 2 files changed, 144 insertions(+), 8 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 1ece8d345b0..593eed36ee8 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -3,9 +3,9 @@ use crate::syntax::ast::Position; use crate::syntax::lexer::Token; use std::io::Read; -/// Number literal parsing. -/// +/// Number literal lexing. /// +/// Assumes the digit is consumed by the cursor (stored in init). /// /// More information: /// - [ECMAScript reference][spec] diff --git a/boa/src/syntax/lexer/operator.rs b/boa/src/syntax/lexer/operator.rs index b106c7d8caa..8fd878ed12f 100644 --- a/boa/src/syntax/lexer/operator.rs +++ b/boa/src/syntax/lexer/operator.rs @@ -1,25 +1,161 @@ use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::Position; -use crate::syntax::lexer::Token; -use std::io::Read; +use crate::syntax::ast::{Position, Punctuator, Span}; +use crate::syntax::lexer::{Token, TokenKind}; +use std::io::{self, ErrorKind, Read}; + +/// `vop` tests the next token to see if we're on an assign operation of just a plain binary operation. +/// +/// If the next value is not an assignment operation it will pattern match the provided values and return the corresponding token. +macro_rules! vop { + ($cursor:ident, $assign_op:expr, $op:expr) => ({ + match $cursor.peek() { + None | Some(Err(_)) => { + Err(Error::syntax("could not preview next value")) + } + Some(Ok('=')) => { + $cursor.next(); + $cursor.next_column(); + $assign_op + } + Some(Ok(_)) => $op, + } + }); + ($cursor:ident, $assign_op:expr, $op:expr, {$($case:pat => $block:expr), +}) => ({ + // let punc = $cursor.peek().ok_or_else(|| Error::syntax("could not preview next value"))?; + match $cursor.peek() { + None | Some(Err(_)) => { + Err(Error::syntax("could not preview next value")) + } + Some(Ok('=')) => { + $cursor.next(); + $cursor.next_column(); + $assign_op + }, + $($case => { + $cursor.next(); + $cursor.next_column(); + $block + })+, + + _ => $op, + } + }); + ($cursor:ident, $op:expr, {$($case:pat => $block:expr),+}) => { + match $cursor.peek().ok_or_else(|| LexerError::syntax("could not preview next value"))? { + $($case => { + $cursor.next()?; + $cursor.next_column(); + $block + })+, + _ => $op + } + } +} + +/// The `op` macro handles binary operations or assignment operations and converts them into tokens. +macro_rules! op { + ($cursor:ident, $start_pos:expr, $assign_op:expr, $op:expr) => ({ + Ok(Token::new( + vop!($cursor, $assign_op, $op)?.into(), + Span::new($start_pos, $cursor.pos()), + )) + }); + ($cursor:ident, $start_pos:expr, $assign_op:expr, $op:expr, {$($case:pat => $block:expr),+}) => ({ + let punc: Result = vop!($cursor, $assign_op, $op, {$($case => $block),+}); + Ok(Token::new( + punc?.into(), + Span::new($start_pos, $cursor.pos()), + )) + }); +} #[derive(Debug, Clone, Copy)] pub(super) struct Operator { init: char, } +/// Operator lexing. +/// +/// Assumes that the cursor has already consumed the operator starting symbol (stored in init). +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: +/// [mdn]: impl Operator { - /// Creates a new string literal lexer. + /// Creates a new operator lexer. pub(super) fn new(init: char) -> Self { Self { init } } } impl Tokenizer for Operator { - fn lex(&mut self, _cursor: &mut Cursor, _start_pos: Position) -> Result + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where R: Read, { - unimplemented!() + match self.init { + '*' => op!(cursor, start_pos, Ok(Punctuator::AssignMul), Ok(Punctuator::Mul), { + Some(Ok('*')) => vop!(cursor, Ok(Punctuator::AssignPow), Ok(Punctuator::Exp)) + }), + '+' => op!(cursor, start_pos, Ok(Punctuator::AssignAdd), Ok(Punctuator::Add), { + Some(Ok('+')) => Ok(Punctuator::Inc) + }), + '-' => op!(cursor, start_pos, Ok(Punctuator::AssignSub), Ok(Punctuator::Sub), { + Some(Ok('-')) => { + Ok(Punctuator::Dec) + } + }), + '%' => op!( + cursor, + start_pos, + Ok(Punctuator::AssignMod), + Ok(Punctuator::Mod) + ), + '|' => op!(cursor, start_pos, Ok(Punctuator::AssignOr), Ok(Punctuator::Or), { + Some(Ok('|')) => Ok(Punctuator::BoolOr) + }), + '&' => op!(cursor, start_pos, Ok(Punctuator::AssignAnd), Ok(Punctuator::And), { + Some(Ok('&')) => Ok(Punctuator::BoolAnd) + }), + '^' => op!( + cursor, + start_pos, + Ok(Punctuator::AssignXor), + Ok(Punctuator::Xor) + ), + '=' => op!(cursor, start_pos, if cursor.next_is('=')? { + Ok(Punctuator::StrictEq) + } else { + Ok(Punctuator::Eq) + }, Ok(Punctuator::Assign), { + Some(Ok('>')) => { + Ok(Punctuator::Arrow) + } + }), + '<' => op!(cursor, start_pos, Ok(Punctuator::LessThanOrEq), Ok(Punctuator::LessThan), { + Some(Ok('<')) => vop!(cursor, Ok(Punctuator::AssignLeftSh), Ok(Punctuator::LeftSh)) + }), + '>' => { + op!(cursor, start_pos, Ok(Punctuator::GreaterThanOrEq), Ok(Punctuator::GreaterThan), { + Some(Ok('>')) => vop!(cursor, Ok(Punctuator::AssignRightSh), Ok(Punctuator::RightSh), { + Some(Ok('>')) => vop!(cursor, Ok(Punctuator::AssignURightSh), Ok(Punctuator::URightSh)) + }) + }) + } + '!' => op!( + cursor, + start_pos, + vop!(cursor, Ok(Punctuator::StrictNotEq), Ok(Punctuator::NotEq)), + Ok(Punctuator::Not) + ), + '~' => Ok(Token::new( + Punctuator::Neg.into(), + Span::new(start_pos, cursor.pos()), + )), + _ => unimplemented!(), + } } } From 17b0f463f4d3cd6605c7ed16ba7ae4211b231c2d Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 18:28:39 +0100 Subject: [PATCH 017/289] Removed unused imports --- boa/src/syntax/lexer/operator.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boa/src/syntax/lexer/operator.rs b/boa/src/syntax/lexer/operator.rs index 8fd878ed12f..5e12890f27d 100644 --- a/boa/src/syntax/lexer/operator.rs +++ b/boa/src/syntax/lexer/operator.rs @@ -1,7 +1,7 @@ use super::{Cursor, Error, Tokenizer}; use crate::syntax::ast::{Position, Punctuator, Span}; -use crate::syntax::lexer::{Token, TokenKind}; -use std::io::{self, ErrorKind, Read}; +use crate::syntax::lexer::Token; +use std::io::Read; /// `vop` tests the next token to see if we're on an assign operation of just a plain binary operation. /// From ec782f1e2808dab49a0a9d0de68d728d06f1ed93 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 12 Jun 2020 21:52:17 +0100 Subject: [PATCH 018/289] Implemented comment lexing, half of regex lexing --- boa/src/syntax/lexer/comment.rs | 71 +++++++++++++++++--- boa/src/syntax/lexer/mod.rs | 5 +- boa/src/syntax/lexer/regex.rs | 109 +++++++++++++++++++++++++++++++ boa/src/syntax/lexer/template.rs | 4 +- 4 files changed, 178 insertions(+), 11 deletions(-) create mode 100644 boa/src/syntax/lexer/regex.rs diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 3b524d29f43..c41fd7fbf9f 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -1,9 +1,7 @@ -//! Coments lexing. - -use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::Position; -use crate::syntax::lexer::Token; -use std::io::Read; +use super::{Cursor, Error, RegexLiteral, Tokenizer}; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{Token, TokenKind}; +use std::io::{ErrorKind, Read}; macro_rules! comment_match { () => { @@ -31,10 +29,67 @@ impl Comment { } impl Tokenizer for Comment { - fn lex(&mut self, _cursor: &mut Cursor, _start_pos: Position) -> Result + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where R: Read, { - unimplemented!() + match cursor.peek() { + None => Err(Error::syntax("Expecting Token /,*,= or regex")), + Some(Err(_)) => Err(Error::from(std::io::Error::new( + ErrorKind::Interrupted, + "Failed to peek next character", + ))), + Some(Ok(ch)) => { + match ch { + '/' => { + // Skip either to the end of the line or to the end of the input + while let Some(ch) = cursor.next() { + match ch { + Err(e) => { + return Err(Error::IO(e)); + } + Ok('\n') => { + break; + } + _ => {} + } + } + cursor.next_line(); + Ok(Token::new( + TokenKind::Comment, + Span::new(start_pos, cursor.pos()), + )) + } + // block comment + '*' => { + loop { + if let Some(ch) = cursor.next() { + match ch { + Err(e) => { + return Err(Error::IO(e)); + } + Ok('\n') => { + cursor.next_line(); + } + Ok('*') => { + if cursor.next_is('/')? { + break; + } + } + _ => {} + } + } else { + return Err(Error::syntax("unterminated multiline comment")); + } + } + Ok(Token::new( + TokenKind::Comment, + Span::new(start_pos, cursor.pos()), + )) + } + _ => RegexLiteral::new().lex(cursor, start_pos), + } + } + } } } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index df4413e202c..512f0901e55 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -22,6 +22,8 @@ mod operator; mod spread; +mod regex; + mod identifier; // Temporary disabled while lexer in progress. @@ -32,7 +34,8 @@ pub use self::error::Error; use self::{ comment::Comment, cursor::Cursor, identifier::Identifier, number::NumberLiteral, - operator::Operator, spread::SpreadLiteral, string::StringLiteral, template::TemplateLiteral, + operator::Operator, regex::RegexLiteral, spread::SpreadLiteral, string::StringLiteral, + template::TemplateLiteral, }; use crate::syntax::ast::{Position, Punctuator, Span}; use std::io::Read; diff --git a/boa/src/syntax/lexer/regex.rs b/boa/src/syntax/lexer/regex.rs new file mode 100644 index 00000000000..b3ec9538a1a --- /dev/null +++ b/boa/src/syntax/lexer/regex.rs @@ -0,0 +1,109 @@ +use super::{Cursor, Error, Tokenizer}; +use crate::syntax::ast::Position; +use crate::syntax::lexer::Token; +use std::io::{self, ErrorKind, Read}; + +/// Regex literal lexing. +/// +/// Lexes Division, Assigndiv or Regex literal. +/// +/// Expects: Initial '/' to already be consumed by cursor. +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: https://www.ecma-international.org/ecma-262/#sec-literals-regular-expression-literals +/// [mdn]: +#[derive(Debug, Clone, Copy)] +pub(super) struct RegexLiteral; + +impl RegexLiteral { + /// Creates a new regex literal lexer. + pub(super) fn new() -> Self { + Self {} + } +} + +impl Tokenizer for RegexLiteral { + fn lex(&mut self, cursor: &mut Cursor, _start_pos: Position) -> Result + where + R: Read, + { + let mut body = String::new(); + + // Lex RegularExpressionBody. + loop { + match cursor.next() { + None => { + // Abrupt end. + return Err(Error::syntax("Abrupt end, regex not terminated")); + } + Some(Err(e)) => { + return Err(Error::from(e)); + } + Some(Ok(c)) => { + match c { + '/' => break, // RegularExpressionBody finished. + '\n' | '\r' | '\u{2028}' | '\u{2029}' => { + // Not allowed in Regex literal. + return Err(Error::syntax("Encountered new line during regex")); + } + '\\' => { + // Escape sequence + body.push('\\'); + match cursor.next() { + None => { + // Abrupt end of regex. + return Err(Error::syntax("Abrupt end, regex not terminated")); + } + Some(Err(_)) => { + return Err(Error::from(io::Error::new( + ErrorKind::Interrupted, + "Failed to peek next character", + ))) + } + Some(Ok(sc)) => { + match sc { + '\n' | '\r' | '\u{2028}' | '\u{2029}' => { + // Not allowed in Regex literal. + return Err(Error::syntax( + "Encountered new line during regex", + )); + } + ch => body.push(ch), + } + } + } + } + _ => body.push(c), + } + } + } + } + + unimplemented!( + "Regex handling, requires ability to peek more than a single character ahead" + ); + // if regex { + // // body was parsed, now look for flags + // let flags = self.take_char_while(char::is_alphabetic)?; + // self.move_columns(body.len() as u32 + 1 + flags.len() as u32); + // self.push_token(TokenKind::regular_expression_literal( + // body, flags.parse()?, + // ), start_pos); + // } else { + // // failed to parse regex, restore original buffer position and + // // parse either div or assigndiv + // self.buffer = original_buffer; + // self.position = original_pos; + // if self.next_is('=') { + // self.push_token(TokenKind::Punctuator( + // Punctuator::AssignDiv, + // ), start_pos); + // } else { + // self.push_token(TokenKind::Punctuator(Punctuator::Div), start_pos); + // } + // } + } +} diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs index e109f01c276..ad36661c4d3 100644 --- a/boa/src/syntax/lexer/template.rs +++ b/boa/src/syntax/lexer/template.rs @@ -9,7 +9,7 @@ macro_rules! template_match { }; } -/// Template literal parsing. +/// Template literal lexing. /// /// Expects: Initial ` to already be consumed by cursor. /// @@ -23,7 +23,7 @@ macro_rules! template_match { pub(super) struct TemplateLiteral; impl TemplateLiteral { - /// Creates a new string literal lexer. + /// Creates a new template literal lexer. pub(super) fn new() -> Self { Self {} } From fa5dc869b0b874a4be406e7e34ad59466b55ba82 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 13 Jun 2020 13:00:04 +0100 Subject: [PATCH 019/289] Start of numerical literal lexing --- boa/src/syntax/lexer/cursor.rs | 1 - boa/src/syntax/lexer/error.rs | 2 +- boa/src/syntax/lexer/identifier.rs | 4 +- boa/src/syntax/lexer/mod.rs | 7 +- boa/src/syntax/lexer/number.rs | 98 +++++++++++++++++-- boa/src/syntax/lexer/token.rs | 24 +++-- .../syntax/parser/expression/primary/mod.rs | 12 +-- 7 files changed, 119 insertions(+), 29 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 1a81bc5dda6..641538e24aa 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -15,7 +15,6 @@ impl Cursor { pub(super) fn pos(&self) -> Position { self.pos } - /// Advances the position to the next column. #[inline] pub(super) fn next_column(&mut self) { diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 3f9ac0be4e0..3c1b65e2a3b 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -27,7 +27,7 @@ impl Error { /// This error is used to represent the case where a piece of javascript /// cannot be lexed/parsed because it is in invalid when strict mdoe is /// enabled. - pub(super) fn _strict(err: M) -> Self + pub(super) fn strict(err: M) -> Self where M: Into>, { diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index dbd03640321..bc98097f6d3 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -1,6 +1,6 @@ use super::{Cursor, Error, Tokenizer}; use crate::syntax::ast::{Position, Span}; -use crate::syntax::lexer::token::NumericLiteral; +use crate::syntax::lexer::token::Numeric; use crate::syntax::lexer::{Token, TokenKind}; use std::io::Read; use std::str::FromStr; @@ -56,7 +56,7 @@ impl Tokenizer for Identifier { "true" => TokenKind::BooleanLiteral(true), "false" => TokenKind::BooleanLiteral(false), "null" => TokenKind::NullLiteral, - "NaN" => TokenKind::NumericLiteral(NumericLiteral::Rational(f64::NAN)), + "NaN" => TokenKind::NumericLiteral(Numeric::Rational(f64::NAN)), slice => { if let Ok(keyword) = FromStr::from_str(slice) { TokenKind::Keyword(keyword) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 512f0901e55..e10c44a95c9 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -132,6 +132,9 @@ where } }; + // TODO, setting strict mode on/off. + let strict_mode = false; + let token = match next_chr { '\r' | '\n' | '\u{2028}' | '\u{2029}' => Ok(Token::new( TokenKind::LineTerminator, @@ -139,7 +142,9 @@ where )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), template_match!() => TemplateLiteral::new().lex(&mut self.cursor, start), - _ if next_chr.is_digit(10) => NumberLiteral::new(next_chr).lex(&mut self.cursor, start), + _ if next_chr.is_digit(10) => { + NumberLiteral::new(next_chr, strict_mode).lex(&mut self.cursor, start) + } _ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => { Identifier::new(next_chr).lex(&mut self.cursor, start) } diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 593eed36ee8..ea2059d78da 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -1,6 +1,6 @@ -use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::Position; -use crate::syntax::lexer::Token; +use super::{Cursor, Error, TokenKind, Tokenizer}; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{token::Numeric, Token}; use std::io::Read; /// Number literal lexing. @@ -16,21 +16,105 @@ use std::io::Read; #[derive(Debug, Clone, Copy)] pub(super) struct NumberLiteral { init: char, + strict_mode: bool, } impl NumberLiteral { /// Creates a new string literal lexer. - pub(super) fn new(init: char) -> Self { - Self { init } + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } +} + +/// This is a helper structure +/// +/// This structure helps with identifying what numerical type it is and what base is it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NumericKind { + Rational, + Integer(u8), + BigInt(u8), +} + +impl NumericKind { + /// Get the base of the number kind. + fn base(self) -> u32 { + match self { + Self::Rational => 10, + Self::Integer(base) => base as u32, + Self::BigInt(base) => base as u32, + } + } + + /// Converts `self` to BigInt kind. + fn to_bigint(self) -> Self { + match self { + Self::Rational => unreachable!("can not convert rational number to BigInt"), + Self::Integer(base) => Self::BigInt(base), + Self::BigInt(base) => Self::BigInt(base), + } } } impl Tokenizer for NumberLiteral { - fn lex(&mut self, _cursor: &mut Cursor, _start_pos: Position) -> Result + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where R: Read, { - unimplemented!("Number literal lexing"); + let mut buf = self.init.to_string(); + + // Default assume the number is a base 10 integer. + let mut kind = NumericKind::Integer(10); + + if self.init == '0' { + match cursor.next() { + None => { + // DecimalLiteral lexing. + // Indicates that the number is just a single 0. + return Ok(Token::new( + TokenKind::NumericLiteral(Numeric::Integer(0)), + Span::new(start_pos, cursor.pos()), + )); + } + Some(Err(e)) => { + return Err(Error::from(e)); + } + Some(Ok('x')) | Some(Ok('X')) => { + // HexIntegerLiteral + kind = NumericKind::Integer(16); + } + Some(Ok('o')) | Some(Ok('O')) => { + // OctalIntegerLiteral + kind = NumericKind::Integer(8); + } + Some(Ok('b')) | Some(Ok('B')) => { + kind = NumericKind::Integer(2); + } + Some(Ok('n')) => { + // DecimalBigIntegerLiteral '0n' + return Ok(Token::new( + TokenKind::NumericLiteral(Numeric::BigInt(0.into())), + Span::new(start_pos, cursor.pos()), + )); + } + Some(Ok(ch)) => { + if ch.is_digit(8) { + // LegacyOctalIntegerLiteral + if self.strict_mode { + // LegacyOctalIntegerLiteral is forbidden with strict mode true. + return Err(Error::strict( + "Implicit octal literals are not allowed in strict mode.", + )); + } else { + buf.push(ch); + kind = NumericKind::Integer(8); + } + } + } + } + } + + unimplemented!(); } } diff --git a/boa/src/syntax/lexer/token.rs b/boa/src/syntax/lexer/token.rs index 9a8b1aabf0a..d7a38c5fdc2 100644 --- a/boa/src/syntax/lexer/token.rs +++ b/boa/src/syntax/lexer/token.rs @@ -60,7 +60,7 @@ impl Display for Token { /// Represents the type differenct types of numeric literals. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, PartialEq, Debug)] -pub enum NumericLiteral { +pub enum Numeric { /// A floating point number Rational(f64), @@ -71,19 +71,19 @@ pub enum NumericLiteral { BigInt(BigInt), } -impl From for NumericLiteral { +impl From for Numeric { fn from(n: f64) -> Self { Self::Rational(n) } } -impl From for NumericLiteral { +impl From for Numeric { fn from(n: i32) -> Self { Self::Integer(n) } } -impl From for NumericLiteral { +impl From for Numeric { fn from(n: BigInt) -> Self { Self::BigInt(n) } @@ -230,7 +230,7 @@ pub enum TokenKind { NullLiteral, /// A numeric literal. - NumericLiteral(NumericLiteral), + NumericLiteral(Numeric), /// A piece of punctuation /// @@ -270,6 +270,12 @@ impl From for TokenKind { } } +impl From for TokenKind { + fn from(num: Numeric) -> Self { + Self::NumericLiteral(num) + } +} + impl TokenKind { /// Creates a `BooleanLiteral` token kind. pub fn boolean_literal(lit: bool) -> Self { @@ -297,7 +303,7 @@ impl TokenKind { /// Creates a `NumericLiteral` token kind. pub fn numeric_literal(lit: L) -> Self where - L: Into, + L: Into, { Self::NumericLiteral(lit.into()) } @@ -350,9 +356,9 @@ impl Display for TokenKind { Self::Identifier(ref ident) => write!(f, "{}", ident), Self::Keyword(ref word) => write!(f, "{}", word), Self::NullLiteral => write!(f, "null"), - Self::NumericLiteral(NumericLiteral::Rational(num)) => write!(f, "{}", num), - Self::NumericLiteral(NumericLiteral::Integer(num)) => write!(f, "{}", num), - Self::NumericLiteral(NumericLiteral::BigInt(ref num)) => write!(f, "{}n", num), + Self::NumericLiteral(Numeric::Rational(num)) => write!(f, "{}", num), + Self::NumericLiteral(Numeric::Integer(num)) => write!(f, "{}", num), + Self::NumericLiteral(Numeric::BigInt(ref num)) => write!(f, "{}n", num), Self::Punctuator(ref punc) => write!(f, "{}", punc), Self::StringLiteral(ref lit) => write!(f, "{}", lit), Self::TemplateLiteral(ref lit) => write!(f, "{}", lit), diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index f17849034a7..bb89eb169dc 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -18,7 +18,7 @@ use self::{ object_initializer::ObjectLiteral, }; use super::Expression; -use crate::syntax::lexer::{token::NumericLiteral, TokenKind}; +use crate::syntax::lexer::{token::Numeric, TokenKind}; use crate::syntax::{ ast::{ node::{Call, Identifier, New, Node}, @@ -92,13 +92,9 @@ impl TokenParser for PrimaryExpression { TokenKind::NullLiteral => Ok(Const::Null.into()), TokenKind::Identifier(ident) => Ok(Identifier::from(ident.as_ref()).into()), // TODO: IdentifierReference TokenKind::StringLiteral(s) => Ok(Const::from(s.as_ref()).into()), - TokenKind::NumericLiteral(NumericLiteral::Integer(num)) => Ok(Const::from(*num).into()), - TokenKind::NumericLiteral(NumericLiteral::Rational(num)) => { - Ok(Const::from(*num).into()) - } - TokenKind::NumericLiteral(NumericLiteral::BigInt(num)) => { - Ok(Const::from(num.clone()).into()) - } + TokenKind::NumericLiteral(Numeric::Integer(num)) => Ok(Const::from(*num).into()), + TokenKind::NumericLiteral(Numeric::Rational(num)) => Ok(Const::from(*num).into()), + TokenKind::NumericLiteral(Numeric::BigInt(num)) => Ok(Const::from(num.clone()).into()), TokenKind::RegularExpressionLiteral(body, flags) => { Ok(Node::from(New::from(Call::new( Identifier::from("RegExp"), From 6dcc784260c1a70f3db584deeb871a0cefa7335f Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 14 Jun 2020 13:25:59 +0100 Subject: [PATCH 020/289] More of numeric lexing --- boa/src/syntax/lexer/number.rs | 346 +++++++++++++-------------------- 1 file changed, 138 insertions(+), 208 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index ea2059d78da..5a203c35931 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -1,7 +1,9 @@ use super::{Cursor, Error, TokenKind, Tokenizer}; +use crate::builtins::BigInt; use crate::syntax::ast::{Position, Span}; use crate::syntax::lexer::{token::Numeric, Token}; use std::io::Read; +use std::str::FromStr; /// Number literal lexing. /// @@ -66,8 +68,10 @@ impl Tokenizer for NumberLiteral { // Default assume the number is a base 10 integer. let mut kind = NumericKind::Integer(10); + let c = cursor.next(); + if self.init == '0' { - match cursor.next() { + match c { None => { // DecimalLiteral lexing. // Indicates that the number is just a single 0. @@ -88,6 +92,7 @@ impl Tokenizer for NumberLiteral { kind = NumericKind::Integer(8); } Some(Ok('b')) | Some(Ok('B')) => { + // BinaryIntegerLiteral kind = NumericKind::Integer(2); } Some(Ok('n')) => { @@ -109,252 +114,179 @@ impl Tokenizer for NumberLiteral { buf.push(ch); kind = NumericKind::Integer(8); } + } else if ch.is_digit(36) { + // Indicates a numerical digit comes after then 0 but it isn't an octal digit + // so therefore this must be a number with an unneeded leading 0. This is + // forbidden in strict mode. + if self.strict_mode { + // LegacyOctalIntegerLiteral is forbidden with strict mode true. + return Err(Error::strict( + "Leading 0's are not allowed in strict mode.", + )); + } else { + buf.push(ch); + } + } else { + // Indicates that the symbol is a non-number, this is valid if it is a dot or similar. + buf.push(ch); } } } } - unimplemented!(); - } -} - -/* -impl Tokenizer for NumberLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - /// This is a helper structure - /// - /// This structure helps with identifying what numerical type it is and what base is it. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - enum NumericKind { - Rational, - Integer(u8), - BigInt(u8), - } - - impl NumericKind { - /// Get the base of the number kind. - fn base(self) -> u32 { - match self { - Self::Rational => 10, - Self::Integer(base) => base as u32, - Self::BigInt(base) => base as u32, - } - } - - /// Converts `self` to BigInt kind. - fn to_bigint(self) -> Self { - match self { - Self::Rational => unreachable!("can not convert rational number to BigInt"), - Self::Integer(base) => Self::BigInt(base), - Self::BigInt(base) => Self::BigInt(base), + while let Some(ch) = cursor.peek() { + match ch { + Err(_e) => { + // TODO, handle. } - } - } - - // TODO: Setup strict mode. - let strict_mode = false; - - let mut buf = self.init.to_string(); - - let mut kind = NumericKind::Integer(10); - - if self.init == '0' { - match cursor.peek() { - None => { - cursor.next_column(); - return Ok(Token::new( - TokenKind::NumericLiteral(NumericLiteral::Integer(0)), - Span::new(start_pos, cursor.pos()), - )); + Ok(c) if c.is_digit(kind.base()) => { + let s = cursor.next().unwrap().unwrap(); + buf.push(s); } - Some(r) => { - match r.map_err(|e| Error::IO(e))? { - 'x' | 'X' => { - cursor.next(); - cursor.next_column(); - kind = NumericKind::Integer(16); - } - 'o' | 'O' => { - cursor.next(); - cursor.next_column(); - kind = NumericKind::Integer(8); - } - 'b' | 'B' => { - cursor.next(); - cursor.next_column(); - kind = NumericKind::Integer(2); - } - ch if ch.is_ascii_digit() => { - let mut is_implicit_octal = true; - while let Some(ch) = cursor.peek(){ - let c = ch.map_err(|e| Error::IO(e))?; - if !c.is_ascii_digit() { - break; - } else if !c.is_digit(8) { - is_implicit_octal = false; - } - cursor.next(); - buf.push(c); - } - if !strict_mode { - if is_implicit_octal { - kind = NumericKind::Integer(8); - } - } else { - return Err(if is_implicit_octal { - Error::strict( - "Implicit octal literals are not allowed in strict mode.", - ) - } else { - Error::strict( - "Decimals with leading zeros are not allowed in strict mode.", - ) - }); - } - } - _ => {} - } + _ => { + // A non-number symbol detected, this might be a dot or similar. + break; } } } - while let Some(ch) = cursor.peek() { - let c = ch.map_err(|e| Error::IO(e))?; - if !c.is_digit(kind.base()) { - break; - } - cursor.next(); - buf.push(c); - } - if cursor.next_is('n')? { + // DecimalBigIntegerLiteral kind = kind.to_bigint(); } if let NumericKind::Integer(10) = kind { - 'digitloop: while let Some(cx) = cursor.peek() { - match cx.map_err(|e| Error::IO(e))? { - '.' => loop { + 'digitloop: while let Some(ch) = cursor.peek() { + match ch { + Err(_e) => { + // TODO + } + Ok('.') => loop { kind = NumericKind::Rational; - cursor.next(); - buf.push('.'); - - let c = match cursor.peek() { - Some(ch) => ch.map_err(|e| Error::IO(e))?, - None => break, - }; - - match c { - 'e' | 'E' => { - cursor.next(); // Consume 'e' or 'E' + match cursor.next() { + None => { + // Finished + break; + } - match cursor.peek() { - None => { - cursor.next(); - } - Some(x) => { - let val = x.map_err(|e| Error::IO(e))?; - match val.to_digit(10) { - Some(0..=9) => { - cursor.next(); // Consume digit. - buf.push(val); - } - _ => { - break 'digitloop; - } - } - } - _ => { - break 'digitloop; - } - } + Some(Err(e)) => { + return Err(Error::from(e)); } - _ => { - if !c.is_digit(10) { - break 'digitloop; - } + + Some(Ok(c)) => { + buf.push(c); } } - }, - 'e' | 'E' => { - cursor.next(); // Consume 'e' or 'E' - kind = NumericKind::Rational; + match cursor.peek() { None => { - cursor.next(); + break; } - Some(x) => { - let val = x.map_err(|e| Error::IO(e))?; - match val.to_digit(10) { - Some(0..=9) => { - cursor.next(); // Consume digit. - buf.push(val); - } - _ => { - break; - } - } + Some(Err(_e)) => { + // TODO } - _ => { - break; + Some(Ok('e')) | Some(Ok('E')) => { + // TODO scientific notation. + + unimplemented!(); + + // match self + // .preview_multiple_next(2) + // .unwrap_or_default() + // .to_digit(10) + // { + // Some(0..=9) | None => { + // buf.push(self.next()); + // } + // _ => { + // break 'digitloop; + // } + // } + } + Some(Ok(cx)) if !cx.is_digit(10) => { + break 'digitloop; } + _ => {} } + }, + Ok('e') | Ok('E') => { + // TODO scientific notation. + unimplemented!(); + + // kind = NumericKind::Rational; + // match self + // .preview_multiple_next(2) + // .unwrap_or_default() + // .to_digit(10) + // { + // Some(0..=9) | None => { + // buf.push(self.next()); + // } + // _ => { + // break; + // } + // } + // buf.push(self.next()); } - '+' | '-' => { + Ok('+') | Ok('-') => { break; } - x if x.is_digit(10) => { - buf.push(x); + Ok(cx) if cx.is_digit(10) => { + // cursor.next(); + match cursor.next() { + None => { + // Finished + break; + } + + Some(Err(e)) => { + return Err(Error::from(e)); + } + + Some(Ok(c)) => { + buf.push(c); + } + } + // buf.push(*cx); } - _ => break, + Ok(_) => break, } } } - // Check the NumericLiteral is not followed by an `IdentifierStart` or `DecimalDigit` character. - match cursor.peek() { - Some(r) => { - let c = r.map_err(|e| Error::IO(e))?; - if c.is_ascii_alphabetic() || c == '$' || c == '_' || c.is_ascii_digit() { - return Err(Error::syntax("NumericLiteral token must not be followed by IdentifierStart nor DecimalDigit characters")); - } - }, - _ => {} - } + // TODO + //self.check_after_numeric_literal()?; let num = match kind { - NumericKind::BigInt(base) => { - NumericLiteral::BigInt( - BigInt::from_string_radix(&buf, base as u32).expect("Could not conver to BigInt") - ) - } - NumericKind::Rational /* base: 10 */ => { - NumericLiteral::Rational( - f64::from_str(&buf) - .map_err(|_| Error::syntax("Could not convert value to f64"))?, - ) - } - NumericKind::Integer(base) => { - if let Ok(num) = i32::from_str_radix(&buf, base as u32) { - NumericLiteral::Integer( - num + NumericKind::BigInt(base) => { + Numeric::BigInt( + BigInt::from_string_radix(&buf, base as u32).expect("Could not convert to BigInt") + ) + } + NumericKind::Rational /* base: 10 */ => { + Numeric::Rational( + f64::from_str(&buf) + .map_err(|_| Error::syntax("Could not convert value to f64"))?, ) - } else { - let b = f64::from(base); - let mut result = 0.0_f64; - for c in buf.chars() { - let digit = f64::from(c.to_digit(base as u32).unwrap()); - result = result * b + digit; + } + NumericKind::Integer(base) => { + if let Ok(num) = i32::from_str_radix(&buf, base as u32) { + Numeric::Integer( + num + ) + } else { + let b = f64::from(base); + let mut result = 0.0_f64; + for c in buf.chars() { + let digit = f64::from(c.to_digit(base as u32).unwrap()); + result = result * b + digit; + } + + Numeric::Rational(result) } - NumericLiteral::Rational(result) } - - } - }; + }; Ok(Token::new( TokenKind::NumericLiteral(num), @@ -362,5 +294,3 @@ impl Tokenizer for NumberLiteral { )) } } - -*/ From ad2c147fd825a81ad889bae582276ef3d254b998 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 14 Jun 2020 15:27:22 +0100 Subject: [PATCH 021/289] Re-enabled the lexer tests --- boa/src/syntax/lexer/mod.rs | 4 +- boa/src/syntax/lexer/tests.rs | 716 +++++++++++++++------------------- 2 files changed, 324 insertions(+), 396 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index e10c44a95c9..6008e9fe25d 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -27,8 +27,8 @@ mod regex; mod identifier; // Temporary disabled while lexer in progress. -// #[cfg(test)] -// mod tests; +#[cfg(test)] +mod tests; pub use self::error::Error; diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 695fa69c703..aaca2d7ac77 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -1,6 +1,7 @@ //! Tests for the lexer. #![allow(clippy::indexing_slicing)] +use super::token::Numeric; use super::*; use crate::syntax::ast::Keyword; @@ -8,73 +9,73 @@ fn span(start: (u32, u32), end: (u32, u32)) -> Span { Span::new(Position::new(start.0, start.1), Position::new(end.0, end.1)) } -#[test] -fn check_single_line_comment() { - let s1 = "var \n//This is a comment\ntrue"; - let mut lexer = Lexer::new(s1.as_bytes()); +fn expect_tokens(lexer: &mut Lexer, expected: &[TokenKind]) +where + R: Read, +{ + for expect in expected.iter() { + assert_eq!(&lexer.next().unwrap().unwrap().kind(), &expect); + } - assert_eq!( - lexer.next().unwrap().unwrap().kind, - TokenKind::Keyword(Keyword::Var) - ); - assert_eq!( - lexer.next().unwrap().unwrap().kind, - TokenKind::LineTerminator - ); - assert_eq!( - lexer.next().unwrap().unwrap().kind, - TokenKind::BooleanLiteral(true) - ); assert!(lexer.next().is_none()); } #[test] fn check_single_line_comment() { let s1 = "var \n//This is a comment\ntrue"; - let mut lexer = Lexer::new(s1); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::Keyword(Keyword::Var)); - assert_eq!(lexer.tokens[1].kind, TokenKind::LineTerminator); - assert_eq!(lexer.tokens[2].kind, TokenKind::BooleanLiteral(true)); + let mut lexer = Lexer::new(s1.as_bytes()); + + let expected = [ + TokenKind::Keyword(Keyword::Var), + TokenKind::LineTerminator, + TokenKind::BooleanLiteral(true), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn check_multi_line_comment() { let s = "var /* await \n break \n*/ x"; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::Keyword(Keyword::Var)); - assert_eq!(lexer.tokens[1].kind, TokenKind::identifier("x")); + let mut lexer = Lexer::new(s.as_bytes()); + + let expected = [TokenKind::Keyword(Keyword::Var), TokenKind::identifier("x")]; + + expect_tokens(&mut lexer, &expected); } #[test] fn check_string() { let s = "'aaa' \"bbb\""; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::string_literal("aaa")); + let mut lexer = Lexer::new(s.as_bytes()); - assert_eq!(lexer.tokens[1].kind, TokenKind::string_literal("bbb")); + let expected = [ + TokenKind::string_literal("aaa"), + TokenKind::string_literal("bbb"), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn check_template_literal_simple() { let s = "`I'm a template literal`"; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new(s.as_bytes()); + assert_eq!( - lexer.tokens[0].kind, - TokenKind::template_literal("I'm a template literal") + lexer.next().unwrap().unwrap().kind(), + &TokenKind::template_literal("I'm a template literal") ); } #[test] fn check_template_literal_unterminated() { let s = "`I'm a template"; - let mut lexer = Lexer::new(s); - match lexer.lex() { - Ok(_) => panic!("Lexer did not detect end of stream"), - Err(e) => { + let mut lexer = Lexer::new(s.as_bytes()); + + match lexer.next() { + None | Some(Ok(_)) => panic!("Lexer did not detect end of stream"), + Some(Err(e)) => { assert_eq!(e.to_string(), "Unterminated template literal"); } } @@ -86,195 +87,61 @@ fn check_punctuators() { let s = "{ ( ) [ ] . ... ; , < > <= >= == != === !== \ + - * % -- << >> >>> & | ^ ! ~ && || ? : \ = += -= *= &= **= ++ ** <<= >>= >>>= &= |= ^= =>"; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); - assert_eq!( - lexer.tokens[0].kind, - TokenKind::Punctuator(Punctuator::OpenBlock) - ); - assert_eq!( - lexer.tokens[1].kind, - TokenKind::Punctuator(Punctuator::OpenParen) - ); - assert_eq!( - lexer.tokens[2].kind, - TokenKind::Punctuator(Punctuator::CloseParen) - ); - assert_eq!( - lexer.tokens[3].kind, - TokenKind::Punctuator(Punctuator::OpenBracket) - ); - assert_eq!( - lexer.tokens[4].kind, - TokenKind::Punctuator(Punctuator::CloseBracket) - ); - assert_eq!(lexer.tokens[5].kind, TokenKind::Punctuator(Punctuator::Dot)); - assert_eq!( - lexer.tokens[6].kind, - TokenKind::Punctuator(Punctuator::Spread) - ); - assert_eq!( - lexer.tokens[7].kind, - TokenKind::Punctuator(Punctuator::Semicolon) - ); - assert_eq!( - lexer.tokens[8].kind, - TokenKind::Punctuator(Punctuator::Comma) - ); - assert_eq!( - lexer.tokens[9].kind, - TokenKind::Punctuator(Punctuator::LessThan) - ); - assert_eq!( - lexer.tokens[10].kind, - TokenKind::Punctuator(Punctuator::GreaterThan) - ); - assert_eq!( - lexer.tokens[11].kind, - TokenKind::Punctuator(Punctuator::LessThanOrEq) - ); - assert_eq!( - lexer.tokens[12].kind, - TokenKind::Punctuator(Punctuator::GreaterThanOrEq) - ); - assert_eq!(lexer.tokens[13].kind, TokenKind::Punctuator(Punctuator::Eq)); - assert_eq!( - lexer.tokens[14].kind, - TokenKind::Punctuator(Punctuator::NotEq) - ); - assert_eq!( - lexer.tokens[15].kind, - TokenKind::Punctuator(Punctuator::StrictEq) - ); - assert_eq!( - lexer.tokens[16].kind, - TokenKind::Punctuator(Punctuator::StrictNotEq) - ); - assert_eq!( - lexer.tokens[17].kind, - TokenKind::Punctuator(Punctuator::Add) - ); - assert_eq!( - lexer.tokens[18].kind, - TokenKind::Punctuator(Punctuator::Sub) - ); - assert_eq!( - lexer.tokens[19].kind, - TokenKind::Punctuator(Punctuator::Mul) - ); - assert_eq!( - lexer.tokens[20].kind, - TokenKind::Punctuator(Punctuator::Mod) - ); - assert_eq!( - lexer.tokens[21].kind, - TokenKind::Punctuator(Punctuator::Dec) - ); - assert_eq!( - lexer.tokens[22].kind, - TokenKind::Punctuator(Punctuator::LeftSh) - ); - assert_eq!( - lexer.tokens[23].kind, - TokenKind::Punctuator(Punctuator::RightSh) - ); - assert_eq!( - lexer.tokens[24].kind, - TokenKind::Punctuator(Punctuator::URightSh) - ); - assert_eq!( - lexer.tokens[25].kind, - TokenKind::Punctuator(Punctuator::And) - ); - assert_eq!(lexer.tokens[26].kind, TokenKind::Punctuator(Punctuator::Or)); - assert_eq!( - lexer.tokens[27].kind, - TokenKind::Punctuator(Punctuator::Xor) - ); - assert_eq!( - lexer.tokens[28].kind, - TokenKind::Punctuator(Punctuator::Not) - ); - assert_eq!( - lexer.tokens[29].kind, - TokenKind::Punctuator(Punctuator::Neg) - ); - assert_eq!( - lexer.tokens[30].kind, - TokenKind::Punctuator(Punctuator::BoolAnd) - ); - assert_eq!( - lexer.tokens[31].kind, - TokenKind::Punctuator(Punctuator::BoolOr) - ); - assert_eq!( - lexer.tokens[32].kind, - TokenKind::Punctuator(Punctuator::Question) - ); - assert_eq!( - lexer.tokens[33].kind, - TokenKind::Punctuator(Punctuator::Colon) - ); - assert_eq!( - lexer.tokens[34].kind, - TokenKind::Punctuator(Punctuator::Assign) - ); - assert_eq!( - lexer.tokens[35].kind, - TokenKind::Punctuator(Punctuator::AssignAdd) - ); - assert_eq!( - lexer.tokens[36].kind, - TokenKind::Punctuator(Punctuator::AssignSub) - ); - assert_eq!( - lexer.tokens[37].kind, - TokenKind::Punctuator(Punctuator::AssignMul) - ); - assert_eq!( - lexer.tokens[38].kind, - TokenKind::Punctuator(Punctuator::AssignAnd) - ); - assert_eq!( - lexer.tokens[39].kind, - TokenKind::Punctuator(Punctuator::AssignPow) - ); - assert_eq!( - lexer.tokens[40].kind, - TokenKind::Punctuator(Punctuator::Inc) - ); - assert_eq!( - lexer.tokens[41].kind, - TokenKind::Punctuator(Punctuator::Exp) - ); - assert_eq!( - lexer.tokens[42].kind, - TokenKind::Punctuator(Punctuator::AssignLeftSh) - ); - assert_eq!( - lexer.tokens[43].kind, - TokenKind::Punctuator(Punctuator::AssignRightSh) - ); - assert_eq!( - lexer.tokens[44].kind, - TokenKind::Punctuator(Punctuator::AssignURightSh) - ); - assert_eq!( - lexer.tokens[45].kind, - TokenKind::Punctuator(Punctuator::AssignAnd) - ); - assert_eq!( - lexer.tokens[46].kind, - TokenKind::Punctuator(Punctuator::AssignOr) - ); - assert_eq!( - lexer.tokens[47].kind, - TokenKind::Punctuator(Punctuator::AssignXor) - ); - assert_eq!( - lexer.tokens[48].kind, - TokenKind::Punctuator(Punctuator::Arrow) - ); + let mut lexer = Lexer::new(s.as_bytes()); + + let expected = [ + TokenKind::Punctuator(Punctuator::OpenBlock), + TokenKind::Punctuator(Punctuator::OpenParen), + TokenKind::Punctuator(Punctuator::CloseParen), + TokenKind::Punctuator(Punctuator::OpenBracket), + TokenKind::Punctuator(Punctuator::CloseBracket), + TokenKind::Punctuator(Punctuator::Dot), + TokenKind::Punctuator(Punctuator::Spread), + TokenKind::Punctuator(Punctuator::Semicolon), + TokenKind::Punctuator(Punctuator::Comma), + TokenKind::Punctuator(Punctuator::LessThan), + TokenKind::Punctuator(Punctuator::GreaterThan), + TokenKind::Punctuator(Punctuator::LessThanOrEq), + TokenKind::Punctuator(Punctuator::GreaterThanOrEq), + TokenKind::Punctuator(Punctuator::Eq), + TokenKind::Punctuator(Punctuator::NotEq), + TokenKind::Punctuator(Punctuator::StrictEq), + TokenKind::Punctuator(Punctuator::StrictNotEq), + TokenKind::Punctuator(Punctuator::Add), + TokenKind::Punctuator(Punctuator::Sub), + TokenKind::Punctuator(Punctuator::Mul), + TokenKind::Punctuator(Punctuator::Mod), + TokenKind::Punctuator(Punctuator::Dec), + TokenKind::Punctuator(Punctuator::LeftSh), + TokenKind::Punctuator(Punctuator::RightSh), + TokenKind::Punctuator(Punctuator::URightSh), + TokenKind::Punctuator(Punctuator::And), + TokenKind::Punctuator(Punctuator::Or), + TokenKind::Punctuator(Punctuator::Xor), + TokenKind::Punctuator(Punctuator::Not), + TokenKind::Punctuator(Punctuator::Neg), + TokenKind::Punctuator(Punctuator::BoolAnd), + TokenKind::Punctuator(Punctuator::BoolOr), + TokenKind::Punctuator(Punctuator::Question), + TokenKind::Punctuator(Punctuator::Colon), + TokenKind::Punctuator(Punctuator::Assign), + TokenKind::Punctuator(Punctuator::AssignAdd), + TokenKind::Punctuator(Punctuator::AssignSub), + TokenKind::Punctuator(Punctuator::AssignMul), + TokenKind::Punctuator(Punctuator::AssignAnd), + TokenKind::Punctuator(Punctuator::AssignPow), + TokenKind::Punctuator(Punctuator::Inc), + TokenKind::Punctuator(Punctuator::Exp), + TokenKind::Punctuator(Punctuator::AssignLeftSh), + TokenKind::Punctuator(Punctuator::AssignRightSh), + TokenKind::Punctuator(Punctuator::AssignURightSh), + TokenKind::Punctuator(Punctuator::AssignAnd), + TokenKind::Punctuator(Punctuator::AssignOr), + TokenKind::Punctuator(Punctuator::AssignXor), + TokenKind::Punctuator(Punctuator::Arrow), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] @@ -284,111 +151,126 @@ fn check_keywords() { do else export extends finally for function if import in instanceof \ new return super switch this throw try typeof var void while with yield"; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::Keyword(Keyword::Await)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Keyword(Keyword::Break)); - assert_eq!(lexer.tokens[2].kind, TokenKind::Keyword(Keyword::Case)); - assert_eq!(lexer.tokens[3].kind, TokenKind::Keyword(Keyword::Catch)); - assert_eq!(lexer.tokens[4].kind, TokenKind::Keyword(Keyword::Class)); - assert_eq!(lexer.tokens[5].kind, TokenKind::Keyword(Keyword::Const)); - assert_eq!(lexer.tokens[6].kind, TokenKind::Keyword(Keyword::Continue)); - assert_eq!(lexer.tokens[7].kind, TokenKind::Keyword(Keyword::Debugger)); - assert_eq!(lexer.tokens[8].kind, TokenKind::Keyword(Keyword::Default)); - assert_eq!(lexer.tokens[9].kind, TokenKind::Keyword(Keyword::Delete)); - assert_eq!(lexer.tokens[10].kind, TokenKind::Keyword(Keyword::Do)); - assert_eq!(lexer.tokens[11].kind, TokenKind::Keyword(Keyword::Else)); - assert_eq!(lexer.tokens[12].kind, TokenKind::Keyword(Keyword::Export)); - assert_eq!(lexer.tokens[13].kind, TokenKind::Keyword(Keyword::Extends)); - assert_eq!(lexer.tokens[14].kind, TokenKind::Keyword(Keyword::Finally)); - assert_eq!(lexer.tokens[15].kind, TokenKind::Keyword(Keyword::For)); - assert_eq!(lexer.tokens[16].kind, TokenKind::Keyword(Keyword::Function)); - assert_eq!(lexer.tokens[17].kind, TokenKind::Keyword(Keyword::If)); - assert_eq!(lexer.tokens[18].kind, TokenKind::Keyword(Keyword::Import)); - assert_eq!(lexer.tokens[19].kind, TokenKind::Keyword(Keyword::In)); - assert_eq!( - lexer.tokens[20].kind, - TokenKind::Keyword(Keyword::InstanceOf) - ); - assert_eq!(lexer.tokens[21].kind, TokenKind::Keyword(Keyword::New)); - assert_eq!(lexer.tokens[22].kind, TokenKind::Keyword(Keyword::Return)); - assert_eq!(lexer.tokens[23].kind, TokenKind::Keyword(Keyword::Super)); - assert_eq!(lexer.tokens[24].kind, TokenKind::Keyword(Keyword::Switch)); - assert_eq!(lexer.tokens[25].kind, TokenKind::Keyword(Keyword::This)); - assert_eq!(lexer.tokens[26].kind, TokenKind::Keyword(Keyword::Throw)); - assert_eq!(lexer.tokens[27].kind, TokenKind::Keyword(Keyword::Try)); - assert_eq!(lexer.tokens[28].kind, TokenKind::Keyword(Keyword::TypeOf)); - assert_eq!(lexer.tokens[29].kind, TokenKind::Keyword(Keyword::Var)); - assert_eq!(lexer.tokens[30].kind, TokenKind::Keyword(Keyword::Void)); - assert_eq!(lexer.tokens[31].kind, TokenKind::Keyword(Keyword::While)); - assert_eq!(lexer.tokens[32].kind, TokenKind::Keyword(Keyword::With)); - assert_eq!(lexer.tokens[33].kind, TokenKind::Keyword(Keyword::Yield)); + let mut lexer = Lexer::new(s.as_bytes()); + + let expected = [ + TokenKind::Keyword(Keyword::Await), + TokenKind::Keyword(Keyword::Break), + TokenKind::Keyword(Keyword::Case), + TokenKind::Keyword(Keyword::Catch), + TokenKind::Keyword(Keyword::Class), + TokenKind::Keyword(Keyword::Const), + TokenKind::Keyword(Keyword::Continue), + TokenKind::Keyword(Keyword::Debugger), + TokenKind::Keyword(Keyword::Default), + TokenKind::Keyword(Keyword::Delete), + TokenKind::Keyword(Keyword::Do), + TokenKind::Keyword(Keyword::Else), + TokenKind::Keyword(Keyword::Export), + TokenKind::Keyword(Keyword::Extends), + TokenKind::Keyword(Keyword::Finally), + TokenKind::Keyword(Keyword::For), + TokenKind::Keyword(Keyword::Function), + TokenKind::Keyword(Keyword::If), + TokenKind::Keyword(Keyword::Import), + TokenKind::Keyword(Keyword::In), + TokenKind::Keyword(Keyword::InstanceOf), + TokenKind::Keyword(Keyword::New), + TokenKind::Keyword(Keyword::Return), + TokenKind::Keyword(Keyword::Super), + TokenKind::Keyword(Keyword::Switch), + TokenKind::Keyword(Keyword::This), + TokenKind::Keyword(Keyword::Throw), + TokenKind::Keyword(Keyword::Try), + TokenKind::Keyword(Keyword::TypeOf), + TokenKind::Keyword(Keyword::Var), + TokenKind::Keyword(Keyword::Void), + TokenKind::Keyword(Keyword::While), + TokenKind::Keyword(Keyword::With), + TokenKind::Keyword(Keyword::Yield), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn check_variable_definition_tokens() { let s = "let a = 'hello';"; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::Keyword(Keyword::Let)); - assert_eq!(lexer.tokens[1].kind, TokenKind::identifier("a")); - assert_eq!( - lexer.tokens[2].kind, - TokenKind::Punctuator(Punctuator::Assign) - ); - assert_eq!(lexer.tokens[3].kind, TokenKind::string_literal("hello")); + let mut lexer = Lexer::new(s.as_bytes()); + + let expected = [ + TokenKind::Keyword(Keyword::Let), + TokenKind::identifier("a"), + TokenKind::Punctuator(Punctuator::Assign), + TokenKind::string_literal("hello"), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn check_positions() { let s = r#"console.log("hello world\u{2764}"); // Test"#; // --------123456789 - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new(s.as_bytes()); + // The first column is 1 (not zero indexed) - assert_eq!(lexer.tokens[0].span(), span((1, 1), (1, 7))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 1), (1, 7))); // Dot Token starts on column 8 - assert_eq!(lexer.tokens[1].span(), span((1, 8), (1, 8))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 8), (1, 8))); // Log Token starts on column 9 - assert_eq!(lexer.tokens[2].span(), span((1, 9), (1, 11))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 9), (1, 11))); // Open parenthesis token starts on column 12 - assert_eq!(lexer.tokens[3].span(), span((1, 12), (1, 12))); + assert_eq!( + lexer.next().unwrap().unwrap().span(), + span((1, 12), (1, 12)) + ); // String token starts on column 13 - assert_eq!(lexer.tokens[4].span(), span((1, 13), (1, 33))); + assert_eq!( + lexer.next().unwrap().unwrap().span(), + span((1, 13), (1, 33)) + ); // Close parenthesis token starts on column 34 - assert_eq!(lexer.tokens[5].span(), span((1, 34), (1, 34))); + assert_eq!( + lexer.next().unwrap().unwrap().span(), + span((1, 34), (1, 34)) + ); // Semi Colon token starts on column 35 - assert_eq!(lexer.tokens[6].span(), span((1, 35), (1, 35))); + assert_eq!( + lexer.next().unwrap().unwrap().span(), + span((1, 35), (1, 35)) + ); } #[test] #[ignore] fn two_divisions_in_expression() { let s = " return a !== 0 || 1 / a === 1 / b;"; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new(s.as_bytes()); // dbg!(&lexer.tokens); - assert_eq!(lexer.tokens[11].span(), span((1, 37), (1, 37))); + assert_eq!( + lexer.skip(11).next().unwrap().unwrap().span(), + span((1, 37), (1, 37)) + ); } #[test] fn check_line_numbers() { let s = "x\ny\n"; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new(s.as_bytes()); - assert_eq!(lexer.tokens[0].span(), span((1, 1), (1, 1))); - assert_eq!(lexer.tokens[1].span(), span((1, 2), (2, 1))); - assert_eq!(lexer.tokens[2].span(), span((2, 1), (2, 1))); - assert_eq!(lexer.tokens[3].span(), span((2, 2), (3, 1))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 1), (1, 1))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 2), (2, 1))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((2, 1), (2, 1))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((2, 2), (3, 1))); } // Increment/Decrement @@ -396,171 +278,217 @@ fn check_line_numbers() { fn check_decrement_advances_lexer_2_places() { // Here we want an example of decrementing an integer let s = "let a = b--;"; - let mut lexer = Lexer::new(s); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[4].kind, TokenKind::Punctuator(Punctuator::Dec)); + let lexer = Lexer::new(s.as_bytes()); + + let mut iter = lexer.skip(4); + + assert_eq!( + iter.next().unwrap().unwrap().kind(), + &TokenKind::Punctuator(Punctuator::Dec) + ); // Decrementing means adding 2 characters '--', the lexer should consume it as a single token // and move the curser forward by 2, meaning the next token should be a semicolon + assert_eq!( - lexer.tokens[5].kind, - TokenKind::Punctuator(Punctuator::Semicolon) + iter.next().unwrap().unwrap().kind(), + &TokenKind::Punctuator(Punctuator::Semicolon) ); } #[test] fn check_nan() { - let mut lexer = Lexer::new("let a = NaN;"); - lexer.lex().expect("failed to lex"); - - match lexer.tokens[3].kind { - TokenKind::NumericLiteral(NumericLiteral::Rational(a)) => { - assert!(a.is_nan()); - } - ref other => panic!("Incorrect token kind found for NaN: {}", other), + let mut lexer = Lexer::new("let a = NaN;".as_bytes()); + match lexer.skip(3).next() { + None | Some(Err(_)) => panic!("No token found when expecting NaN"), + Some(Ok(token)) => match token.kind() { + TokenKind::NumericLiteral(Numeric::Rational(a)) => { + assert!(a.is_nan()); + } + ref other => panic!("Incorrect token kind found for NaN: {}", other), + }, } } #[test] fn numbers() { let mut lexer = Lexer::new( - "1 2 0x34 056 7.89 42. 5e3 5e+3 5e-3 0b10 0O123 0999 1.0e1 1.0e-1 1.0E1 1E1 0.0 0.12", - ); - - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); - assert_eq!(lexer.tokens[1].kind, TokenKind::numeric_literal(2)); - assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(52)); - assert_eq!(lexer.tokens[3].kind, TokenKind::numeric_literal(46)); - assert_eq!(lexer.tokens[4].kind, TokenKind::numeric_literal(7.89)); - assert_eq!(lexer.tokens[5].kind, TokenKind::numeric_literal(42.0)); - assert_eq!(lexer.tokens[6].kind, TokenKind::numeric_literal(5000.0)); - assert_eq!(lexer.tokens[7].kind, TokenKind::numeric_literal(5000.0)); - assert_eq!(lexer.tokens[8].kind, TokenKind::numeric_literal(0.005)); - assert_eq!(lexer.tokens[9].kind, TokenKind::numeric_literal(2)); - assert_eq!(lexer.tokens[10].kind, TokenKind::numeric_literal(83)); - assert_eq!(lexer.tokens[11].kind, TokenKind::numeric_literal(999)); - assert_eq!(lexer.tokens[12].kind, TokenKind::numeric_literal(10.0)); - assert_eq!(lexer.tokens[13].kind, TokenKind::numeric_literal(0.1)); - assert_eq!(lexer.tokens[14].kind, TokenKind::numeric_literal(10.0)); - assert_eq!(lexer.tokens[15].kind, TokenKind::numeric_literal(10.0)); - assert_eq!(lexer.tokens[16].kind, TokenKind::numeric_literal(0.0)); - assert_eq!(lexer.tokens[17].kind, TokenKind::numeric_literal(0.12)); + "1 2 0x34 056 7.89 42. 5e3 5e+3 5e-3 0b10 0O123 0999 1.0e1 1.0e-1 1.0E1 1E1 0.0 0.12" + .as_bytes(), + ); + + let expected = [ + TokenKind::numeric_literal(1), + TokenKind::numeric_literal(2), + TokenKind::numeric_literal(52), + TokenKind::numeric_literal(46), + TokenKind::numeric_literal(7.89), + TokenKind::numeric_literal(42.0), + TokenKind::numeric_literal(5000.0), + TokenKind::numeric_literal(5000.0), + TokenKind::numeric_literal(0.005), + TokenKind::numeric_literal(2), + TokenKind::numeric_literal(83), + TokenKind::numeric_literal(999), + TokenKind::numeric_literal(10.0), + TokenKind::numeric_literal(0.1), + TokenKind::numeric_literal(10.0), + TokenKind::numeric_literal(10.0), + TokenKind::numeric_literal(0.0), + TokenKind::numeric_literal(0.12), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn implicit_octal_edge_case() { - let mut lexer = Lexer::new("044.5 094.5"); + let mut lexer = Lexer::new("044.5 094.5".as_bytes()); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(36)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot)); - assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(5)); + let expected = [ + TokenKind::numeric_literal(36), + TokenKind::Punctuator(Punctuator::Dot), + TokenKind::numeric_literal(5), + TokenKind::numeric_literal(94.5), + ]; - assert_eq!(lexer.tokens[3].kind, TokenKind::numeric_literal(94.5)); + expect_tokens(&mut lexer, &expected); } #[test] fn hexadecimal_edge_case() { - let mut lexer = Lexer::new("0xffff.ff 0xffffff"); + let mut lexer = Lexer::new("0xffff.ff 0xffffff".as_bytes()); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(0xffff)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot)); - assert_eq!(lexer.tokens[2].kind, TokenKind::identifier("ff")); + let expected = [ + TokenKind::numeric_literal(0xffff), + TokenKind::Punctuator(Punctuator::Dot), + TokenKind::identifier("ff"), + TokenKind::numeric_literal(0x00ff_ffff), + ]; - assert_eq!( - lexer.tokens[3].kind, - TokenKind::numeric_literal(0x00ff_ffff) - ); + expect_tokens(&mut lexer, &expected); } #[test] fn single_number_without_semicolon() { - let mut lexer = Lexer::new("1"); - lexer.lex().expect("failed to lex"); + let mut lexer = Lexer::new("1".as_bytes()); + match lexer.next() { + Some(Ok(_)) => {} + _ => { + panic!("Failed to lex 1 without semicolon"); + } + } } #[test] fn number_followed_by_dot() { - let mut lexer = Lexer::new("1.."); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1.0)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot)); + let mut lexer = Lexer::new("1..".as_bytes()); + + let expected = [ + TokenKind::numeric_literal(1.0), + TokenKind::Punctuator(Punctuator::Dot), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn regex_literal() { - let mut lexer = Lexer::new("/(?:)/"); - lexer.lex().expect("failed to lex"); - assert_eq!( - lexer.tokens[0].kind, - TokenKind::regular_expression_literal("(?:)", "".parse().unwrap()) - ); + let mut lexer = Lexer::new("/(?:)/".as_bytes()); + + let expected = [TokenKind::regular_expression_literal( + "(?:)", + "".parse().unwrap(), + )]; + + expect_tokens(&mut lexer, &expected); } #[test] fn regex_literal_flags() { - let mut lexer = Lexer::new(r"/\/[^\/]*\/*/gmi"); - lexer.lex().expect("failed to lex"); - assert_eq!( - lexer.tokens[0].kind, - TokenKind::regular_expression_literal("\\/[^\\/]*\\/*", "gmi".parse().unwrap()) - ); + let mut lexer = Lexer::new(r"/\/[^\/]*\/*/gmi".as_bytes()); + + let expected = [TokenKind::regular_expression_literal( + "\\/[^\\/]*\\/*", + "gmi".parse().unwrap(), + )]; + + expect_tokens(&mut lexer, &expected); } #[test] fn addition_no_spaces() { - let mut lexer = Lexer::new("1+1"); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1)); + let mut lexer = Lexer::new("1+1".as_bytes()); + + let expected = [ + TokenKind::numeric_literal(1), + TokenKind::Punctuator(Punctuator::Add), + TokenKind::numeric_literal(1), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn addition_no_spaces_left_side() { - let mut lexer = Lexer::new("1+ 1"); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1)); + let mut lexer = Lexer::new("1+ 1".as_bytes()); + + let expected = [ + TokenKind::numeric_literal(1), + TokenKind::Punctuator(Punctuator::Add), + TokenKind::numeric_literal(1), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn addition_no_spaces_right_side() { - let mut lexer = Lexer::new("1 +1"); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1)); + let mut lexer = Lexer::new("1 +1".as_bytes()); + + let expected = [ + TokenKind::numeric_literal(1), + TokenKind::Punctuator(Punctuator::Add), + TokenKind::numeric_literal(1), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn addition_no_spaces_e_number_left_side() { - let mut lexer = Lexer::new("1e2+ 1"); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(100.0)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1)); + let mut lexer = Lexer::new("1e2+ 1".as_bytes()); + + let expected = [ + TokenKind::numeric_literal(100.0), + TokenKind::Punctuator(Punctuator::Add), + TokenKind::numeric_literal(1), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn addition_no_spaces_e_number_right_side() { - let mut lexer = Lexer::new("1 +1e3"); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1000.0)); + let mut lexer = Lexer::new("1 +1e3".as_bytes()); + + let expected = [ + TokenKind::numeric_literal(1), + TokenKind::Punctuator(Punctuator::Add), + TokenKind::numeric_literal(1000.0), + ]; + + expect_tokens(&mut lexer, &expected); } #[test] fn addition_no_spaces_e_number() { - let mut lexer = Lexer::new("1e3+1e11"); - lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1000.0)); - assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!( - lexer.tokens[2].kind, - TokenKind::numeric_literal(100_000_000_000.0) - ); -} */ \ No newline at end of file + let mut lexer = Lexer::new("1e3+1e11".as_bytes()); + + let expected = [ + TokenKind::numeric_literal(1000.0), + TokenKind::Punctuator(Punctuator::Add), + TokenKind::numeric_literal(100_000_000_000.0), + ]; + + expect_tokens(&mut lexer, &expected); +} From 0398c8bc9dd274fc96f50df175d0cc564e203d61 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 14 Jun 2020 15:34:47 +0100 Subject: [PATCH 022/289] Fixed some clippy warnings --- boa/src/syntax/lexer/tests.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index aaca2d7ac77..d87f760482f 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -253,10 +253,9 @@ fn check_positions() { fn two_divisions_in_expression() { let s = " return a !== 0 || 1 / a === 1 / b;"; let mut lexer = Lexer::new(s.as_bytes()); - // dbg!(&lexer.tokens); assert_eq!( - lexer.skip(11).next().unwrap().unwrap().span(), + lexer.nth(11).unwrap().unwrap().span(), span((1, 37), (1, 37)) ); } @@ -298,7 +297,7 @@ fn check_decrement_advances_lexer_2_places() { #[test] fn check_nan() { let mut lexer = Lexer::new("let a = NaN;".as_bytes()); - match lexer.skip(3).next() { + match lexer.nth(3) { None | Some(Err(_)) => panic!("No token found when expecting NaN"), Some(Ok(token)) => match token.kind() { TokenKind::NumericLiteral(Numeric::Rational(a)) => { @@ -371,11 +370,10 @@ fn hexadecimal_edge_case() { #[test] fn single_number_without_semicolon() { let mut lexer = Lexer::new("1".as_bytes()); - match lexer.next() { - Some(Ok(_)) => {} - _ => { - panic!("Failed to lex 1 without semicolon"); - } + if let Some(Ok(x)) = lexer.next() { + assert_eq!(x.kind(), &TokenKind::numeric_literal(Numeric::Integer(1))); + } else { + panic!("Failed to lex 1 without semicolon"); } } From 0d9c9dade1c08d5dd184549981bd58cc57f51a11 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 14 Jun 2020 15:41:20 +0100 Subject: [PATCH 023/289] Made clippy happy --- boa/src/syntax/lexer/tests.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index d87f760482f..473c6b5cf63 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -276,8 +276,7 @@ fn check_line_numbers() { #[test] fn check_decrement_advances_lexer_2_places() { // Here we want an example of decrementing an integer - let s = "let a = b--;"; - let lexer = Lexer::new(s.as_bytes()); + let lexer = Lexer::new(&b"let a = b--;"[0..]); let mut iter = lexer.skip(4); @@ -296,7 +295,7 @@ fn check_decrement_advances_lexer_2_places() { #[test] fn check_nan() { - let mut lexer = Lexer::new("let a = NaN;".as_bytes()); + let mut lexer = Lexer::new(&b"let a = NaN;"[0..]); match lexer.nth(3) { None | Some(Err(_)) => panic!("No token found when expecting NaN"), Some(Ok(token)) => match token.kind() { @@ -341,7 +340,7 @@ fn numbers() { #[test] fn implicit_octal_edge_case() { - let mut lexer = Lexer::new("044.5 094.5".as_bytes()); + let mut lexer = Lexer::new(&b"044.5 094.5"[0..]); let expected = [ TokenKind::numeric_literal(36), @@ -355,7 +354,7 @@ fn implicit_octal_edge_case() { #[test] fn hexadecimal_edge_case() { - let mut lexer = Lexer::new("0xffff.ff 0xffffff".as_bytes()); + let mut lexer = Lexer::new(&b"0xffff.ff 0xffffff"[0..]); let expected = [ TokenKind::numeric_literal(0xffff), @@ -369,7 +368,7 @@ fn hexadecimal_edge_case() { #[test] fn single_number_without_semicolon() { - let mut lexer = Lexer::new("1".as_bytes()); + let mut lexer = Lexer::new(&b"1"[0..]); if let Some(Ok(x)) = lexer.next() { assert_eq!(x.kind(), &TokenKind::numeric_literal(Numeric::Integer(1))); } else { @@ -379,7 +378,7 @@ fn single_number_without_semicolon() { #[test] fn number_followed_by_dot() { - let mut lexer = Lexer::new("1..".as_bytes()); + let mut lexer = Lexer::new(&b"1.."[0..]); let expected = [ TokenKind::numeric_literal(1.0), @@ -391,7 +390,7 @@ fn number_followed_by_dot() { #[test] fn regex_literal() { - let mut lexer = Lexer::new("/(?:)/".as_bytes()); + let mut lexer = Lexer::new(&b"/(?:)/"[0..]); let expected = [TokenKind::regular_expression_literal( "(?:)", @@ -403,7 +402,7 @@ fn regex_literal() { #[test] fn regex_literal_flags() { - let mut lexer = Lexer::new(r"/\/[^\/]*\/*/gmi".as_bytes()); + let mut lexer = Lexer::new(&br"/\/[^\/]*\/*/gmi"[0..]); let expected = [TokenKind::regular_expression_literal( "\\/[^\\/]*\\/*", @@ -415,7 +414,7 @@ fn regex_literal_flags() { #[test] fn addition_no_spaces() { - let mut lexer = Lexer::new("1+1".as_bytes()); + let mut lexer = Lexer::new(&b"1+1"[0..]); let expected = [ TokenKind::numeric_literal(1), @@ -428,7 +427,7 @@ fn addition_no_spaces() { #[test] fn addition_no_spaces_left_side() { - let mut lexer = Lexer::new("1+ 1".as_bytes()); + let mut lexer = Lexer::new(&b"1+ 1"[0..]); let expected = [ TokenKind::numeric_literal(1), @@ -441,7 +440,7 @@ fn addition_no_spaces_left_side() { #[test] fn addition_no_spaces_right_side() { - let mut lexer = Lexer::new("1 +1".as_bytes()); + let mut lexer = Lexer::new(&b"1 +1"[0..]); let expected = [ TokenKind::numeric_literal(1), @@ -454,7 +453,7 @@ fn addition_no_spaces_right_side() { #[test] fn addition_no_spaces_e_number_left_side() { - let mut lexer = Lexer::new("1e2+ 1".as_bytes()); + let mut lexer = Lexer::new(&b"1e2+ 1"[0..]); let expected = [ TokenKind::numeric_literal(100.0), @@ -467,7 +466,7 @@ fn addition_no_spaces_e_number_left_side() { #[test] fn addition_no_spaces_e_number_right_side() { - let mut lexer = Lexer::new("1 +1e3".as_bytes()); + let mut lexer = Lexer::new(&b"1 +1e3"[0..]); let expected = [ TokenKind::numeric_literal(1), @@ -480,7 +479,7 @@ fn addition_no_spaces_e_number_right_side() { #[test] fn addition_no_spaces_e_number() { - let mut lexer = Lexer::new("1e3+1e11".as_bytes()); + let mut lexer = Lexer::new(&b"1e3+1e11"[0..]); let expected = [ TokenKind::numeric_literal(1000.0), From dd27481f5efa3aa3214229fc5210c9dfda0f8cf3 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 14 Jun 2020 16:33:25 +0100 Subject: [PATCH 024/289] Comment tokens now skipped --- boa/src/syntax/lexer/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 6008e9fe25d..1a1857c2d12 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -204,7 +204,16 @@ where } }; - Some(token) + if let Ok(t) = token { + if t.kind() == &TokenKind::Comment { + // Skip comment + self.next() + } else { + Some(Ok(t)) + } + } else { + Some(token) + } } } From 75f67ce2de14566185dc3df73d9091dd2cdc7b6a Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 14 Jun 2020 17:52:28 +0100 Subject: [PATCH 025/289] fmt, small test fixes --- boa/src/syntax/lexer/tests.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 473c6b5cf63..a49ebb77147 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -17,7 +17,10 @@ where assert_eq!(&lexer.next().unwrap().unwrap().kind(), &expect); } - assert!(lexer.next().is_none()); + assert!( + lexer.next().is_none(), + "Unexpected extra token lexed at end of input" + ); } #[test] @@ -74,10 +77,8 @@ fn check_template_literal_unterminated() { let mut lexer = Lexer::new(s.as_bytes()); match lexer.next() { - None | Some(Ok(_)) => panic!("Lexer did not detect end of stream"), - Some(Err(e)) => { - assert_eq!(e.to_string(), "Unterminated template literal"); - } + Some(Err(Error::IO(_))) => {} + _ => panic!("Lexer did not handle unterminated literal with error"), } } @@ -203,6 +204,7 @@ fn check_variable_definition_tokens() { TokenKind::identifier("a"), TokenKind::Punctuator(Punctuator::Assign), TokenKind::string_literal("hello"), + TokenKind::Punctuator(Punctuator::Semicolon), ]; expect_tokens(&mut lexer, &expected); From c968ad6bddcc9dbe4fb1225c776142ec37d2f65b Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Jun 2020 00:24:51 +0100 Subject: [PATCH 026/289] Take until cursor implementation --- boa/src/syntax/ast/position.rs | 5 ++++- boa/src/syntax/lexer/cursor.rs | 27 ++++++++++++++++++++++++--- boa/src/syntax/lexer/tests.rs | 15 ++++++++------- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/boa/src/syntax/ast/position.rs b/boa/src/syntax/ast/position.rs index 15bef038408..5f3e832021a 100644 --- a/boa/src/syntax/ast/position.rs +++ b/boa/src/syntax/ast/position.rs @@ -7,7 +7,10 @@ use serde::{Deserialize, Serialize}; /// A position in the JavaScript source code. /// -/// Stores both the column number and the line number +/// Stores both the column number and the line number. +/// +/// Note that spans are of the form [begining, end) i.e. that the begining position is inclusive and the end position is exclusive. +/// See test check_positions from syntax/lexer/tests.rs for an example. /// /// ## Similar Implementations /// [V8: Location](https://cs.chromium.org/chromium/src/v8/src/parsing/scanner.h?type=cs&q=isValid+Location&g=0&l=216) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 641538e24aa..753fa4dcdfe 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -1,5 +1,5 @@ use crate::syntax::ast::Position; -use std::io::{self, Bytes, Read}; +use std::io::{self, Bytes, Read, Error, ErrorKind}; /// Cursor over the source code. #[derive(Debug)] @@ -76,8 +76,29 @@ where /// Fills the buffer with all characters until the stop character is found. /// /// Note: It will not add the stop character to the buffer. - pub(super) fn take_until(&mut self, _stop: char, _buf: &mut String) -> io::Result<()> { - unimplemented!() + /// + /// Returns syntax + pub(super) fn take_until(&mut self, stop: char, buf: &mut String) -> io::Result<()> { + loop { + if self.next_is(stop)? { + return Ok(()); + } else { + match self.next() { + None => { + return Err(io::Error::new( + ErrorKind::UnexpectedEof, + format!("Unexpected end of file when looking for character {}", stop), + )); + } + Some(Err(e)) => { + return Err(e); + } + Some(Ok(ch)) => { + buf.push(ch); + } + } + } + } } /// Retrieves the given number of characters and adds them to the buffer. diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index a49ebb77147..2da64bc29c6 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -5,6 +5,7 @@ use super::token::Numeric; use super::*; use crate::syntax::ast::Keyword; + fn span(start: (u32, u32), end: (u32, u32)) -> Span { Span::new(Position::new(start.0, start.1), Position::new(end.0, end.1)) } @@ -217,36 +218,36 @@ fn check_positions() { let mut lexer = Lexer::new(s.as_bytes()); // The first column is 1 (not zero indexed) - assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 1), (1, 7))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 1), (1, 8))); // Dot Token starts on column 8 - assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 8), (1, 8))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 8), (1, 9))); // Log Token starts on column 9 - assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 9), (1, 11))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 9), (1, 12))); // Open parenthesis token starts on column 12 assert_eq!( lexer.next().unwrap().unwrap().span(), - span((1, 12), (1, 12)) + span((1, 12), (1, 13)) ); // String token starts on column 13 assert_eq!( lexer.next().unwrap().unwrap().span(), - span((1, 13), (1, 33)) + span((1, 13), (1, 34)) ); // Close parenthesis token starts on column 34 assert_eq!( lexer.next().unwrap().unwrap().span(), - span((1, 34), (1, 34)) + span((1, 34), (1, 35)) ); // Semi Colon token starts on column 35 assert_eq!( lexer.next().unwrap().unwrap().span(), - span((1, 35), (1, 35)) + span((1, 35), (1, 36)) ); } From 789956cb7f7211c0005e05ffb05efb05418f1a50 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Jun 2020 00:30:22 +0100 Subject: [PATCH 027/289] Small fixes --- boa/src/syntax/lexer/cursor.rs | 4 ++-- boa/src/syntax/lexer/tests.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 753fa4dcdfe..51b16fca45a 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -1,5 +1,5 @@ use crate::syntax::ast::Position; -use std::io::{self, Bytes, Read, Error, ErrorKind}; +use std::io::{self, Bytes, ErrorKind, Read}; /// Cursor over the source code. #[derive(Debug)] @@ -77,7 +77,7 @@ where /// /// Note: It will not add the stop character to the buffer. /// - /// Returns syntax + /// Returns syntax pub(super) fn take_until(&mut self, stop: char, buf: &mut String) -> io::Result<()> { loop { if self.next_is(stop)? { diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 2da64bc29c6..4b58bf8e22c 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -5,7 +5,6 @@ use super::token::Numeric; use super::*; use crate::syntax::ast::Keyword; - fn span(start: (u32, u32), end: (u32, u32)) -> Span { Span::new(Position::new(start.0, start.1), Position::new(end.0, end.1)) } From e791023d0940ce09a152c0a1e8eafaef0e5e3140 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Jun 2020 11:34:37 +0100 Subject: [PATCH 028/289] Looking at a possible way of doing number lexing differently --- boa/src/syntax/lexer/error.rs | 1 + boa/src/syntax/lexer/number.rs | 81 ++++++++++---- boa/src/syntax/lexer/number_wip.rs | 171 +++++++++++++++++++++++++++++ boa/src/syntax/lexer/tests.rs | 11 ++ 4 files changed, 244 insertions(+), 20 deletions(-) create mode 100644 boa/src/syntax/lexer/number_wip.rs diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 3c1b65e2a3b..5f3edd1a694 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -5,6 +5,7 @@ pub enum Error { IO(io::Error), Syntax(Box), StrictMode(Box), // Not 100% decided on this name. + // Reverted(), } impl From for Error { diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 5a203c35931..683be5d668e 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -68,10 +68,12 @@ impl Tokenizer for NumberLiteral { // Default assume the number is a base 10 integer. let mut kind = NumericKind::Integer(10); - let c = cursor.next(); - if self.init == '0' { - match c { + if cursor.next_is('x') | cursor.next_is('X') { + + } + + match peek_ch { None => { // DecimalLiteral lexing. // Indicates that the number is just a single 0. @@ -81,18 +83,31 @@ impl Tokenizer for NumberLiteral { )); } Some(Err(e)) => { - return Err(Error::from(e)); + todo!(); + // TODO } Some(Ok('x')) | Some(Ok('X')) => { // HexIntegerLiteral + + cursor.next(); // Consume the 0x. + buf.pop(); + kind = NumericKind::Integer(16); } Some(Ok('o')) | Some(Ok('O')) => { // OctalIntegerLiteral + + cursor.next(); // Consume the 0o. + buf.pop(); + kind = NumericKind::Integer(8); } Some(Ok('b')) | Some(Ok('B')) => { // BinaryIntegerLiteral + + cursor.next(); // Consume the 0b. + buf.pop(); + kind = NumericKind::Integer(2); } Some(Ok('n')) => { @@ -111,10 +126,12 @@ impl Tokenizer for NumberLiteral { "Implicit octal literals are not allowed in strict mode.", )); } else { - buf.push(ch); + cursor.next(); + buf.push(*ch); + kind = NumericKind::Integer(8); } - } else if ch.is_digit(36) { + } else if ch.is_digit(10) { // Indicates a numerical digit comes after then 0 but it isn't an octal digit // so therefore this must be a number with an unneeded leading 0. This is // forbidden in strict mode. @@ -124,32 +141,56 @@ impl Tokenizer for NumberLiteral { "Leading 0's are not allowed in strict mode.", )); } else { - buf.push(ch); + cursor.next(); + buf.push(*ch); } } else { // Indicates that the symbol is a non-number, this is valid if it is a dot or similar. - buf.push(ch); } } } } - while let Some(ch) = cursor.peek() { - match ch { - Err(_e) => { - // TODO, handle. - } - Ok(c) if c.is_digit(kind.base()) => { - let s = cursor.next().unwrap().unwrap(); - buf.push(s); - } - _ => { - // A non-number symbol detected, this might be a dot or similar. - break; + println!("{:?}", cursor.peek()); + + // if let Some(ch) = c { + // buf.push(ch?); + // } + + loop { + if let Some(ch) = cursor.peek() { + match ch { + Err(_e) => { + // TODO, handle. + } + Ok(c) if c.is_digit(kind.base()) => { + let s = cursor.next().unwrap().unwrap(); + buf.push(s); + } + _ => { + // A non-number symbol detected, this might be a dot or similar. + break; + } } } } + // while let Some(ch) = cursor.peek() { + // match ch { + // Err(_e) => { + // // TODO, handle. + // } + // Ok(c) if c.is_digit(kind.base()) => { + // let s = cursor.next().unwrap().unwrap(); + // buf.push(s); + // } + // _ => { + // // A non-number symbol detected, this might be a dot or similar. + // break; + // } + // } + // } + if cursor.next_is('n')? { // DecimalBigIntegerLiteral kind = kind.to_bigint(); diff --git a/boa/src/syntax/lexer/number_wip.rs b/boa/src/syntax/lexer/number_wip.rs new file mode 100644 index 00000000000..5cb4adccd07 --- /dev/null +++ b/boa/src/syntax/lexer/number_wip.rs @@ -0,0 +1,171 @@ +use super::{Cursor, Error, TokenKind, Tokenizer}; +use crate::builtins::BigInt; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{token::Numeric, Token}; +use std::io::Read; +use std::str::FromStr; + +/// Number literal lexing. +/// +/// Assumes the initial digit is consumed by the cursor (stored in init). +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals +/// [mdn]: +#[derive(Debug, Clone, Copy)] +pub(super) struct NumberLiteral { + init: char, + strict_mode: bool, +} + +impl NumberLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } +} + +/// This is a helper structure +/// +/// This structure helps with identifying what numerical type it is and what base is it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NumericKind { + Rational, + Integer(u8), + BigInt(u8), +} + +impl NumericKind { + /// Get the base of the number kind. + fn base(self) -> u32 { + match self { + Self::Rational => 10, + Self::Integer(base) => base as u32, + Self::BigInt(base) => base as u32, + } + } + + /// Converts `self` to BigInt kind. + fn to_bigint(self) -> Self { + match self { + Self::Rational => unreachable!("can not convert rational number to BigInt"), + Self::Integer(base) => Self::BigInt(base), + Self::BigInt(base) => Self::BigInt(base), + } + } +} + +impl Tokenizer for NumberLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + if let Ok(token) = DecimalLiteral::new(self.init, self.strict_mode).lex(cursor, start_pos) { + return Ok(token); // Parsed successfully. + } + if let Ok(token) = DecimalBigIntegerLiteral::new(self.init).lex(cursor, start_pos) { + return Ok(token); // Parsed successfully. + } + if let Ok(token) = NonDecimalIntegerLiteral::new(self.init).lex(cursor, start_pos) { + return Ok(token); // Parsed successfully. + } + + Err(Error::Reverted()) + + + // Ok(Token::new( + // TokenKind::NumericLiteral(num), + // Span::new(start_pos, cursor.pos()), + // )) + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct DecimalLiteral { + init: char, + strict_mode: bool, +} + +impl DecimalLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } + + pub(super) fn parse_numerical_literal() -> Token { + + } +} + +impl Tokenizer for DecimalLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + if let Ok(dil_token) = DecimalIntegerLiteral::new(self.init, self.strict_mode).lex(cursor, start_pos) { + if cursor.next_is('.')? { + // Expecting 0 or more decimal digits + if let Ok(dd_token) = DecimalDigits::new(self.strict_mode).lex(cursor, start_pos) { + if let Ok(ep_token) = ExponentPart::new(self.strict_mode).lex(cursor, start_pos) { + + } else { + let val = dil_token.kind(); + } + } + } + + + return Ok(token); // Parsed successfully. + } + + Err(Error::Reverted()) + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct DecimalBigIntegerLiteral { + init: char, + strict_mode: bool, +} + +impl DecimalBigIntegerLiteral{ + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } +} + +impl Tokenizer for DecimalBigIntegerLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + Err(Error::Reverted()) + } +} + + +#[derive(Debug, Clone, Copy)] +pub(super) struct NonDecimalIntegerLiteral { + init: char, + strict_mode: bool, +} + +impl NonDecimalIntegerLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } +} + +impl Tokenizer for NonDecimalIntegerLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + Err(Error::Reverted()) + } +} \ No newline at end of file diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 4b58bf8e22c..a980d8afe48 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -309,6 +309,17 @@ fn check_nan() { } } +#[test] +fn single_int() { + let mut lexer = Lexer::new(&b"52"[0..]); + + let expected = [ + TokenKind::numeric_literal(52), + ]; + + expect_tokens(&mut lexer, &expected); +} + #[test] fn numbers() { let mut lexer = Lexer::new( From 34021f0b230d9f02f9aaf95c63b6d968e93876c2 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Jun 2020 16:31:31 +0100 Subject: [PATCH 029/289] Number lexing mostly working (299 tests passed) --- boa/src/syntax/lexer/error.rs | 2 +- boa/src/syntax/lexer/number.rs | 93 +++--- boa/src/syntax/lexer/number_wip.rs | 171 ----------- ignore_temp/number_grammar_based.rs | 261 +++++++++++++++++ ignore_temp/number_old.rs | 424 ++++++++++++++++++++++++++++ 5 files changed, 721 insertions(+), 230 deletions(-) delete mode 100644 boa/src/syntax/lexer/number_wip.rs create mode 100644 ignore_temp/number_grammar_based.rs create mode 100644 ignore_temp/number_old.rs diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 5f3edd1a694..042488595c4 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -5,7 +5,7 @@ pub enum Error { IO(io::Error), Syntax(Box), StrictMode(Box), // Not 100% decided on this name. - // Reverted(), + // Reverted(String), } impl From for Error { diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 683be5d668e..8d7289f667f 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -68,12 +68,10 @@ impl Tokenizer for NumberLiteral { // Default assume the number is a base 10 integer. let mut kind = NumericKind::Integer(10); - if self.init == '0' { - if cursor.next_is('x') | cursor.next_is('X') { - - } + let c = cursor.peek(); - match peek_ch { + if self.init == '0' { + match c { None => { // DecimalLiteral lexing. // Indicates that the number is just a single 0. @@ -84,33 +82,34 @@ impl Tokenizer for NumberLiteral { } Some(Err(e)) => { todo!(); - // TODO } Some(Ok('x')) | Some(Ok('X')) => { - // HexIntegerLiteral - - cursor.next(); // Consume the 0x. + // Remove the initial '0' from buffer. + cursor.next(); buf.pop(); + // HexIntegerLiteral kind = NumericKind::Integer(16); } Some(Ok('o')) | Some(Ok('O')) => { - // OctalIntegerLiteral - - cursor.next(); // Consume the 0o. + // Remove the initial '0' from buffer. + cursor.next(); buf.pop(); + // OctalIntegerLiteral kind = NumericKind::Integer(8); } Some(Ok('b')) | Some(Ok('B')) => { - // BinaryIntegerLiteral - - cursor.next(); // Consume the 0b. + // Remove the initial '0' from buffer. + cursor.next(); buf.pop(); + // BinaryIntegerLiteral kind = NumericKind::Integer(2); } Some(Ok('n')) => { + cursor.next(); + // DecimalBigIntegerLiteral '0n' return Ok(Token::new( TokenKind::NumericLiteral(Numeric::BigInt(0.into())), @@ -126,8 +125,11 @@ impl Tokenizer for NumberLiteral { "Implicit octal literals are not allowed in strict mode.", )); } else { - cursor.next(); - buf.push(*ch); + // Remove the initial '0' from buffer. + buf.pop(); + + let char = cursor.next().unwrap().unwrap(); + buf.push(char); kind = NumericKind::Integer(8); } @@ -136,61 +138,36 @@ impl Tokenizer for NumberLiteral { // so therefore this must be a number with an unneeded leading 0. This is // forbidden in strict mode. if self.strict_mode { - // LegacyOctalIntegerLiteral is forbidden with strict mode true. return Err(Error::strict( "Leading 0's are not allowed in strict mode.", )); } else { - cursor.next(); - buf.push(*ch); + let char = cursor.next().unwrap().unwrap(); + buf.push(char); } } else { - // Indicates that the symbol is a non-number, this is valid if it is a dot or similar. + // Indicates that the symbol is a non-number. } } } } - println!("{:?}", cursor.peek()); - - // if let Some(ch) = c { - // buf.push(ch?); - // } - - loop { - if let Some(ch) = cursor.peek() { - match ch { - Err(_e) => { - // TODO, handle. - } - Ok(c) if c.is_digit(kind.base()) => { - let s = cursor.next().unwrap().unwrap(); - buf.push(s); - } - _ => { - // A non-number symbol detected, this might be a dot or similar. - break; - } + while let Some(ch) = cursor.peek() { + match ch { + Err(_e) => { + // TODO, handle. + } + Ok(c) if c.is_digit(kind.base()) => { + let s = cursor.next().unwrap().unwrap(); + buf.push(s); + } + _ => { + // A non-number symbol detected, this might be a dot or similar. + break; } } } - // while let Some(ch) = cursor.peek() { - // match ch { - // Err(_e) => { - // // TODO, handle. - // } - // Ok(c) if c.is_digit(kind.base()) => { - // let s = cursor.next().unwrap().unwrap(); - // buf.push(s); - // } - // _ => { - // // A non-number symbol detected, this might be a dot or similar. - // break; - // } - // } - // } - if cursor.next_is('n')? { // DecimalBigIntegerLiteral kind = kind.to_bigint(); @@ -334,4 +311,4 @@ impl Tokenizer for NumberLiteral { Span::new(start_pos, cursor.pos()), )) } -} +} \ No newline at end of file diff --git a/boa/src/syntax/lexer/number_wip.rs b/boa/src/syntax/lexer/number_wip.rs deleted file mode 100644 index 5cb4adccd07..00000000000 --- a/boa/src/syntax/lexer/number_wip.rs +++ /dev/null @@ -1,171 +0,0 @@ -use super::{Cursor, Error, TokenKind, Tokenizer}; -use crate::builtins::BigInt; -use crate::syntax::ast::{Position, Span}; -use crate::syntax::lexer::{token::Numeric, Token}; -use std::io::Read; -use std::str::FromStr; - -/// Number literal lexing. -/// -/// Assumes the initial digit is consumed by the cursor (stored in init). -/// -/// More information: -/// - [ECMAScript reference][spec] -/// - [MDN documentation][mdn] -/// -/// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals -/// [mdn]: -#[derive(Debug, Clone, Copy)] -pub(super) struct NumberLiteral { - init: char, - strict_mode: bool, -} - -impl NumberLiteral { - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode } - } -} - -/// This is a helper structure -/// -/// This structure helps with identifying what numerical type it is and what base is it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum NumericKind { - Rational, - Integer(u8), - BigInt(u8), -} - -impl NumericKind { - /// Get the base of the number kind. - fn base(self) -> u32 { - match self { - Self::Rational => 10, - Self::Integer(base) => base as u32, - Self::BigInt(base) => base as u32, - } - } - - /// Converts `self` to BigInt kind. - fn to_bigint(self) -> Self { - match self { - Self::Rational => unreachable!("can not convert rational number to BigInt"), - Self::Integer(base) => Self::BigInt(base), - Self::BigInt(base) => Self::BigInt(base), - } - } -} - -impl Tokenizer for NumberLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - if let Ok(token) = DecimalLiteral::new(self.init, self.strict_mode).lex(cursor, start_pos) { - return Ok(token); // Parsed successfully. - } - if let Ok(token) = DecimalBigIntegerLiteral::new(self.init).lex(cursor, start_pos) { - return Ok(token); // Parsed successfully. - } - if let Ok(token) = NonDecimalIntegerLiteral::new(self.init).lex(cursor, start_pos) { - return Ok(token); // Parsed successfully. - } - - Err(Error::Reverted()) - - - // Ok(Token::new( - // TokenKind::NumericLiteral(num), - // Span::new(start_pos, cursor.pos()), - // )) - } -} - -#[derive(Debug, Clone, Copy)] -pub(super) struct DecimalLiteral { - init: char, - strict_mode: bool, -} - -impl DecimalLiteral { - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode } - } - - pub(super) fn parse_numerical_literal() -> Token { - - } -} - -impl Tokenizer for DecimalLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - if let Ok(dil_token) = DecimalIntegerLiteral::new(self.init, self.strict_mode).lex(cursor, start_pos) { - if cursor.next_is('.')? { - // Expecting 0 or more decimal digits - if let Ok(dd_token) = DecimalDigits::new(self.strict_mode).lex(cursor, start_pos) { - if let Ok(ep_token) = ExponentPart::new(self.strict_mode).lex(cursor, start_pos) { - - } else { - let val = dil_token.kind(); - } - } - } - - - return Ok(token); // Parsed successfully. - } - - Err(Error::Reverted()) - } -} - -#[derive(Debug, Clone, Copy)] -pub(super) struct DecimalBigIntegerLiteral { - init: char, - strict_mode: bool, -} - -impl DecimalBigIntegerLiteral{ - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode } - } -} - -impl Tokenizer for DecimalBigIntegerLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - Err(Error::Reverted()) - } -} - - -#[derive(Debug, Clone, Copy)] -pub(super) struct NonDecimalIntegerLiteral { - init: char, - strict_mode: bool, -} - -impl NonDecimalIntegerLiteral { - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode } - } -} - -impl Tokenizer for NonDecimalIntegerLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - Err(Error::Reverted()) - } -} \ No newline at end of file diff --git a/ignore_temp/number_grammar_based.rs b/ignore_temp/number_grammar_based.rs new file mode 100644 index 00000000000..963ad4238b2 --- /dev/null +++ b/ignore_temp/number_grammar_based.rs @@ -0,0 +1,261 @@ +use super::{Cursor, Error, TokenKind, Tokenizer}; +use crate::builtins::BigInt; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{token::Numeric, Token}; +use std::io::Read; +use std::str::FromStr; + +/// Number literal lexing. +/// +/// Assumes the initial digit is consumed by the cursor (stored in init). +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals +/// [mdn]: +#[derive(Debug, Clone, Copy)] +pub(super) struct NumberLiteral { + init: char, + strict_mode: bool, +} + +impl NumberLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } +} + +/// This is a helper structure +/// +/// This structure helps with identifying what numerical type it is and what base is it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NumericKind { + Rational, + Integer(u8), + BigInt(u8), +} + +impl NumericKind { + /// Get the base of the number kind. + fn base(self) -> u32 { + match self { + Self::Rational => 10, + Self::Integer(base) => base as u32, + Self::BigInt(base) => base as u32, + } + } + + /// Converts `self` to BigInt kind. + fn to_bigint(self) -> Self { + match self { + Self::Rational => unreachable!("can not convert rational number to BigInt"), + Self::Integer(base) => Self::BigInt(base), + Self::BigInt(base) => Self::BigInt(base), + } + } +} + +impl Tokenizer for NumberLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + let buf = self.init.to_string(); + + if let Ok(token) = DecimalLiteral::new(self.init, self.strict_mode).lex(cursor, start_pos) { + return Ok(token); // Parsed successfully. + } + if let Ok(token) = DecimalBigIntegerLiteral::new(self.init).lex(cursor, start_pos) { + return Ok(token); // Parsed successfully. + } + if let Ok(token) = NonDecimalIntegerLiteral::new(self.init).lex(cursor, start_pos) { + return Ok(token); // Parsed successfully. + } + + Err(Error::Reverted()) + + + // Ok(Token::new( + // TokenKind::NumericLiteral(num), + // Span::new(start_pos, cursor.pos()), + // )) + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct DecimalLiteral { + init: char, + strict_mode: bool, +} + +impl DecimalLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode} + } +} + +impl Tokenizer for DecimalLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + + + let dil = DecimalIntegerLiteral::new(self.init, self.strict_mode).lex(cursor, start_pos); + match dil { + Ok(dil_token) => { + // DecimalIntegerLiteral + + if cursor.next_is('.')? { + // DecimalIntegerLiteral. + + // Consume the '.' + cursor.next(); + + + // May be followed by DecimalDigits + let dd = DecimalDigits::new(self.strict_mode).lex(cursor, start_pos); + match dd { + Ok(dd_token) => { + // DecimalIntegerLiteral.DecimalDigits + let ep = ExponentPart::new(self.strict_mode).lex(cursor, start_pos); + match ep { + Ok(ep_token) => { + // DecimalIntegerLiteral.DecimalDigits ExponentPart + // Terminal pattern. + dil + dd + ep + } + Err(Error::Reverted()) => { + // DecimalIntegerLiteral.DecimalDigits + // Terminal pattern. + dil + dd + } + Err (e) => { + // Some other error preventing lexing. + Err(e) + } + } + } + Err(Error::Reverted()) => { + // DecimalIntegerLiteral. + // Terminal pattern. + dd + } + Err(e) => { + // Some other error preventing lexing. + Err(e) + } + } + } else { + // DecimalIntegerLiteral + + // May be followed by ExponentPart + let ep = ExponentPart::new(self.strict_mode).lex(cursor, start_pos); + match ep { + Ok(ep_token) => { + // DecimalIntegerLiteral ExponentPart + // Terminal pattern. + dil + ep + } + Err(Error::Reverted()) => { + // DecimalIntegerLiteral + dil + } + Err (e) => { + // Some other error preventing lexing. + Err(e) + } + } + } + } + Err(Error::Reverted(buf)) => { + // If a decimal literal doesn't start with a DecimalIntegerLiteral it must start with a '.' followed by DecimalDigits. + if cursor.next_is('.')? { + // . + let dd = DecimalDigits::new(self.strict_mode).lex(cursor, start_pos); + match dd { + Ok(dd_token) => { + // . DecimalDigits + + // May be followed by ExponentPart + let ep = ExponentPart::new(self.strict_mode).lex(cursor, start_pos); + match ep { + Ok(ep_token) => { + // . DecimalDigits ExponentPart + dd + ep + } + Err(Error::Reverted()) => { + // . DecimalDigits + dd + } + Err (e) => { + // Some other error preventing lexing. + Err(e) + } + } + } + Err(e) => { + // A DecimalDigits couldn't be lexed or some other error prevents lexing. + Err(e) + } + } + } else { + Err(Error::Reverted()) + } + } + Err(e) => { + // Some other error. + Err(e) + } + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct DecimalBigIntegerLiteral { + init: char, + strict_mode: bool, +} + +impl DecimalBigIntegerLiteral{ + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } +} + +impl Tokenizer for DecimalBigIntegerLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + Err(Error::Reverted()) + } +} + + +#[derive(Debug, Clone, Copy)] +pub(super) struct NonDecimalIntegerLiteral { + init: char, + strict_mode: bool, +} + +impl NonDecimalIntegerLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } +} + +impl Tokenizer for NonDecimalIntegerLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + Err(Error::Reverted()) + } +} \ No newline at end of file diff --git a/ignore_temp/number_old.rs b/ignore_temp/number_old.rs new file mode 100644 index 00000000000..796ef68abdc --- /dev/null +++ b/ignore_temp/number_old.rs @@ -0,0 +1,424 @@ +use super::{Cursor, Error, TokenKind, Tokenizer}; +use crate::builtins::BigInt; +use crate::syntax::ast::{Position, Span}; +use crate::syntax::lexer::{token::Numeric, Token}; +use std::io::Read; +use std::str::FromStr; + +/// Number literal lexing. +/// +/// Assumes the digit is consumed by the cursor (stored in init). +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals +/// [mdn]: +#[derive(Debug, Clone, Copy)] +pub(super) struct NumberLiteral { + init: char, + strict_mode: bool, +} + +impl NumberLiteral { + /// Creates a new string literal lexer. + pub(super) fn new(init: char, strict_mode: bool) -> Self { + Self { init, strict_mode } + } +} + +/// This is a helper structure +/// +/// This structure helps with identifying what numerical type it is and what base is it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NumericKind { + Rational, + Integer(u8), + BigInt(u8), +} + +impl NumericKind { + /// Get the base of the number kind. + fn base(self) -> u32 { + match self { + Self::Rational => 10, + Self::Integer(base) => base as u32, + Self::BigInt(base) => base as u32, + } + } + + /// Converts `self` to BigInt kind. + fn to_bigint(self) -> Self { + match self { + Self::Rational => unreachable!("can not convert rational number to BigInt"), + Self::Integer(base) => Self::BigInt(base), + Self::BigInt(base) => Self::BigInt(base), + } + } +} + +impl Tokenizer for NumberLiteral { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + let mut buf = self.init.to_string(); + + // Default assume the number is a base 10 integer. + let mut kind = NumericKind::Integer(10); + + if self.init == '0' { + if cursor.next_is('x') | cursor.next_is('X') { + // HexIntegerLiteral + + // Consume the 0x. + cursor.next(); + buf.pop(); + + kind = NumericKind::Integer(16); + } else if cursor.next_is('o') | cursor.next_is('O') { + // OctalIntegerLiteral + + // Consume the 0o. + cursor.next(); + buf.pop(); + + kind = NumericKind::Integer(8); + } else if cursor.next_is('b') | cursor.next_is('B') { + // BinaryIntegerLiteral + + // Consume the 0b. + cursor.next(); + buf.pop(); + + kind = NumericKind::Integer(2); + } else if cursor.next_is('n') { + // DecimalBigIntegerLiteral '0n' + + // Consume the 'n' + cursor.next(); + + return Ok(Token::new( + TokenKind::NumericLiteral(Numeric::BigInt(0.into())), + Span::new(start_pos, cursor.pos()), + )); + } else { + let ch = cursor.peek(); + if ch.is_some() { + if ch.is_digit(8) { + // LegacyOctalIntegerLiteral + if self.strict_mode { + // LegacyOctalIntegerLiteral is forbidden with strict mode true. + return Err(Error::strict( + "Implicit octal literals are not allowed in strict mode.", + )); + } else { + cursor.next(); + buf.push(*ch); + + kind = NumericKind::Integer(8); + } + } else if ch.is_digit(10) { + // Indicates a numerical digit comes after then 0 but it isn't an octal digit + // so therefore this must be a number with an unneeded leading 0. This is + // forbidden in strict mode. + if self.strict_mode { + // LegacyOctalIntegerLiteral is forbidden with strict mode true. + return Err(Error::strict( + "Leading 0's are not allowed in strict mode.", + )); + } else { + cursor.next(); + buf.push(*ch); + } + } else { + // Indicates that the symbol is a non-number, this is valid if it is a dot or similar. + } + } + + + + } + + if ch.is_digit(8) { + // // LegacyOctalIntegerLiteral + // if self.strict_mode { + // // LegacyOctalIntegerLiteral is forbidden with strict mode true. + // return Err(Error::strict( + // "Implicit octal literals are not allowed in strict mode.", + // )); + // } else { + // cursor.next(); + // buf.push(*ch); + + // kind = NumericKind::Integer(8); + // } + } + } + + + + + // match peek_ch { + // None => { + // // DecimalLiteral lexing. + // // Indicates that the number is just a single 0. + // return Ok(Token::new( + // TokenKind::NumericLiteral(Numeric::Integer(0)), + // Span::new(start_pos, cursor.pos()), + // )); + // } + // Some(Err(e)) => { + // todo!(); + // // TODO + // } + // Some(Ok('x')) | Some(Ok('X')) => { + // // HexIntegerLiteral + + // cursor.next(); // Consume the 0x. + // buf.pop(); + + // kind = NumericKind::Integer(16); + // } + // Some(Ok('o')) | Some(Ok('O')) => { + // // OctalIntegerLiteral + + // cursor.next(); // Consume the 0o. + // buf.pop(); + + // kind = NumericKind::Integer(8); + // } + // Some(Ok('b')) | Some(Ok('B')) => { + // // BinaryIntegerLiteral + + // cursor.next(); // Consume the 0b. + // buf.pop(); + + // kind = NumericKind::Integer(2); + // } + // Some(Ok('n')) => { + // // DecimalBigIntegerLiteral '0n' + // return Ok(Token::new( + // TokenKind::NumericLiteral(Numeric::BigInt(0.into())), + // Span::new(start_pos, cursor.pos()), + // )); + // } + // Some(Ok(ch)) => { + // if ch.is_digit(8) { + // // LegacyOctalIntegerLiteral + // if self.strict_mode { + // // LegacyOctalIntegerLiteral is forbidden with strict mode true. + // return Err(Error::strict( + // "Implicit octal literals are not allowed in strict mode.", + // )); + // } else { + // cursor.next(); + // buf.push(*ch); + + // kind = NumericKind::Integer(8); + // } + // } else if ch.is_digit(10) { + // // Indicates a numerical digit comes after then 0 but it isn't an octal digit + // // so therefore this must be a number with an unneeded leading 0. This is + // // forbidden in strict mode. + // if self.strict_mode { + // // LegacyOctalIntegerLiteral is forbidden with strict mode true. + // return Err(Error::strict( + // "Leading 0's are not allowed in strict mode.", + // )); + // } else { + // cursor.next(); + // buf.push(*ch); + // } + // } else { + // // Indicates that the symbol is a non-number, this is valid if it is a dot or similar. + // } + // } + } + } + + println!("{:?}", cursor.peek()); + + // if let Some(ch) = c { + // buf.push(ch?); + // } + + loop { + if let Some(ch) = cursor.peek() { + match ch { + Err(_e) => { + // TODO, handle. + } + Ok(c) if c.is_digit(kind.base()) => { + let s = cursor.next().unwrap().unwrap(); + buf.push(s); + } + _ => { + // A non-number symbol detected, this might be a dot or similar. + break; + } + } + } + } + + // while let Some(ch) = cursor.peek() { + // match ch { + // Err(_e) => { + // // TODO, handle. + // } + // Ok(c) if c.is_digit(kind.base()) => { + // let s = cursor.next().unwrap().unwrap(); + // buf.push(s); + // } + // _ => { + // // A non-number symbol detected, this might be a dot or similar. + // break; + // } + // } + // } + + if cursor.next_is('n')? { + // DecimalBigIntegerLiteral + kind = kind.to_bigint(); + } + + if let NumericKind::Integer(10) = kind { + 'digitloop: while let Some(ch) = cursor.peek() { + match ch { + Err(_e) => { + // TODO + } + Ok('.') => loop { + kind = NumericKind::Rational; + match cursor.next() { + None => { + // Finished + break; + } + + Some(Err(e)) => { + return Err(Error::from(e)); + } + + Some(Ok(c)) => { + buf.push(c); + } + } + + match cursor.peek() { + None => { + break; + } + Some(Err(_e)) => { + // TODO + } + Some(Ok('e')) | Some(Ok('E')) => { + // TODO scientific notation. + + unimplemented!(); + + // match self + // .preview_multiple_next(2) + // .unwrap_or_default() + // .to_digit(10) + // { + // Some(0..=9) | None => { + // buf.push(self.next()); + // } + // _ => { + // break 'digitloop; + // } + // } + } + Some(Ok(cx)) if !cx.is_digit(10) => { + break 'digitloop; + } + _ => {} + } + }, + Ok('e') | Ok('E') => { + // TODO scientific notation. + unimplemented!(); + + // kind = NumericKind::Rational; + // match self + // .preview_multiple_next(2) + // .unwrap_or_default() + // .to_digit(10) + // { + // Some(0..=9) | None => { + // buf.push(self.next()); + // } + // _ => { + // break; + // } + // } + // buf.push(self.next()); + } + Ok('+') | Ok('-') => { + break; + } + Ok(cx) if cx.is_digit(10) => { + // cursor.next(); + match cursor.next() { + None => { + // Finished + break; + } + + Some(Err(e)) => { + return Err(Error::from(e)); + } + + Some(Ok(c)) => { + buf.push(c); + } + } + // buf.push(*cx); + } + Ok(_) => break, + } + } + } + + // TODO + //self.check_after_numeric_literal()?; + + let num = match kind { + NumericKind::BigInt(base) => { + Numeric::BigInt( + BigInt::from_string_radix(&buf, base as u32).expect("Could not convert to BigInt") + ) + } + NumericKind::Rational /* base: 10 */ => { + Numeric::Rational( + f64::from_str(&buf) + .map_err(|_| Error::syntax("Could not convert value to f64"))?, + ) + } + NumericKind::Integer(base) => { + if let Ok(num) = i32::from_str_radix(&buf, base as u32) { + Numeric::Integer( + num + ) + } else { + let b = f64::from(base); + let mut result = 0.0_f64; + for c in buf.chars() { + let digit = f64::from(c.to_digit(base as u32).unwrap()); + result = result * b + digit; + } + + Numeric::Rational(result) + } + + } + }; + + Ok(Token::new( + TokenKind::NumericLiteral(num), + Span::new(start_pos, cursor.pos()), + )) + } +} From 6e03c41ef353796d8a5ca86436777a10b32d7a24 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Jun 2020 17:41:04 +0100 Subject: [PATCH 030/289] Added to cursor: take_until_pred, next_if_pred --- boa/src/syntax/lexer/cursor.rs | 45 +++++++- boa/src/syntax/lexer/error.rs | 2 +- boa/src/syntax/lexer/number.rs | 192 +++++++++++++++++---------------- boa/src/syntax/lexer/tests.rs | 39 ++++++- 4 files changed, 177 insertions(+), 101 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 51b16fca45a..0775be968a8 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -73,11 +73,25 @@ where }) } + /// Applies the predicate to the next character and returns the result. + /// Returns false if there is no next character. + /// + /// The buffer is not incremented. + #[inline] + pub(super) fn next_is_pred(&mut self, pred: &F) -> io::Result + where + F: Fn(char) -> bool, + { + Ok(match self.peek() { + None => false, + Some(Ok(peek)) => pred(*peek), + Some(Err(e)) => todo!(), + }) + } + /// Fills the buffer with all characters until the stop character is found. /// /// Note: It will not add the stop character to the buffer. - /// - /// Returns syntax pub(super) fn take_until(&mut self, stop: char, buf: &mut String) -> io::Result<()> { loop { if self.next_is(stop)? { @@ -101,6 +115,33 @@ where } } + /// Fills the buffer with characters until the first character (x) for which the predicate (pred) is false + /// (or the next character is none). + /// + /// Note that all characters up until x are added to the buffer including the character right before. + pub(super) fn take_until_pred(&mut self, buf: &mut String, pred: &F) -> io::Result<()> + where + F: Fn(char) -> bool, + { + loop { + if !self.next_is_pred(pred)? { + return Ok(()); + } else { + match self.next() { + None => { + unreachable!(); + } + Some(Err(e)) => { + return Err(e); + } + Some(Ok(ch)) => { + buf.push(ch); + } + } + } + } + } + /// Retrieves the given number of characters and adds them to the buffer. pub(super) fn _take(&mut self, _count: usize, _buf: &mut String) -> io::Result<()> { unimplemented!() diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 042488595c4..a9688eebde4 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -5,7 +5,7 @@ pub enum Error { IO(io::Error), Syntax(Box), StrictMode(Box), // Not 100% decided on this name. - // Reverted(String), + // Reverted(String), } impl From for Error { diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 8d7289f667f..494d1e7232d 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -168,109 +168,111 @@ impl Tokenizer for NumberLiteral { } } - if cursor.next_is('n')? { - // DecimalBigIntegerLiteral - kind = kind.to_bigint(); - } + todo!("Rest of number literal lexing"); - if let NumericKind::Integer(10) = kind { - 'digitloop: while let Some(ch) = cursor.peek() { - match ch { - Err(_e) => { - // TODO - } - Ok('.') => loop { - kind = NumericKind::Rational; - match cursor.next() { - None => { - // Finished - break; - } + // match cursor.peek() { + // Some(Ok('n')) => { + // // DecimalBigIntegerLiteral + // kind = kind.to_bigint(); + // } + // } - Some(Err(e)) => { - return Err(Error::from(e)); - } + // if let NumericKind::Integer(10) = kind { + // 'digitloop: while let Some(ch) = cursor.peek() { + // match ch { + // Err(_e) => { + // // TODO + // } + // Ok('.') => loop { + // kind = NumericKind::Rational; + // match cursor.next() { + // None => { + // // Finished + // break; + // } - Some(Ok(c)) => { - buf.push(c); - } - } + // Some(Err(e)) => { + // return Err(Error::from(e)); + // } - match cursor.peek() { - None => { - break; - } - Some(Err(_e)) => { - // TODO - } - Some(Ok('e')) | Some(Ok('E')) => { - // TODO scientific notation. + // Some(Ok(c)) => { + // buf.push(c); + // } + // } - unimplemented!(); + // match cursor.peek() { + // None => { + // break; + // } + // Some(Err(_e)) => { + // // TODO + // } + // Some(Ok('e')) | Some(Ok('E')) => { + // cursor.next(); // Consume the ExponentIndicator. - // match self - // .preview_multiple_next(2) - // .unwrap_or_default() - // .to_digit(10) - // { - // Some(0..=9) | None => { - // buf.push(self.next()); - // } - // _ => { - // break 'digitloop; - // } - // } - } - Some(Ok(cx)) if !cx.is_digit(10) => { - break 'digitloop; - } - _ => {} - } - }, - Ok('e') | Ok('E') => { - // TODO scientific notation. - unimplemented!(); + // match self + // .preview_multiple_next(2) + // .unwrap_or_default() + // .to_digit(10) + // { + // Some(0..=9) | None => { + // buf.push(self.next()); + // } + // _ => { + // break 'digitloop; + // } + // } + // } + // Some(Ok(cx)) if !cx.is_digit(10) => { + // break 'digitloop; + // } + // _ => {} + // } + // }, + // Ok('e') | Ok('E') => { + // // TODO scientific notation. + // unimplemented!(); - // kind = NumericKind::Rational; - // match self - // .preview_multiple_next(2) - // .unwrap_or_default() - // .to_digit(10) - // { - // Some(0..=9) | None => { - // buf.push(self.next()); - // } - // _ => { - // break; - // } - // } - // buf.push(self.next()); - } - Ok('+') | Ok('-') => { - break; - } - Ok(cx) if cx.is_digit(10) => { - // cursor.next(); - match cursor.next() { - None => { - // Finished - break; - } + // // kind = NumericKind::Rational; + // // match self + // // .preview_multiple_next(2) + // // .unwrap_or_default() + // // .to_digit(10) + // // { + // // Some(0..=9) | None => { + // // buf.push(self.next()); + // // } + // // _ => { + // // break; + // // } + // // } + // // buf.push(self.next()); + // } + // Ok('+') | Ok('-') => { + // break; + // } + // Ok(cx) if cx.is_digit(10) => { + // // cursor.next(); + // match cursor.next() { + // None => { + // // Finished + // break; + // } - Some(Err(e)) => { - return Err(Error::from(e)); - } + // Some(Err(e)) => { + // return Err(Error::from(e)); + // } - Some(Ok(c)) => { - buf.push(c); - } - } - // buf.push(*cx); - } - Ok(_) => break, - } - } - } + // Some(Ok(c)) => { + // buf.push(c); + // } + // } + // // buf.push(*cx); + // } + // Ok(_) => break, + // } + // } + // } // TODO //self.check_after_numeric_literal()?; @@ -311,4 +313,4 @@ impl Tokenizer for NumberLiteral { Span::new(start_pos, cursor.pos()), )) } -} \ No newline at end of file +} diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index a980d8afe48..bef673ad8b5 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -313,9 +313,7 @@ fn check_nan() { fn single_int() { let mut lexer = Lexer::new(&b"52"[0..]); - let expected = [ - TokenKind::numeric_literal(52), - ]; + let expected = [TokenKind::numeric_literal(52)]; expect_tokens(&mut lexer, &expected); } @@ -502,3 +500,38 @@ fn addition_no_spaces_e_number() { expect_tokens(&mut lexer, &expected); } + +#[test] +fn take_until_pred_simple() { + let mut cur = Cursor::new(&b"abcdefghijk"[0..]); + + let mut buf: String = String::new(); + + cur.take_until_pred(&mut buf, &|c| c == 'a' || c == 'b' || c == 'c') + .unwrap(); + + assert_eq!(buf, "abc"); +} + +#[test] +fn take_until_pred_immediate_stop() { + let mut cur = Cursor::new(&b"abcdefghijk"[0..]); + + let mut buf: String = String::new(); + + cur.take_until_pred(&mut buf, &|c| c == 'd').unwrap(); + + assert_eq!(buf, ""); +} + +#[test] +fn take_until_pred_entire_str() { + let mut cur = Cursor::new(&b"abcdefghijk"[0..]); + + let mut buf: String = String::new(); + + cur.take_until_pred(&mut buf, &|c| c.is_alphabetic()) + .unwrap(); + + assert_eq!(buf, "abcdefghijk"); +} From 0bde651535bce28d9b49b7912d5b980768fd58d5 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Jun 2020 19:55:34 +0100 Subject: [PATCH 031/289] Almost finished rework of numerical lexer --- boa/src/syntax/lexer/number.rs | 66 ++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 494d1e7232d..4a5134347a5 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -152,30 +152,58 @@ impl Tokenizer for NumberLiteral { } } - while let Some(ch) = cursor.peek() { - match ch { - Err(_e) => { - // TODO, handle. - } - Ok(c) if c.is_digit(kind.base()) => { - let s = cursor.next().unwrap().unwrap(); - buf.push(s); + // Consume digits until a non-digit character is encountered or all the characters are consumed. + cursor.take_until_pred(buf, |c| c.is_digit(kind.base())); + + // The non-digit character could be: + // 'n' To indicate a BigIntLiteralSuffix. + // '.' To indicate a decimal seperator. + // 'e' | 'E' To indicate an ExponentPart. + match cursor.peek() { + Some(Ok('n')) => { + // DecimalBigIntegerLiteral + // Lexing finished. + kind = kind.to_bigint(); + } + Some(Ok('.')) => { + if kind.base() != 10 { + todo!("Non base 10 numbers with decimal seperators"); } - _ => { - // A non-number symbol detected, this might be a dot or similar. - break; + + // Consume digits until a non-digit character is encountered or all the characters are consumed. + cursor.take_until_pred(buf, |c| c.is_digit(kind.base())); + + // The non-digit character at this point must be an 'e' or 'E' to indicate an Exponent Part. + // Another '.' or 'n' is not allowed. + match cursor.peek() { + Some(Ok('n')) => { + Err(Error::syntax("Found 'n' after non-integer number")); + } + Some(Ok('.')) => { + Err(Error::syntax("Found second '.' within decimal number")); + } + Some (Err(e)) => { + todo!(); + } + None => { + // Finished lexing. + kind = NumericKind::Rational; + } } + } - } + Some(Ok('e')) | Some(Ok('E')) => { - todo!("Rest of number literal lexing"); + } - // match cursor.peek() { - // Some(Ok('n')) => { - // // DecimalBigIntegerLiteral - // kind = kind.to_bigint(); - // } - // } + Some(Err(e)) => { + todo!(); + } + + None => { + // Indicates lexing finished. + } + } // if let NumericKind::Integer(10) = kind { // 'digitloop: while let Some(ch) = cursor.peek() { From 6e26812acbaf11f3d7534efacc99b9aefc38c03d Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 15 Jun 2020 22:43:27 +0100 Subject: [PATCH 032/289] Number lexing --- boa/src/syntax/lexer/number.rs | 184 +++++++++++++-------------------- 1 file changed, 72 insertions(+), 112 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 4a5134347a5..9c250f854c1 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -58,6 +58,54 @@ impl NumericKind { } } +fn take_signed_integer( + buf: &mut String, + cursor: &mut Cursor, + kind: &NumericKind, +) -> Result<(), Error> +where + R: Read, +{ + // The next part must be SignedInteger. + // This is optionally a '+' or '-' followed by 1 or more DecimalDigits. + match cursor.next() { + Some(Ok('+')) => { + buf.push('+'); + if !cursor.next_is_pred(&|c: char| c.is_digit(kind.base()))? { + // A digit must follow the + or - symbol. + return Err(Error::syntax("No digit found after + symbol")); + } + } + Some(Ok('-')) => { + buf.push('-'); + if !cursor.next_is_pred(&|c: char| c.is_digit(kind.base()))? { + // A digit must follow the + or - symbol. + return Err(Error::syntax("No digit found after - symbol")); + } + } + Some(Ok(c)) if c.is_digit(kind.base()) => { + buf.push(c); + } + Some(Ok(c)) => { + return Err(Error::syntax(format!( + "When lexing exponential value found unexpected char: '{}'", + c + ))); + } + Some(Err(e)) => { + return Err(e.into()); + } + None => { + return Err(Error::syntax("No exponential value found")); + } + } + + // Consume the decimal digits. + cursor.take_until_pred(buf, &|c: char| c.is_digit(kind.base()))?; + + Ok(()) +} + impl Tokenizer for NumberLiteral { fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where @@ -81,7 +129,7 @@ impl Tokenizer for NumberLiteral { )); } Some(Err(e)) => { - todo!(); + // todo!(); } Some(Ok('x')) | Some(Ok('X')) => { // Remove the initial '0' from buffer. @@ -153,7 +201,7 @@ impl Tokenizer for NumberLiteral { } // Consume digits until a non-digit character is encountered or all the characters are consumed. - cursor.take_until_pred(buf, |c| c.is_digit(kind.base())); + cursor.take_until_pred(&mut buf, &|c: char| c.is_digit(kind.base()))?; // The non-digit character could be: // 'n' To indicate a BigIntLiteralSuffix. @@ -166,144 +214,56 @@ impl Tokenizer for NumberLiteral { kind = kind.to_bigint(); } Some(Ok('.')) => { + kind = NumericKind::Rational; if kind.base() != 10 { todo!("Non base 10 numbers with decimal seperators"); } // Consume digits until a non-digit character is encountered or all the characters are consumed. - cursor.take_until_pred(buf, |c| c.is_digit(kind.base())); + cursor.take_until_pred(&mut buf, &|c: char| c.is_digit(kind.base()))?; - // The non-digit character at this point must be an 'e' or 'E' to indicate an Exponent Part. + // The non-digit character at this point must be an 'e' or 'E' to indicate an Exponent Part. // Another '.' or 'n' is not allowed. match cursor.peek() { Some(Ok('n')) => { - Err(Error::syntax("Found 'n' after non-integer number")); + return Err(Error::syntax( + "Found BigIntLiteralSuffix after non-integer number", + )); } Some(Ok('.')) => { - Err(Error::syntax("Found second '.' within decimal number")); + return Err(Error::syntax("Found second '.' within decimal number")); } - Some (Err(e)) => { - todo!(); + Some(Ok('e')) | Some(Ok('E')) => { + // Consume the ExponentIndicator. + cursor.next(); + + take_signed_integer(&mut buf, cursor, &kind)?; + } + Some(Err(e)) => { + // todo!(); } - None => { + Some(Ok(_)) | None => { // Finished lexing. kind = NumericKind::Rational; } } - } Some(Ok('e')) | Some(Ok('E')) => { + // Consume the ExponentIndicator. + cursor.next(); + take_signed_integer(&mut buf, cursor, &kind)?; } - Some(Err(e)) => { - todo!(); + // todo!(); } - None => { + Some(Ok(_)) | None => { // Indicates lexing finished. } } - // if let NumericKind::Integer(10) = kind { - // 'digitloop: while let Some(ch) = cursor.peek() { - // match ch { - // Err(_e) => { - // // TODO - // } - // Ok('.') => loop { - // kind = NumericKind::Rational; - // match cursor.next() { - // None => { - // // Finished - // break; - // } - - // Some(Err(e)) => { - // return Err(Error::from(e)); - // } - - // Some(Ok(c)) => { - // buf.push(c); - // } - // } - - // match cursor.peek() { - // None => { - // break; - // } - // Some(Err(_e)) => { - // // TODO - // } - // Some(Ok('e')) | Some(Ok('E')) => { - // cursor.next(); // Consume the ExponentIndicator. - - // match self - // .preview_multiple_next(2) - // .unwrap_or_default() - // .to_digit(10) - // { - // Some(0..=9) | None => { - // buf.push(self.next()); - // } - // _ => { - // break 'digitloop; - // } - // } - // } - // Some(Ok(cx)) if !cx.is_digit(10) => { - // break 'digitloop; - // } - // _ => {} - // } - // }, - // Ok('e') | Ok('E') => { - // // TODO scientific notation. - // unimplemented!(); - - // // kind = NumericKind::Rational; - // // match self - // // .preview_multiple_next(2) - // // .unwrap_or_default() - // // .to_digit(10) - // // { - // // Some(0..=9) | None => { - // // buf.push(self.next()); - // // } - // // _ => { - // // break; - // // } - // // } - // // buf.push(self.next()); - // } - // Ok('+') | Ok('-') => { - // break; - // } - // Ok(cx) if cx.is_digit(10) => { - // // cursor.next(); - // match cursor.next() { - // None => { - // // Finished - // break; - // } - - // Some(Err(e)) => { - // return Err(Error::from(e)); - // } - - // Some(Ok(c)) => { - // buf.push(c); - // } - // } - // // buf.push(*cx); - // } - // Ok(_) => break, - // } - // } - // } - - // TODO - //self.check_after_numeric_literal()?; + // self.check_after_numeric_literal()?; let num = match kind { NumericKind::BigInt(base) => { From a4ccbd64671b04c4fd4fda7abf953f337aa0dfe2 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 15 Jun 2020 23:30:35 +0100 Subject: [PATCH 033/289] Fixed bug relating to decimal seperator not being consumed --- boa/src/syntax/lexer/cursor.rs | 3 +- boa/src/syntax/lexer/number.rs | 65 +++++++++++++++++++--------------- 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 0775be968a8..5bd8cec0bf8 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -129,7 +129,8 @@ where } else { match self.next() { None => { - unreachable!(); + unimplemented!(); + // unreachable!(); } Some(Err(e)) => { return Err(e); diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 9c250f854c1..d3318e92131 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -214,9 +214,14 @@ impl Tokenizer for NumberLiteral { kind = kind.to_bigint(); } Some(Ok('.')) => { + // Consume the . + cursor.next(); + kind = NumericKind::Rational; if kind.base() != 10 { - todo!("Non base 10 numbers with decimal seperators"); + return Err(Error::syntax( + "Attempted to lex non-base 10 number with decimal seperator", + )); } // Consume digits until a non-digit character is encountered or all the characters are consumed. @@ -231,7 +236,7 @@ impl Tokenizer for NumberLiteral { )); } Some(Ok('.')) => { - return Err(Error::syntax("Found second '.' within decimal number")); + return Err(Error::syntax("Found second . within decimal number")); } Some(Ok('e')) | Some(Ok('E')) => { // Consume the ExponentIndicator. @@ -239,13 +244,14 @@ impl Tokenizer for NumberLiteral { take_signed_integer(&mut buf, cursor, &kind)?; } - Some(Err(e)) => { + Some(Err(_e)) => { // todo!(); } Some(Ok(_)) | None => { // Finished lexing. kind = NumericKind::Rational; } + _ => {} } } Some(Ok('e')) | Some(Ok('E')) => { @@ -261,40 +267,43 @@ impl Tokenizer for NumberLiteral { Some(Ok(_)) | None => { // Indicates lexing finished. } + + _ => {} } + // unimplemented!(); + // self.check_after_numeric_literal()?; let num = match kind { - NumericKind::BigInt(base) => { - Numeric::BigInt( - BigInt::from_string_radix(&buf, base as u32).expect("Could not convert to BigInt") - ) - } - NumericKind::Rational /* base: 10 */ => { - Numeric::Rational( - f64::from_str(&buf) - .map_err(|_| Error::syntax("Could not convert value to f64"))?, + NumericKind::BigInt(base) => { + Numeric::BigInt( + BigInt::from_string_radix(&buf, base as u32).expect("Could not convert to BigInt") ) - } - NumericKind::Integer(base) => { - if let Ok(num) = i32::from_str_radix(&buf, base as u32) { - Numeric::Integer( - num - ) - } else { - let b = f64::from(base); - let mut result = 0.0_f64; - for c in buf.chars() { - let digit = f64::from(c.to_digit(base as u32).unwrap()); - result = result * b + digit; - } - - Numeric::Rational(result) + } + NumericKind::Rational /* base: 10 */ => { + Numeric::Rational( + f64::from_str(&buf) + .map_err(|_| Error::syntax("Could not convert value to f64"))?, + ) + } + NumericKind::Integer(base) => { + if let Ok(num) = i32::from_str_radix(&buf, base as u32) { + // unimplemented!(); + // Numeric::Integer(0) + Numeric::Integer(num) + } else { + let b = f64::from(base); + let mut result = 0.0_f64; + for c in buf.chars() { + let digit = f64::from(c.to_digit(base as u32).unwrap()); + result = result * b + digit; } + Numeric::Rational(result) } - }; + } + }; Ok(Token::new( TokenKind::NumericLiteral(num), From 22479e655347f9e09faa949b2c3827326e60014e Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 15 Jun 2020 23:51:13 +0100 Subject: [PATCH 034/289] Number lexing, handing of exponentials added, decimal numbers not working properly --- boa/src/syntax/lexer/number.rs | 52 +++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index d3318e92131..260c4d6bcf0 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -203,6 +203,8 @@ impl Tokenizer for NumberLiteral { // Consume digits until a non-digit character is encountered or all the characters are consumed. cursor.take_until_pred(&mut buf, &|c: char| c.is_digit(kind.base()))?; + let mut exp_str = String::new(); + // The non-digit character could be: // 'n' To indicate a BigIntLiteralSuffix. // '.' To indicate a decimal seperator. @@ -242,7 +244,7 @@ impl Tokenizer for NumberLiteral { // Consume the ExponentIndicator. cursor.next(); - take_signed_integer(&mut buf, cursor, &kind)?; + take_signed_integer(&mut exp_str, cursor, &kind)?; } Some(Err(_e)) => { // todo!(); @@ -258,17 +260,17 @@ impl Tokenizer for NumberLiteral { // Consume the ExponentIndicator. cursor.next(); - take_signed_integer(&mut buf, cursor, &kind)?; + // buf.push('e'); + + take_signed_integer(&mut exp_str, cursor, &kind)?; } - Some(Err(e)) => { + Some(Err(_e)) => { // todo!(); } Some(Ok(_)) | None => { // Indicates lexing finished. } - - _ => {} } // unimplemented!(); @@ -282,16 +284,30 @@ impl Tokenizer for NumberLiteral { ) } NumericKind::Rational /* base: 10 */ => { - Numeric::Rational( - f64::from_str(&buf) - .map_err(|_| Error::syntax("Could not convert value to f64"))?, - ) + let r = f64::from_str(&buf).map_err(|_| Error::syntax("Could not convert value to f64"))?; + if exp_str == "" { + Numeric::Rational( + r + ) + } else { + let n = f64::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; + + Numeric::Rational( + r * f64::powf(10.0, n) + ) + } } NumericKind::Integer(base) => { if let Ok(num) = i32::from_str_radix(&buf, base as u32) { - // unimplemented!(); - // Numeric::Integer(0) - Numeric::Integer(num) + if exp_str == "" { + Numeric::Integer(num) + } else { + let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; + + Numeric::Integer( + num * i32::pow(10, n as u32) + ) + } } else { let b = f64::from(base); let mut result = 0.0_f64; @@ -300,7 +316,17 @@ impl Tokenizer for NumberLiteral { result = result * b + digit; } - Numeric::Rational(result) + if exp_str == "" { + Numeric::Rational( + result + ) + } else { + let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; + + Numeric::Rational( + result * f64::powi(10.0, n) + ) + } } } }; From 619b3566721706cb13ed109003a41f2f4a62503c Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 15 Jun 2020 23:52:19 +0100 Subject: [PATCH 035/289] Added negative number to numbers lexer test --- boa/src/syntax/lexer/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index bef673ad8b5..210f3ac9540 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -321,7 +321,7 @@ fn single_int() { #[test] fn numbers() { let mut lexer = Lexer::new( - "1 2 0x34 056 7.89 42. 5e3 5e+3 5e-3 0b10 0O123 0999 1.0e1 1.0e-1 1.0E1 1E1 0.0 0.12" + "1 2 0x34 056 7.89 42. 5e3 5e+3 5e-3 0b10 0O123 0999 1.0e1 1.0e-1 1.0E1 1E1 0.0 0.12 -32" .as_bytes(), ); @@ -344,6 +344,7 @@ fn numbers() { TokenKind::numeric_literal(10.0), TokenKind::numeric_literal(0.0), TokenKind::numeric_literal(0.12), + TokenKind::numeric_literal(-32), ]; expect_tokens(&mut lexer, &expected); From 439984c4e781bd650600fd05b48592388ca73dd8 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 10:47:18 +0100 Subject: [PATCH 036/289] Fixed decimal number lexing --- boa/src/syntax/lexer/number.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 260c4d6bcf0..59a049c358e 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -218,6 +218,7 @@ impl Tokenizer for NumberLiteral { Some(Ok('.')) => { // Consume the . cursor.next(); + buf.push('.'); kind = NumericKind::Rational; if kind.base() != 10 { From 37e2e3ca14cb8cac86c6fb07c189242d29072ecc Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 11:56:33 +0100 Subject: [PATCH 037/289] Progress on fixing bigInt lexign --- boa/src/syntax/lexer/number.rs | 19 +++++++++++++++---- boa/src/syntax/lexer/tests.rs | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 59a049c358e..14b689d1ad1 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -213,6 +213,10 @@ impl Tokenizer for NumberLiteral { Some(Ok('n')) => { // DecimalBigIntegerLiteral // Lexing finished. + + // Consume the n + cursor.next(); + kind = kind.to_bigint(); } Some(Ok('.')) => { @@ -234,12 +238,19 @@ impl Tokenizer for NumberLiteral { // Another '.' or 'n' is not allowed. match cursor.peek() { Some(Ok('n')) => { - return Err(Error::syntax( - "Found BigIntLiteralSuffix after non-integer number", - )); + // Found BigIntLiteralSuffix after non-integer number + + // Finish lexing number. + + // return Err(Error::syntax( + // "Found BigIntLiteralSuffix after non-integer number", + // )); } Some(Ok('.')) => { - return Err(Error::syntax("Found second . within decimal number")); + // Found second . within decimal number + // Finish lexing number. + + // return Err(Error::syntax("Found second . within decimal number")); } Some(Ok('e')) | Some(Ok('E')) => { // Consume the ExponentIndicator. diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 210f3ac9540..fe4a6df13a4 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -332,8 +332,8 @@ fn numbers() { TokenKind::numeric_literal(46), TokenKind::numeric_literal(7.89), TokenKind::numeric_literal(42.0), - TokenKind::numeric_literal(5000.0), - TokenKind::numeric_literal(5000.0), + TokenKind::numeric_literal(5000), + TokenKind::numeric_literal(5000), TokenKind::numeric_literal(0.005), TokenKind::numeric_literal(2), TokenKind::numeric_literal(83), From 50694f4d3727565a22aab0aca7997d6da2df5d88 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 14:44:24 +0100 Subject: [PATCH 038/289] Reverted returning an Integer for exponential notation --- boa/src/syntax/lexer/number.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 14b689d1ad1..ba7d4185fac 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -316,8 +316,8 @@ impl Tokenizer for NumberLiteral { } else { let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; - Numeric::Integer( - num * i32::pow(10, n as u32) + Numeric::Rational( + (num as f64) * f64::powi(10.0, n) ) } } else { From 07c5068fa38e6121d715c9899dd940de55f911b9 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 15:00:46 +0100 Subject: [PATCH 039/289] Fixed hexadeecimal edge case --- boa/src/syntax/lexer/number.rs | 81 +++++++++++++++++----------------- boa/src/syntax/lexer/tests.rs | 7 +-- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index ba7d4185fac..e79823801e8 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -221,51 +221,50 @@ impl Tokenizer for NumberLiteral { } Some(Ok('.')) => { // Consume the . - cursor.next(); - buf.push('.'); - - kind = NumericKind::Rational; - if kind.base() != 10 { - return Err(Error::syntax( - "Attempted to lex non-base 10 number with decimal seperator", - )); - } - - // Consume digits until a non-digit character is encountered or all the characters are consumed. - cursor.take_until_pred(&mut buf, &|c: char| c.is_digit(kind.base()))?; - - // The non-digit character at this point must be an 'e' or 'E' to indicate an Exponent Part. - // Another '.' or 'n' is not allowed. - match cursor.peek() { - Some(Ok('n')) => { - // Found BigIntLiteralSuffix after non-integer number - - // Finish lexing number. + + if kind.base() == 10 { + // Only base 10 numbers can have a decimal seperator. + // Number literal lexing finished if a . is found for a number in a different base. - // return Err(Error::syntax( - // "Found BigIntLiteralSuffix after non-integer number", - // )); - } - Some(Ok('.')) => { - // Found second . within decimal number - // Finish lexing number. + cursor.next(); + buf.push('.'); + kind = NumericKind::Rational; + + // Consume digits until a non-digit character is encountered or all the characters are consumed. + cursor.take_until_pred(&mut buf, &|c: char| c.is_digit(kind.base()))?; + + // The non-digit character at this point must be an 'e' or 'E' to indicate an Exponent Part. + // Another '.' or 'n' is not allowed. + match cursor.peek() { + Some(Ok('n')) => { + // Found BigIntLiteralSuffix after non-integer number + + // Finish lexing number. + + // return Err(Error::syntax( + // "Found BigIntLiteralSuffix after non-integer number", + // )); + } + Some(Ok('.')) => { + // Found second . within decimal number + // Finish lexing number. - // return Err(Error::syntax("Found second . within decimal number")); - } - Some(Ok('e')) | Some(Ok('E')) => { - // Consume the ExponentIndicator. - cursor.next(); + // return Err(Error::syntax("Found second . within decimal number")); + } + Some(Ok('e')) | Some(Ok('E')) => { + // Consume the ExponentIndicator. + cursor.next(); - take_signed_integer(&mut exp_str, cursor, &kind)?; - } - Some(Err(_e)) => { - // todo!(); - } - Some(Ok(_)) | None => { - // Finished lexing. - kind = NumericKind::Rational; + take_signed_integer(&mut exp_str, cursor, &kind)?; + } + Some(Err(_e)) => { + // todo!(); + } + Some(Ok(_)) | None => { + // Finished lexing. + kind = NumericKind::Rational; + } } - _ => {} } } Some(Ok('e')) | Some(Ok('E')) => { diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index fe4a6df13a4..7db5d218f24 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -332,8 +332,8 @@ fn numbers() { TokenKind::numeric_literal(46), TokenKind::numeric_literal(7.89), TokenKind::numeric_literal(42.0), - TokenKind::numeric_literal(5000), - TokenKind::numeric_literal(5000), + TokenKind::numeric_literal(5000.0), + TokenKind::numeric_literal(5000.0), TokenKind::numeric_literal(0.005), TokenKind::numeric_literal(2), TokenKind::numeric_literal(83), @@ -344,7 +344,8 @@ fn numbers() { TokenKind::numeric_literal(10.0), TokenKind::numeric_literal(0.0), TokenKind::numeric_literal(0.12), - TokenKind::numeric_literal(-32), + TokenKind::Punctuator(Punctuator::Sub), + TokenKind::numeric_literal(32), ]; expect_tokens(&mut lexer, &expected); From 66318e0fff5dc7a380d75a0a59d8ee3509701c39 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 15:11:09 +0100 Subject: [PATCH 040/289] Added check_after_numerical_literal --- boa/src/syntax/lexer/number.rs | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index e79823801e8..ad2bfc303ae 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -106,6 +106,25 @@ where Ok(()) } +/// Utility function for checking the NumericLiteral is not followed by an `IdentifierStart` or `DecimalDigit` character. +/// +/// More information: +/// - [ECMAScript Specification][spec] +/// +/// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals +fn check_after_numeric_literal(cursor: &mut Cursor) -> Result<(), Error> +where + R: Read, +{ + if cursor.next_is_pred(&|ch: char| { + ch.is_ascii_alphabetic() || ch == '$' || ch == '_' || ch.is_ascii_digit() + })? { + Err(Error::syntax("NumericLiteral token must not be followed by IdentifierStart nor DecimalDigit characters")) + } else { + Ok(()) + } +} + impl Tokenizer for NumberLiteral { fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where @@ -221,9 +240,9 @@ impl Tokenizer for NumberLiteral { } Some(Ok('.')) => { // Consume the . - + if kind.base() == 10 { - // Only base 10 numbers can have a decimal seperator. + // Only base 10 numbers can have a decimal seperator. // Number literal lexing finished if a . is found for a number in a different base. cursor.next(); @@ -238,7 +257,7 @@ impl Tokenizer for NumberLiteral { match cursor.peek() { Some(Ok('n')) => { // Found BigIntLiteralSuffix after non-integer number - + // Finish lexing number. // return Err(Error::syntax( @@ -284,9 +303,7 @@ impl Tokenizer for NumberLiteral { } } - // unimplemented!(); - - // self.check_after_numeric_literal()?; + check_after_numeric_literal(cursor)?; let num = match kind { NumericKind::BigInt(base) => { From b42e6ba21af384a3ff8910366730c0a0d4066aea Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 15:30:36 +0100 Subject: [PATCH 041/289] Added tests related to illegal code points following numerical literal --- boa/src/syntax/lexer/tests.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 7db5d218f24..c2c2bfa3cac 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -537,3 +537,27 @@ fn take_until_pred_entire_str() { assert_eq!(buf, "abcdefghijk"); } + +#[test] +fn illegal_following_numeric_literal() { + // Checks as per https://tc39.es/ecma262/#sec-literals-numeric-literals that a NumericLiteral cannot + // be immediately followed by an IdentifierStart or DecimalDigit. + + // Decimal Digit + let mut lexer = Lexer::new(&b"11.6n3"[0..]); + assert!(lexer.next().unwrap().err().is_some(), "DecimalDigit following NumericLiteral not rejected as expected"); + + // Identifier Start + let mut lexer = Lexer::new(&b"17.4$"[0..]); + assert!(lexer.next().unwrap().err().is_some(), "IdentifierStart '$' following NumericLiteral not rejected as expected"); + + let mut lexer = Lexer::new(&b"17.4_"[0..]); + assert!(lexer.next().unwrap().err().is_some(), "IdentifierStart '_' following NumericLiteral not rejected as expected"); +} + +#[test] +fn illegal_code_point_following_numeric_literal() { + let mut lexer = Lexer::new(&b"17.4\\u{0009}"[0..]); + assert!(lexer.next().unwrap().err().is_some(), "IdentifierStart '\\u{0009}' following NumericLiteral not rejected as expected"); + +} \ No newline at end of file From d826ce79ec81ed9f6c824cb6ccb32650fb3a6440 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 18:37:32 +0100 Subject: [PATCH 042/289] Goal symbol aware '/' lexing --- boa/src/syntax/lexer/comment.rs | 121 +++++++++++++++----------------- boa/src/syntax/lexer/mod.rs | 60 ++++++++++++++-- boa/src/syntax/lexer/regex.rs | 7 -- boa/src/syntax/lexer/tests.rs | 6 +- 4 files changed, 112 insertions(+), 82 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index c41fd7fbf9f..a9aa208326d 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -1,17 +1,13 @@ -use super::{Cursor, Error, RegexLiteral, Tokenizer}; +use super::{Cursor, Error, Tokenizer}; use crate::syntax::ast::{Position, Span}; use crate::syntax::lexer::{Token, TokenKind}; -use std::io::{ErrorKind, Read}; +use std::io::Read; -macro_rules! comment_match { - () => { - '/' - }; -} +pub(super) struct SingleLineComment; -/// Skips comments. +/// Lexes a single line comment. /// -/// Assumes that the '/' char is already consumed. +/// Assumes that the initial '//' is already consumed. /// /// More information: /// - [ECMAScript reference][spec] @@ -19,77 +15,70 @@ macro_rules! comment_match { /// /// [spec]: /// [mdn]: -pub(super) struct Comment; - -impl Comment { - /// Creates a new comment lexer. - pub(super) fn new() -> Self { - Self {} +impl Tokenizer for SingleLineComment { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + // Skip either to the end of the line or to the end of the input + while let Some(ch) = cursor.next() { + match ch { + Err(e) => { + return Err(Error::IO(e)); + } + Ok('\n') => { + break; + } + _ => {} + } + } + cursor.next_line(); + Ok(Token::new( + TokenKind::Comment, + Span::new(start_pos, cursor.pos()), + )) } } -impl Tokenizer for Comment { +/// Lexes a block (multi-line) comment. +/// +/// Assumes that the initial '/*' is already consumed. +/// +/// More information: +/// - [ECMAScript reference][spec] +/// - [MDN documentation][mdn] +/// +/// [spec]: +/// [mdn]: +pub(super) struct BlockComment; +impl Tokenizer for BlockComment { fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where R: Read, { - match cursor.peek() { - None => Err(Error::syntax("Expecting Token /,*,= or regex")), - Some(Err(_)) => Err(Error::from(std::io::Error::new( - ErrorKind::Interrupted, - "Failed to peek next character", - ))), - Some(Ok(ch)) => { + loop { + if let Some(ch) = cursor.next() { match ch { - '/' => { - // Skip either to the end of the line or to the end of the input - while let Some(ch) = cursor.next() { - match ch { - Err(e) => { - return Err(Error::IO(e)); - } - Ok('\n') => { - break; - } - _ => {} - } - } + Err(e) => { + return Err(Error::IO(e)); + } + Ok('\n') => { cursor.next_line(); - Ok(Token::new( - TokenKind::Comment, - Span::new(start_pos, cursor.pos()), - )) } - // block comment - '*' => { - loop { - if let Some(ch) = cursor.next() { - match ch { - Err(e) => { - return Err(Error::IO(e)); - } - Ok('\n') => { - cursor.next_line(); - } - Ok('*') => { - if cursor.next_is('/')? { - break; - } - } - _ => {} - } - } else { - return Err(Error::syntax("unterminated multiline comment")); - } + Ok('*') => { + if cursor.next_is('/')? { + break; } - Ok(Token::new( - TokenKind::Comment, - Span::new(start_pos, cursor.pos()), - )) } - _ => RegexLiteral::new().lex(cursor, start_pos), + _ => {} } + } else { + return Err(Error::syntax("unterminated multiline comment")); } } + Ok(Token::new( + TokenKind::Comment, + Span::new(start_pos, cursor.pos()), + )) } } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 1a1857c2d12..086b226a67f 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -33,7 +33,7 @@ mod tests; pub use self::error::Error; use self::{ - comment::Comment, cursor::Cursor, identifier::Identifier, number::NumberLiteral, + comment::{SingleLineComment, BlockComment}, cursor::Cursor, identifier::Identifier, number::NumberLiteral, operator::Operator, regex::RegexLiteral, spread::SpreadLiteral, string::StringLiteral, template::TemplateLiteral, }; @@ -74,9 +74,13 @@ impl Lexer { } /// Sets the goal symbol for the lexer. - pub(crate) fn _set_goal(&mut self, elm: InputElement) { + pub(crate) fn set_goal(&mut self, elm: InputElement) { self.goal_symbol = elm; } + + pub(crate) fn get_goal(&self) -> InputElement { + self.goal_symbol + } } impl Lexer @@ -91,6 +95,48 @@ where goal_symbol: Default::default(), } } + + // Handles lexing of a token starting '/' with the '/' already being consumed. + // This could be a divide symbol or the start of a regex. + // + // A '/' symbol can always be a comment but if as tested above it is not then + // that means it could be multiple different tokens depending on the input token. + // + // As per https://tc39.es/ecma262/#sec-ecmascript-language-lexical-grammar + fn lex_slash_token(&mut self, start: Position) -> Result { + if let Some(c) = self.cursor.peek() { + match c { + Err(e) => { + todo!(); + } + Ok('/') => { + self.cursor.next(); // Consume the + SingleLineComment.lex(&mut self.cursor, start) + } + Ok('*') => { + self.cursor.next(); + BlockComment.lex(&mut self.cursor, start) + } + Ok(_) => { + match self.get_goal() { + InputElement::Div | InputElement::TemplateTail => { + // Only div punctuator allowed, regex not. + Ok(Token::new( + Punctuator::Div.into(), + Span::new(start, self.cursor.pos()), + )) + } + InputElement::RegExp | InputElement::RegExpOrTemplateTail => { + // Can be a regular expression. + RegexLiteral.lex(&mut self.cursor, start) + } + } + } + } + } else { + Err(Error::syntax("Expecting Token /,*,= or regex")) + } + } } /// ECMAScript goal symbols. @@ -99,14 +145,14 @@ where #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum InputElement { Div, - _RegExp, - _RegExpOrTemplateTail, - _TemplateTail, + RegExp, + RegExpOrTemplateTail, + TemplateTail, } impl Default for InputElement { fn default() -> Self { - InputElement::Div + InputElement::RegExpOrTemplateTail // Decided on InputElementDiv as default for now based on documentation from // } @@ -189,7 +235,7 @@ where Punctuator::Question.into(), Span::new(start, self.cursor.pos()), )), - comment_match!() => Comment::new().lex(&mut self.cursor, start), + '/' => self.lex_slash_token(start), '*' | '+' | '-' | '%' | '|' | '&' | '^' | '=' | '<' | '>' | '!' | '~' => { Operator::new(next_chr).lex(&mut self.cursor, start) } diff --git a/boa/src/syntax/lexer/regex.rs b/boa/src/syntax/lexer/regex.rs index b3ec9538a1a..ef519aedc9c 100644 --- a/boa/src/syntax/lexer/regex.rs +++ b/boa/src/syntax/lexer/regex.rs @@ -18,13 +18,6 @@ use std::io::{self, ErrorKind, Read}; #[derive(Debug, Clone, Copy)] pub(super) struct RegexLiteral; -impl RegexLiteral { - /// Creates a new regex literal lexer. - pub(super) fn new() -> Self { - Self {} - } -} - impl Tokenizer for RegexLiteral { fn lex(&mut self, cursor: &mut Cursor, _start_pos: Position) -> Result where diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index c2c2bfa3cac..ab50f39c47a 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -251,7 +251,6 @@ fn check_positions() { } #[test] -#[ignore] fn two_divisions_in_expression() { let s = " return a !== 0 || 1 / a === 1 / b;"; let mut lexer = Lexer::new(s.as_bytes()); @@ -557,7 +556,10 @@ fn illegal_following_numeric_literal() { #[test] fn illegal_code_point_following_numeric_literal() { - let mut lexer = Lexer::new(&b"17.4\\u{0009}"[0..]); + // Checks as per https://tc39.es/ecma262/#sec-literals-numeric-literals that a NumericLiteral cannot + // be immediately followed by an IdentifierStart where the IdentifierStart + + let mut lexer = Lexer::new(&b"17.4\\u{1000}"[0..]); assert!(lexer.next().unwrap().err().is_some(), "IdentifierStart '\\u{0009}' following NumericLiteral not rejected as expected"); } \ No newline at end of file From 308c252141992b42a3700da0a5a8196f40675cfb Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 18:42:44 +0100 Subject: [PATCH 043/289] Fmt --- boa/src/syntax/lexer/cursor.rs | 6 +++--- boa/src/syntax/lexer/mod.rs | 14 ++++++++++---- boa/src/syntax/lexer/tests.rs | 27 +++++++++++++++++++-------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 5bd8cec0bf8..0d079c843fa 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -144,19 +144,19 @@ where } /// Retrieves the given number of characters and adds them to the buffer. - pub(super) fn _take(&mut self, _count: usize, _buf: &mut String) -> io::Result<()> { + pub(super) fn take(&mut self, count: usize, buf: &mut String) -> io::Result<()> { unimplemented!() } /// It will fill the buffer with checked ASCII bytes. - pub(super) fn fill_bytes(&mut self, _buf: &mut [u8]) -> io::Result<()> { + pub(super) fn fill_bytes(&mut self, buf: &mut [u8]) -> io::Result<()> { unimplemented!() } /// Retrieves the next character as an ASCII character. /// /// It will make sure that the next character is an ASCII byte, or return an error otherwise. - pub(super) fn _next_as_byte(&mut self) -> Option> { + pub(super) fn next_as_byte(&mut self) -> Option> { unimplemented!() } } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 086b226a67f..354ecc026a6 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -33,8 +33,14 @@ mod tests; pub use self::error::Error; use self::{ - comment::{SingleLineComment, BlockComment}, cursor::Cursor, identifier::Identifier, number::NumberLiteral, - operator::Operator, regex::RegexLiteral, spread::SpreadLiteral, string::StringLiteral, + comment::{BlockComment, SingleLineComment}, + cursor::Cursor, + identifier::Identifier, + number::NumberLiteral, + operator::Operator, + regex::RegexLiteral, + spread::SpreadLiteral, + string::StringLiteral, template::TemplateLiteral, }; use crate::syntax::ast::{Position, Punctuator, Span}; @@ -101,7 +107,7 @@ where // // A '/' symbol can always be a comment but if as tested above it is not then // that means it could be multiple different tokens depending on the input token. - // + // // As per https://tc39.es/ecma262/#sec-ecmascript-language-lexical-grammar fn lex_slash_token(&mut self, start: Position) -> Result { if let Some(c) = self.cursor.peek() { @@ -109,7 +115,7 @@ where Err(e) => { todo!(); } - Ok('/') => { + Ok('/') => { self.cursor.next(); // Consume the SingleLineComment.lex(&mut self.cursor, start) } diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index ab50f39c47a..770196c5662 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -539,27 +539,38 @@ fn take_until_pred_entire_str() { #[test] fn illegal_following_numeric_literal() { - // Checks as per https://tc39.es/ecma262/#sec-literals-numeric-literals that a NumericLiteral cannot + // Checks as per https://tc39.es/ecma262/#sec-literals-numeric-literals that a NumericLiteral cannot // be immediately followed by an IdentifierStart or DecimalDigit. // Decimal Digit let mut lexer = Lexer::new(&b"11.6n3"[0..]); - assert!(lexer.next().unwrap().err().is_some(), "DecimalDigit following NumericLiteral not rejected as expected"); + assert!( + lexer.next().unwrap().err().is_some(), + "DecimalDigit following NumericLiteral not rejected as expected" + ); // Identifier Start let mut lexer = Lexer::new(&b"17.4$"[0..]); - assert!(lexer.next().unwrap().err().is_some(), "IdentifierStart '$' following NumericLiteral not rejected as expected"); + assert!( + lexer.next().unwrap().err().is_some(), + "IdentifierStart '$' following NumericLiteral not rejected as expected" + ); let mut lexer = Lexer::new(&b"17.4_"[0..]); - assert!(lexer.next().unwrap().err().is_some(), "IdentifierStart '_' following NumericLiteral not rejected as expected"); + assert!( + lexer.next().unwrap().err().is_some(), + "IdentifierStart '_' following NumericLiteral not rejected as expected" + ); } #[test] fn illegal_code_point_following_numeric_literal() { - // Checks as per https://tc39.es/ecma262/#sec-literals-numeric-literals that a NumericLiteral cannot + // Checks as per https://tc39.es/ecma262/#sec-literals-numeric-literals that a NumericLiteral cannot // be immediately followed by an IdentifierStart where the IdentifierStart let mut lexer = Lexer::new(&b"17.4\\u{1000}"[0..]); - assert!(lexer.next().unwrap().err().is_some(), "IdentifierStart '\\u{0009}' following NumericLiteral not rejected as expected"); - -} \ No newline at end of file + assert!( + lexer.next().unwrap().err().is_some(), + "IdentifierStart '\\u{0009}' following NumericLiteral not rejected as expected" + ); +} From ef30f04614506b413885c48cd5dbc41d4f54747e Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 19:05:11 +0100 Subject: [PATCH 044/289] Implemented regex lexing with goal symbols --- boa/src/syntax/lexer/mod.rs | 21 +++++++++++++++----- boa/src/syntax/lexer/regex.rs | 36 +++++++++++------------------------ 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 354ecc026a6..dfc3730a4e6 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -123,14 +123,25 @@ where self.cursor.next(); BlockComment.lex(&mut self.cursor, start) } - Ok(_) => { + Ok(c) => { + let ch = *c; match self.get_goal() { InputElement::Div | InputElement::TemplateTail => { // Only div punctuator allowed, regex not. - Ok(Token::new( - Punctuator::Div.into(), - Span::new(start, self.cursor.pos()), - )) + + if ch == '=' { + // Indicates this is an AssignDiv. + self.cursor.next(); // Consume the '=' + Ok(Token::new( + Punctuator::AssignDiv.into(), + Span::new(start, self.cursor.pos()), + )) + } else { + Ok(Token::new( + Punctuator::Div.into(), + Span::new(start, self.cursor.pos()), + )) + } } InputElement::RegExp | InputElement::RegExpOrTemplateTail => { // Can be a regular expression. diff --git a/boa/src/syntax/lexer/regex.rs b/boa/src/syntax/lexer/regex.rs index ef519aedc9c..06a9aaf83d6 100644 --- a/boa/src/syntax/lexer/regex.rs +++ b/boa/src/syntax/lexer/regex.rs @@ -1,7 +1,8 @@ -use super::{Cursor, Error, Tokenizer}; +use super::{Cursor, Error, Tokenizer, Span}; use crate::syntax::ast::Position; use crate::syntax::lexer::Token; use std::io::{self, ErrorKind, Read}; +use crate::syntax::lexer::TokenKind; /// Regex literal lexing. /// @@ -19,7 +20,7 @@ use std::io::{self, ErrorKind, Read}; pub(super) struct RegexLiteral; impl Tokenizer for RegexLiteral { - fn lex(&mut self, cursor: &mut Cursor, _start_pos: Position) -> Result + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where R: Read, { @@ -75,28 +76,13 @@ impl Tokenizer for RegexLiteral { } } - unimplemented!( - "Regex handling, requires ability to peek more than a single character ahead" - ); - // if regex { - // // body was parsed, now look for flags - // let flags = self.take_char_while(char::is_alphabetic)?; - // self.move_columns(body.len() as u32 + 1 + flags.len() as u32); - // self.push_token(TokenKind::regular_expression_literal( - // body, flags.parse()?, - // ), start_pos); - // } else { - // // failed to parse regex, restore original buffer position and - // // parse either div or assigndiv - // self.buffer = original_buffer; - // self.position = original_pos; - // if self.next_is('=') { - // self.push_token(TokenKind::Punctuator( - // Punctuator::AssignDiv, - // ), start_pos); - // } else { - // self.push_token(TokenKind::Punctuator(Punctuator::Div), start_pos); - // } - // } + // body was parsed, now look for flags + let mut flags = String::new(); + cursor.take_until_pred(&mut flags, &char::is_alphabetic); + + Ok(Token::new( + TokenKind::regular_expression_literal(body, flags.parse()?), + Span::new(start_pos, cursor.pos()), + )) } } From 81a400110f322122325fd873b299c9cd940adfc0 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 19:19:44 +0100 Subject: [PATCH 045/289] Starting to insert set_goal symbol statements --- boa/src/syntax/lexer/mod.rs | 16 ++++++++++++++-- boa/src/syntax/lexer/regex.rs | 4 ++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index dfc3730a4e6..0a168eaa6e0 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -204,9 +204,21 @@ where Span::new(start, self.cursor.pos()), )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), - template_match!() => TemplateLiteral::new().lex(&mut self.cursor, start), + template_match!() => { + // If a template has started then only expecting a template tail. + self.set_goal(InputElement::TemplateTail); + let result = TemplateLiteral::new().lex(&mut self.cursor, start); + + // A regex may follow a template literal but a DivPunctuator may not. + self.set_goal(InputElement::default()); + result + } _ if next_chr.is_digit(10) => { - NumberLiteral::new(next_chr, strict_mode).lex(&mut self.cursor, start) + let result = NumberLiteral::new(next_chr, strict_mode).lex(&mut self.cursor, start); + // A regex may not directly follow a NumericLiteral but a DivPunctuator may. + // Note that the goal cannot be set to InputElementTemplateTail at this point as a TemplateSubstitutionTail would be invalid. + self.set_goal(InputElement::Div); + result } _ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => { Identifier::new(next_chr).lex(&mut self.cursor, start) diff --git a/boa/src/syntax/lexer/regex.rs b/boa/src/syntax/lexer/regex.rs index 06a9aaf83d6..be503ee481d 100644 --- a/boa/src/syntax/lexer/regex.rs +++ b/boa/src/syntax/lexer/regex.rs @@ -1,8 +1,8 @@ -use super::{Cursor, Error, Tokenizer, Span}; +use super::{Cursor, Error, Span, Tokenizer}; use crate::syntax::ast::Position; use crate::syntax::lexer::Token; -use std::io::{self, ErrorKind, Read}; use crate::syntax::lexer::TokenKind; +use std::io::{self, ErrorKind, Read}; /// Regex literal lexing. /// From d062b0eacb7cc5feb706f94e9380603c30cc3fa6 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 19:26:02 +0100 Subject: [PATCH 046/289] Two_divisions_in_expression test, modified, passing --- boa/src/syntax/lexer/tests.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 770196c5662..39b02515e18 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -255,10 +255,23 @@ fn two_divisions_in_expression() { let s = " return a !== 0 || 1 / a === 1 / b;"; let mut lexer = Lexer::new(s.as_bytes()); - assert_eq!( - lexer.nth(11).unwrap().unwrap().span(), - span((1, 37), (1, 37)) - ); + let expected = [ + TokenKind::Keyword(Keyword::Return), + TokenKind::Identifier("a".into()), + TokenKind::Punctuator(Punctuator::StrictNotEq), + TokenKind::NumericLiteral(Numeric::Integer(0)), + TokenKind::Punctuator(Punctuator::BoolOr), + TokenKind::NumericLiteral(Numeric::Integer(1)), + TokenKind::Punctuator(Punctuator::Div), + TokenKind::Identifier("a".into()), + TokenKind::Punctuator(Punctuator::StrictEq), + TokenKind::NumericLiteral(Numeric::Integer(1)), + TokenKind::Punctuator(Punctuator::Div), + TokenKind::Identifier("b".into()), + TokenKind::Punctuator(Punctuator::Semicolon) + ]; + + expect_tokens(&mut lexer, &expected); } #[test] From 2d7fc84a81a210e186e48c20dfed21ee2e4876c7 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 19:56:46 +0100 Subject: [PATCH 047/289] Small changes to goal symbols --- boa/src/syntax/lexer/mod.rs | 21 +++++++++++++++------ boa/src/syntax/lexer/tests.rs | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 0a168eaa6e0..4c267b15dd4 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -206,11 +206,11 @@ where '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), template_match!() => { // If a template has started then only expecting a template tail. - self.set_goal(InputElement::TemplateTail); + self.set_goal(InputElement::TemplateTail); let result = TemplateLiteral::new().lex(&mut self.cursor, start); - // A regex may follow a template literal but a DivPunctuator may not. - self.set_goal(InputElement::default()); + // A regex may follow a template literal but a DivPunctuator or TemplateSubstitutionTail may not. + self.set_goal(InputElement::RegExp); result } _ if next_chr.is_digit(10) => { @@ -221,7 +221,12 @@ where result } _ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => { - Identifier::new(next_chr).lex(&mut self.cursor, start) + let result = Identifier::new(next_chr).lex(&mut self.cursor, start); + + // A regex may not directly follow an Identifier but a DivPunctuator may. + // Note that the goal cannot be set to InputElementTemplateTail at this point as a TemplateSubstitutionTail would be invalid. + self.set_goal(InputElement::Div); + result } ';' => Ok(Token::new( Punctuator::Semicolon.into(), @@ -265,8 +270,12 @@ where Span::new(start, self.cursor.pos()), )), '/' => self.lex_slash_token(start), - '*' | '+' | '-' | '%' | '|' | '&' | '^' | '=' | '<' | '>' | '!' | '~' => { - Operator::new(next_chr).lex(&mut self.cursor, start) + '=' | '*' | '+' | '-' | '%' | '|' | '&' | '^' | '<' | '>' | '!' | '~' => { + let result = Operator::new(next_chr).lex(&mut self.cursor, start); + + self.set_goal(InputElement::RegExpOrTemplateTail); + + result } _ => { let details = format!( diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 39b02515e18..85df4a1f0fd 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -268,7 +268,7 @@ fn two_divisions_in_expression() { TokenKind::NumericLiteral(Numeric::Integer(1)), TokenKind::Punctuator(Punctuator::Div), TokenKind::Identifier("b".into()), - TokenKind::Punctuator(Punctuator::Semicolon) + TokenKind::Punctuator(Punctuator::Semicolon), ]; expect_tokens(&mut lexer, &expected); From 815f782e527ed4570cd9c0b20e0935afb6981ead Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 23:08:15 +0100 Subject: [PATCH 048/289] Small fix, not performing a new-line during a carriage return --- boa/src/syntax/lexer/cursor.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 0d079c843fa..7317955a7a1 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -176,8 +176,7 @@ where match chr { Some(Ok('\r')) => { - self.carriage_return(); - self.next_line() + self.carriage_return() } Some(Ok('\u{2028}')) | Some(Ok('\u{2029}')) => self.next_line(), Some(Ok(_)) => self.next_column(), From a249b42c98333c3a21c9cd420dbf351243196d67 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 22:58:10 +0000 Subject: [PATCH 049/289] fixed bug in cursor, line not incremented on \n --- boa/src/syntax/lexer/cursor.rs | 2 +- boa/src/syntax/lexer/tests.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 7317955a7a1..26c31ce9654 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -178,7 +178,7 @@ where Some(Ok('\r')) => { self.carriage_return() } - Some(Ok('\u{2028}')) | Some(Ok('\u{2029}')) => self.next_line(), + Some(Ok('\n')) | Some(Ok('\u{2028}')) | Some(Ok('\u{2029}')) => self.next_line(), Some(Ok(_)) => self.next_column(), _ => {} } diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 85df4a1f0fd..f48b5636caa 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -280,9 +280,9 @@ fn check_line_numbers() { let mut lexer = Lexer::new(s.as_bytes()); - assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 1), (1, 1))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 1), (1, 2))); assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 2), (2, 1))); - assert_eq!(lexer.next().unwrap().unwrap().span(), span((2, 1), (2, 1))); + assert_eq!(lexer.next().unwrap().unwrap().span(), span((2, 1), (2, 2))); assert_eq!(lexer.next().unwrap().unwrap().span(), span((2, 2), (3, 1))); } From 923d10f1aa41e39d41ee8e38d853d0890a7c288e Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 23:11:02 +0000 Subject: [PATCH 050/289] Created new check_positions test, isolated positioning problem to the \u{} codepoint --- boa/src/syntax/lexer/cursor.rs | 4 +--- boa/src/syntax/lexer/mod.rs | 2 +- boa/src/syntax/lexer/tests.rs | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 26c31ce9654..4a3adeb4ea8 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -175,9 +175,7 @@ where }; match chr { - Some(Ok('\r')) => { - self.carriage_return() - } + Some(Ok('\r')) => self.carriage_return(), Some(Ok('\n')) | Some(Ok('\u{2028}')) | Some(Ok('\u{2029}')) => self.next_line(), Some(Ok(_)) => self.next_column(), _ => {} diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 4c267b15dd4..7973bbb6b12 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -272,7 +272,7 @@ where '/' => self.lex_slash_token(start), '=' | '*' | '+' | '-' | '%' | '|' | '&' | '^' | '<' | '>' | '!' | '~' => { let result = Operator::new(next_chr).lex(&mut self.cursor, start); - + self.set_goal(InputElement::RegExpOrTemplateTail); result diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index f48b5636caa..d929af032e4 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -212,6 +212,46 @@ fn check_variable_definition_tokens() { #[test] fn check_positions() { + let s = r#"console.log("hello world"); // Test"#; + // --------123456789 + let mut lexer = Lexer::new(s.as_bytes()); + + // The first column is 1 (not zero indexed) + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 1), (1, 8))); + + // Dot Token starts on column 8 + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 8), (1, 9))); + + // Log Token starts on column 9 + assert_eq!(lexer.next().unwrap().unwrap().span(), span((1, 9), (1, 12))); + + // Open parenthesis token starts on column 12 + assert_eq!( + lexer.next().unwrap().unwrap().span(), + span((1, 12), (1, 13)) + ); + + // String token starts on column 13 + assert_eq!( + lexer.next().unwrap().unwrap().span(), + span((1, 13), (1, 26)) + ); + + // Close parenthesis token starts on column 26. + assert_eq!( + lexer.next().unwrap().unwrap().span(), + span((1, 26), (1, 27)) + ); + + // Semi Colon token starts on column 35 + assert_eq!( + lexer.next().unwrap().unwrap().span(), + span((1, 27), (1, 28)) + ); +} + +#[test] +fn check_positions_codepoint() { let s = r#"console.log("hello world\u{2764}"); // Test"#; // --------123456789 let mut lexer = Lexer::new(s.as_bytes()); From c0565f215fdc9b7b36a229867a96b25102dedb6e Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Tue, 16 Jun 2020 23:40:59 +0000 Subject: [PATCH 051/289] Fix for stringliteral lexer not handling { and } in codepoints --- boa/src/syntax/lexer/string.rs | 4 ++++ boa/src/syntax/lexer/tests.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/boa/src/syntax/lexer/string.rs b/boa/src/syntax/lexer/string.rs index 0711f55cb27..d3b3556e71d 100644 --- a/boa/src/syntax/lexer/string.rs +++ b/boa/src/syntax/lexer/string.rs @@ -108,10 +108,14 @@ impl Tokenizer for StringLiteral { // Support \u{X..X} (Unicode Codepoint) if cursor.next_is('{')? { + cursor.next(); // Consume the '{'. + // The biggest code point is 0x10FFFF let mut code_point = String::with_capacity(6); cursor.take_until('}', &mut code_point)?; + cursor.next(); // Consume the '}'. + // We know this is a single unicode codepoint, convert to u32 let as_num = u32::from_str_radix(&code_point, 16).map_err(|_| { diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index d929af032e4..8bf3c5f11f5 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -252,7 +252,7 @@ fn check_positions() { #[test] fn check_positions_codepoint() { - let s = r#"console.log("hello world\u{2764}"); // Test"#; + let s = r#"console.log("hello world\u{{2764}}"); // Test"#; // --------123456789 let mut lexer = Lexer::new(s.as_bytes()); From 9064a485660490b1940b4dde767774d1493da906 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 17 Jun 2020 14:03:10 +0000 Subject: [PATCH 052/289] Expontential handling in number lexing tries to use integer if possible --- boa/src/syntax/lexer/number.rs | 25 +++++++++++++++++++++---- boa/src/syntax/lexer/tests.rs | 9 ++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index ad2bfc303ae..f773d294a1d 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -2,6 +2,7 @@ use super::{Cursor, Error, TokenKind, Tokenizer}; use crate::builtins::BigInt; use crate::syntax::ast::{Position, Span}; use crate::syntax::lexer::{token::Numeric, Token}; +use std::convert::TryFrom; use std::io::Read; use std::str::FromStr; @@ -331,10 +332,26 @@ impl Tokenizer for NumberLiteral { Numeric::Integer(num) } else { let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; - - Numeric::Rational( - (num as f64) * f64::powi(10.0, n) - ) + + if n < 0 { // A negative exponent is expected to produce a decimal value. + Numeric::Rational( + (num as f64) * f64::powi(10.0, n) + ) + } else { + if let Some(exp) = i32::checked_pow(10, n as u32) { + if let Some(val) = i32::checked_mul(num, exp) { + Numeric::Integer(val) + } else { + Numeric::Rational( + (num as f64) * (exp as f64) + ) + } + } else { + Numeric::Rational( + (num as f64) * f64::powi(10.0, n) + ) + } + } } } else { let b = f64::from(base); diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 8bf3c5f11f5..7f2621a392c 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -384,8 +384,8 @@ fn numbers() { TokenKind::numeric_literal(46), TokenKind::numeric_literal(7.89), TokenKind::numeric_literal(42.0), - TokenKind::numeric_literal(5000.0), - TokenKind::numeric_literal(5000.0), + TokenKind::numeric_literal(5000), + TokenKind::numeric_literal(5000), TokenKind::numeric_literal(0.005), TokenKind::numeric_literal(2), TokenKind::numeric_literal(83), @@ -393,7 +393,7 @@ fn numbers() { TokenKind::numeric_literal(10.0), TokenKind::numeric_literal(0.1), TokenKind::numeric_literal(10.0), - TokenKind::numeric_literal(10.0), + TokenKind::numeric_literal(10), TokenKind::numeric_literal(0.0), TokenKind::numeric_literal(0.12), TokenKind::Punctuator(Punctuator::Sub), @@ -620,8 +620,7 @@ fn illegal_following_numeric_literal() { fn illegal_code_point_following_numeric_literal() { // Checks as per https://tc39.es/ecma262/#sec-literals-numeric-literals that a NumericLiteral cannot // be immediately followed by an IdentifierStart where the IdentifierStart - - let mut lexer = Lexer::new(&b"17.4\\u{1000}"[0..]); + let mut lexer = Lexer::new(r#"17.4\u{{2764}}"#.as_bytes()); assert!( lexer.next().unwrap().err().is_some(), "IdentifierStart '\\u{0009}' following NumericLiteral not rejected as expected" From 707a35ed98d29fcb801fe67f21964d650415ed14 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 17 Jun 2020 14:09:45 +0000 Subject: [PATCH 053/289] Added big_numbers test to test lexing really large numbers --- boa/src/syntax/lexer/tests.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 7f2621a392c..d29dc787c4a 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -403,6 +403,20 @@ fn numbers() { expect_tokens(&mut lexer, &expected); } +#[test] +fn big_numbers() { + let mut lexer = Lexer::new(&b"1.0e25 1.0e36 9.0e50 10000000000000000000000000"[0..]); + + let expected = [ + TokenKind::numeric_literal(10000000000000000000000000.0), + TokenKind::numeric_literal(1000000000000000000000000000000000000.0), + TokenKind::numeric_literal(900000000000000000000000000000000000000000000000000.0), + TokenKind::numeric_literal(10000000000000000000000000.0), + ]; + + expect_tokens(&mut lexer, &expected); +} + #[test] fn implicit_octal_edge_case() { let mut lexer = Lexer::new(&b"044.5 094.5"[0..]); From 5911551901e882e58c598c9332a673d718fbf651 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 17 Jun 2020 15:39:54 +0000 Subject: [PATCH 054/289] Big numbers lexer tests, big literal numbers fails --- boa/src/syntax/lexer/tests.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index d29dc787c4a..8e7be92bd3e 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -404,19 +404,27 @@ fn numbers() { } #[test] -fn big_numbers() { - let mut lexer = Lexer::new(&b"1.0e25 1.0e36 9.0e50 10000000000000000000000000"[0..]); +fn big_exp_numbers() { + let mut lexer = Lexer::new(&b"1.0e25 1.0e36 9.0e50"[0..]); let expected = [ TokenKind::numeric_literal(10000000000000000000000000.0), TokenKind::numeric_literal(1000000000000000000000000000000000000.0), TokenKind::numeric_literal(900000000000000000000000000000000000000000000000000.0), - TokenKind::numeric_literal(10000000000000000000000000.0), ]; expect_tokens(&mut lexer, &expected); } +#[test] +fn big_literal_numbers() { + let mut lexer = Lexer::new(&b"10000000000000000000000000"[0..]); + + let expected = [TokenKind::numeric_literal(10000000000000000000000000.0)]; + + expect_tokens(&mut lexer, &expected); +} + #[test] fn implicit_octal_edge_case() { let mut lexer = Lexer::new(&b"044.5 094.5"[0..]); From 75cabc06da8b84f2a85a784e6cff603deffb8e4f Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 17 Jun 2020 15:44:41 +0000 Subject: [PATCH 055/289] Updating tests to reflect recent exponent number lexing change --- boa/src/syntax/lexer/tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 8e7be92bd3e..816c4a29ab3 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -543,7 +543,7 @@ fn addition_no_spaces_e_number_left_side() { let mut lexer = Lexer::new(&b"1e2+ 1"[0..]); let expected = [ - TokenKind::numeric_literal(100.0), + TokenKind::numeric_literal(100), TokenKind::Punctuator(Punctuator::Add), TokenKind::numeric_literal(1), ]; @@ -558,7 +558,7 @@ fn addition_no_spaces_e_number_right_side() { let expected = [ TokenKind::numeric_literal(1), TokenKind::Punctuator(Punctuator::Add), - TokenKind::numeric_literal(1000.0), + TokenKind::numeric_literal(1000), ]; expect_tokens(&mut lexer, &expected); @@ -569,7 +569,7 @@ fn addition_no_spaces_e_number() { let mut lexer = Lexer::new(&b"1e3+1e11"[0..]); let expected = [ - TokenKind::numeric_literal(1000.0), + TokenKind::numeric_literal(1000), TokenKind::Punctuator(Punctuator::Add), TokenKind::numeric_literal(100_000_000_000.0), ]; @@ -645,6 +645,6 @@ fn illegal_code_point_following_numeric_literal() { let mut lexer = Lexer::new(r#"17.4\u{{2764}}"#.as_bytes()); assert!( lexer.next().unwrap().err().is_some(), - "IdentifierStart '\\u{0009}' following NumericLiteral not rejected as expected" + "IdentifierStart \\u{{2764}} following NumericLiteral not rejected as expected" ); } From c1faef78573d0dc167f8ff1d484dedaa69ba32e6 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 17 Jun 2020 17:09:57 +0000 Subject: [PATCH 056/289] Fixed bug in cursor where UTF-8 values weren't decoded correctly --- boa/src/syntax/lexer/cursor.rs | 26 ++++++++++++++++---------- boa/src/syntax/lexer/tests.rs | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 4a3adeb4ea8..439dad72eb8 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -243,16 +243,22 @@ where *b = next; } - let int = u32::from_le_bytes(buf); - - match char::try_from(int).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "stream did not contain valid UTF-8", - ) - }) { - Ok(chr) => chr, - Err(e) => return Some(Err(e)), + match std::str::from_utf8(&buf) { + Ok(s) => match s.chars().next() { + Some(chr) => chr, + None => { + return Some(Err(io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ))); + } + }, + Err(_) => { + return Some(Err(io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ))); + } } }; diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 816c4a29ab3..8fc5fd759e3 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -648,3 +648,22 @@ fn illegal_code_point_following_numeric_literal() { "IdentifierStart \\u{{2764}} following NumericLiteral not rejected as expected" ); } + +#[test] +fn non_english_str() { + let str = r#"'中文';"#; + + let mut lexer = Lexer::new(str.as_bytes()); + + println!("{:?}", lexer.next()); + + // for l in lexer { + // println!("{:?}", l); + // } + + // let expected = [ + + // ]; + + // expect_tokens(&mut lexer, &expected); +} From 5df304521fae9ac62750cec3c60e578385bd6d67 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 17 Jun 2020 17:12:11 +0000 Subject: [PATCH 057/289] Non_english_str test --- boa/src/syntax/lexer/tests.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 8fc5fd759e3..9562d5ce9c7 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -655,15 +655,10 @@ fn non_english_str() { let mut lexer = Lexer::new(str.as_bytes()); - println!("{:?}", lexer.next()); - - // for l in lexer { - // println!("{:?}", l); - // } - - // let expected = [ - - // ]; + let expected = [ + TokenKind::StringLiteral("中文".into()), + TokenKind::Punctuator(Punctuator::Semicolon), + ]; - // expect_tokens(&mut lexer, &expected); + expect_tokens(&mut lexer, &expected); } From 287099be1f3c04d9157f67ff3a201547161a9c8c Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 20 Jun 2020 17:59:52 +0000 Subject: [PATCH 058/289] Removed assignment of InputGoal before TemplateLiteral lexing --- boa/src/syntax/lexer/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 7973bbb6b12..811e661f6c1 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -205,8 +205,6 @@ where )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), template_match!() => { - // If a template has started then only expecting a template tail. - self.set_goal(InputElement::TemplateTail); let result = TemplateLiteral::new().lex(&mut self.cursor, start); // A regex may follow a template literal but a DivPunctuator or TemplateSubstitutionTail may not. From 05c745a164772b02ed26e426336df327a8c79ca2 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 21 Jun 2020 21:25:14 +0000 Subject: [PATCH 059/289] cursor -> parser --- boa/src/syntax/lexer/mod.rs | 5 -- .../parser/{cursor.rs => cursor_old.rs} | 7 +- .../expression/assignment/arrow_function.rs | 14 ++-- .../expression/assignment/conditional.rs | 4 +- .../expression/assignment/exponentiation.rs | 6 +- .../parser/expression/assignment/mod.rs | 6 +- .../expression/left_hand_side/arguments.rs | 6 +- .../parser/expression/left_hand_side/call.rs | 6 +- .../expression/left_hand_side/member.rs | 6 +- .../parser/expression/left_hand_side/mod.rs | 6 +- boa/src/syntax/parser/expression/mod.rs | 4 +- .../primary/array_initializer/mod.rs | 6 +- .../expression/primary/function_expression.rs | 6 +- .../syntax/parser/expression/primary/mod.rs | 6 +- .../primary/object_initializer/mod.rs | 6 +- boa/src/syntax/parser/expression/unary.rs | 6 +- boa/src/syntax/parser/expression/update.rs | 6 +- boa/src/syntax/parser/function/mod.rs | 18 ++--- boa/src/syntax/parser/mod.rs | 76 +++++++++++-------- boa/src/syntax/parser/statement/block/mod.rs | 6 +- .../syntax/parser/statement/break_stm/mod.rs | 6 +- .../parser/statement/continue_stm/mod.rs | 6 +- .../parser/statement/declaration/hoistable.rs | 10 +-- .../parser/statement/declaration/lexical.rs | 14 ++-- .../parser/statement/declaration/mod.rs | 6 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 6 +- .../statement/iteration/do_while_statement.rs | 6 +- .../statement/iteration/for_statement.rs | 6 +- boa/src/syntax/parser/statement/mod.rs | 22 +++--- .../syntax/parser/statement/return_stm/mod.rs | 6 +- boa/src/syntax/parser/statement/switch/mod.rs | 10 +-- boa/src/syntax/parser/statement/throw/mod.rs | 6 +- .../syntax/parser/statement/try_stm/catch.rs | 4 +- .../parser/statement/try_stm/finally.rs | 4 +- .../syntax/parser/statement/try_stm/mod.rs | 6 +- boa/src/syntax/parser/statement/variable.rs | 14 ++-- 36 files changed, 172 insertions(+), 166 deletions(-) rename boa/src/syntax/parser/{cursor.rs => cursor_old.rs} (98%) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 811e661f6c1..c88a12e9d07 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -87,12 +87,7 @@ impl Lexer { pub(crate) fn get_goal(&self) -> InputElement { self.goal_symbol } -} -impl Lexer -where - R: Read, -{ /// Creates a new lexer. #[inline] pub fn new(reader: R) -> Self { diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor_old.rs similarity index 98% rename from boa/src/syntax/parser/cursor.rs rename to boa/src/syntax/parser/cursor_old.rs index 8ffd93d0e0e..dfa29db3b5f 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor_old.rs @@ -8,14 +8,15 @@ use crate::syntax::lexer::{Token, TokenKind}; /// /// This internal structure gives basic testable operations to the parser. #[derive(Debug, Clone, Default)] -pub(super) struct Cursor<'a> { +pub(super) struct Cursor { /// The tokens being input. - tokens: &'a [Token], + // tokens: &'a [Token], + lexer: crate::syntax::lexer::Lexer, /// The current position within the tokens. pos: usize, } -impl<'a> Cursor<'a> { +impl<'a> Cursor { /// Creates a new cursor. pub(super) fn new(tokens: &'a [Token]) -> Self { Self { diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index 1ada0363c73..7c2a48cf2b4 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -19,7 +19,7 @@ use crate::{ error::{ErrorContext, ParseError, ParseResult}, function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - AllowAwait, AllowIn, AllowYield, Cursor, TokenParser, + AllowAwait, AllowIn, AllowYield, Parser, TokenParser, }, }, BoaProfiler, @@ -60,10 +60,10 @@ impl ArrowFunction { } } -impl TokenParser for ArrowFunction { +impl TokenParser for ArrowFunction { type Output = ArrowFunctionDecl; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ArrowFunction", "Parsing"); let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token.kind { @@ -107,10 +107,10 @@ impl ConciseBody { } } -impl TokenParser for ConciseBody { +impl TokenParser for ConciseBody { type Output = StatementList; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { match cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Punctuator(Punctuator::OpenBlock) => { let _ = cursor.next(); @@ -147,10 +147,10 @@ impl ExpressionBody { } } -impl TokenParser for ExpressionBody { +impl TokenParser for ExpressionBody { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { AssignmentExpression::new(self.allow_in, false, self.allow_await).parse(cursor) } } diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index e903f27400f..4442bfb347f 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -54,10 +54,10 @@ impl ConditionalExpression { } } -impl TokenParser for ConditionalExpression { +impl TokenParser for ConditionalExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("Conditional", "Parsing"); // TODO: coalesce expression let lhs = LogicalORExpression::new(self.allow_in, self.allow_yield, self.allow_await) diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index ef18aa772f8..57c102e17f7 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -17,7 +17,7 @@ use crate::{ }, parser::{ expression::{unary::UnaryExpression, update::UpdateExpression}, - AllowAwait, AllowYield, Cursor, ParseResult, TokenParser, + AllowAwait, AllowYield, Parser, ParseResult, TokenParser, }, }, BoaProfiler, @@ -71,10 +71,10 @@ impl ExponentiationExpression { } } -impl TokenParser for ExponentiationExpression { +impl TokenParser for ExponentiationExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExponentiationExpression", "Parsing"); if Self::is_unary_expression(cursor) { return UnaryExpression::new(self.allow_yield, self.allow_await).parse(cursor); diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 99f5be42d32..8763cb0d120 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -19,7 +19,7 @@ use crate::{ node::{Assign, BinOp, Node}, Keyword, Punctuator, }, - parser::{AllowAwait, AllowIn, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, + parser::{AllowAwait, AllowIn, AllowYield, Parser, ParseError, ParseResult, TokenParser}, }, BoaProfiler, }; @@ -70,10 +70,10 @@ impl AssignmentExpression { } } -impl TokenParser for AssignmentExpression { +impl TokenParser for AssignmentExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); // Arrow function let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index 829d7271b9c..7b7e77f5f89 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -12,7 +12,7 @@ use crate::{ syntax::{ ast::{node::Spread, Node, Punctuator}, parser::{ - expression::AssignmentExpression, AllowAwait, AllowYield, Cursor, ParseError, + expression::AssignmentExpression, AllowAwait, AllowYield, Parser, ParseError, TokenParser, }, }, @@ -47,10 +47,10 @@ impl Arguments { } } -impl TokenParser for Arguments { +impl TokenParser for Arguments { type Output = Box<[Node]>; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Arguments", "Parsing"); cursor.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 29113073bc8..ba0fbc541cb 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -19,7 +19,7 @@ use crate::{ Punctuator, }, parser::{ - expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, ParseResult, + expression::Expression, AllowAwait, AllowYield, Parser, ParseError, ParseResult, TokenParser, }, }, @@ -54,10 +54,10 @@ impl CallExpression { } } -impl TokenParser for CallExpression { +impl TokenParser for CallExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("CallExpression", "Parsing"); let mut lhs = match cursor.peek(0) { Some(tk) if tk.kind == TokenKind::Punctuator(Punctuator::OpenParen) => { diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index c3dff1939fa..ffc5d746a2c 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -18,7 +18,7 @@ use crate::{ }, parser::{ expression::{primary::PrimaryExpression, Expression}, - AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser, + AllowAwait, AllowYield, Parser, ParseError, ParseResult, TokenParser, }, }, BoaProfiler, @@ -50,10 +50,10 @@ impl MemberExpression { } } -impl TokenParser for MemberExpression { +impl TokenParser for MemberExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("MemberExpression", "Parsing"); let mut lhs = if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind == TokenKind::Keyword(Keyword::New) diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index c739cf2e1d2..62ff592611a 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -16,7 +16,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Node, Punctuator}, - parser::{AllowAwait, AllowYield, Cursor, ParseResult, TokenParser}, + parser::{AllowAwait, AllowYield, Parser, ParseResult, TokenParser}, }, BoaProfiler, }; @@ -49,10 +49,10 @@ impl LeftHandSideExpression { } } -impl TokenParser for LeftHandSideExpression { +impl TokenParser for LeftHandSideExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("LeftHandSIdeExpression", "Parsing"); // TODO: Implement NewExpression: new MemberExpression let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index a2a8ad3acb2..375d84b42d0 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -17,7 +17,7 @@ mod update; use self::assignment::ExponentiationExpression; pub(super) use self::{assignment::AssignmentExpression, primary::Initializer}; -use super::{AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser}; +use super::{AllowAwait, AllowIn, AllowYield, Parser, ParseResult, TokenParser}; use crate::syntax::lexer::TokenKind; use crate::{ profiler::BoaProfiler, @@ -54,7 +54,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo impl TokenParser for $name { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; while let Some(tok) = cursor.peek(0) { diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index a5cd2b826a6..c303aa32931 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -17,7 +17,7 @@ use crate::{ Const, Punctuator, }, parser::{ - expression::AssignmentExpression, AllowAwait, AllowYield, Cursor, ParseError, + expression::AssignmentExpression, AllowAwait, AllowYield, Parser, ParseError, TokenParser, }, }, @@ -52,10 +52,10 @@ impl ArrayLiteral { } } -impl TokenParser for ArrayLiteral { +impl TokenParser for ArrayLiteral { type Output = ArrayDecl; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ArrayLiteral", "Parsing"); let mut elements = Vec::new(); diff --git a/boa/src/syntax/parser/expression/primary/function_expression.rs b/boa/src/syntax/parser/expression/primary/function_expression.rs index 8ebbf7347e4..87aad24c38a 100644 --- a/boa/src/syntax/parser/expression/primary/function_expression.rs +++ b/boa/src/syntax/parser/expression/primary/function_expression.rs @@ -13,7 +13,7 @@ use crate::{ parser::{ function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - Cursor, ParseError, TokenParser, + Parser, ParseError, TokenParser, }, }, BoaProfiler, @@ -30,10 +30,10 @@ use crate::{ #[derive(Debug, Clone, Copy)] pub(super) struct FunctionExpression; -impl TokenParser for FunctionExpression { +impl TokenParser for FunctionExpression { type Output = FunctionExpr; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("FunctionExpression", "Parsing"); let name = BindingIdentifier::new(false, false).try_parse(cursor); diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index bb89eb169dc..743483c770b 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -24,7 +24,7 @@ use crate::syntax::{ node::{Call, Identifier, New, Node}, Const, Keyword, Punctuator, }, - parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, + parser::{AllowAwait, AllowYield, Parser, ParseError, ParseResult, TokenParser}, }; pub(in crate::syntax::parser) use object_initializer::Initializer; @@ -56,10 +56,10 @@ impl PrimaryExpression { } } -impl TokenParser for PrimaryExpression { +impl TokenParser for PrimaryExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; match &tok.kind { diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index f3560be63a4..73e82f3c5e5 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -19,7 +19,7 @@ use crate::{ parser::{ expression::AssignmentExpression, function::{FormalParameters, FunctionBody}, - AllowAwait, AllowIn, AllowYield, Cursor, ParseError, ParseResult, TokenParser, + AllowAwait, AllowIn, AllowYield, Parser, ParseError, ParseResult, TokenParser, }, }, BoaProfiler, @@ -53,10 +53,10 @@ impl ObjectLiteral { } } -impl TokenParser for ObjectLiteral { +impl TokenParser for ObjectLiteral { type Output = Object; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ObjectLiteral", "Parsing"); let mut elements = Vec::new(); diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 2da3ca06b31..0b75ade88e5 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -15,7 +15,7 @@ use crate::syntax::{ Keyword, Punctuator, }, parser::{ - expression::update::UpdateExpression, AllowAwait, AllowYield, Cursor, ParseError, + expression::update::UpdateExpression, AllowAwait, AllowYield, Parser, ParseError, ParseResult, TokenParser, }, }; @@ -48,10 +48,10 @@ impl UnaryExpression { } } -impl TokenParser for UnaryExpression { +impl TokenParser for UnaryExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Delete) => { diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index 09e508c1a26..c54e395a8d7 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -9,7 +9,7 @@ use super::left_hand_side::LeftHandSideExpression; use crate::syntax::lexer::TokenKind; use crate::syntax::{ ast::{node, op::UnaryOp, Node, Punctuator}, - parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, + parser::{AllowAwait, AllowYield, Parser, ParseError, ParseResult, TokenParser}, }; /// Parses an update expression. @@ -38,10 +38,10 @@ impl UpdateExpression { } } -impl TokenParser for UpdateExpression { +impl TokenParser for UpdateExpression { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Punctuator(Punctuator::Inc) => { diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 725bf0d90c1..bbabf04b4e0 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -19,7 +19,7 @@ use crate::syntax::{ parser::{ expression::Initializer, statement::{BindingIdentifier, StatementList}, - AllowAwait, AllowYield, Cursor, ParseError, TokenParser, + AllowAwait, AllowYield, Parser, ParseError, TokenParser, }, }; @@ -51,10 +51,10 @@ impl FormalParameters { } } -impl TokenParser for FormalParameters { +impl TokenParser for FormalParameters { type Output = Box<[node::FormalParameter]>; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let mut params = Vec::new(); if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind @@ -134,10 +134,10 @@ impl BindingRestElement { } } -impl TokenParser for BindingRestElement { +impl TokenParser for BindingRestElement { type Output = node::FormalParameter; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { // FIXME: we are reading the spread operator before the rest element. // cursor.expect(Punctuator::Spread, "rest parameter")?; @@ -176,10 +176,10 @@ impl FormalParameter { } } -impl TokenParser for FormalParameter { +impl TokenParser for FormalParameter { type Output = node::FormalParameter; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { // TODO: BindingPattern let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; @@ -224,10 +224,10 @@ impl FunctionStatementList { } } -impl TokenParser for FunctionStatementList { +impl TokenParser for FunctionStatementList { type Output = node::StatementList; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { if let Some(tk) = cursor.peek(0) { if tk.kind == Punctuator::CloseBlock.into() { return Ok(Vec::new().into()); diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 2ee067e5540..8f29b4d5743 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -1,6 +1,5 @@ //! Boa parser implementation. -mod cursor; pub mod error; mod expression; mod function; @@ -11,32 +10,38 @@ mod tests; use self::error::{ParseError, ParseResult}; use crate::syntax::ast::node::StatementList; use crate::syntax::lexer::Token; -use cursor::Cursor; +use crate::syntax::lexer::Lexer; +use crate::syntax::ast::Node; + +use ParseError as Error; + +use std::io::Read; /// Trait implemented by parsers. /// /// This makes it possible to abstract over the underlying implementation of a parser. -trait TokenParser: Sized { +trait TokenParser: Sized +where + R: Read +{ /// Output type for the parser. type Output; // = Node; waiting for https://github.com/rust-lang/rust/issues/29661 /// Parses the token stream using the current parser. /// /// This method needs to be provided by the implementor type. - fn parse(self, cursor: &mut Cursor<'_>) -> Result; - - /// Tries to parse the following tokens with this parser. - /// - /// It will return the cursor to the initial position if an error occurs during parsing. - fn try_parse(self, cursor: &mut Cursor<'_>) -> Option { - let initial_pos = cursor.pos(); - if let Ok(node) = self.parse(cursor) { - Some(node) - } else { - cursor.seek(initial_pos); - None - } - } + fn parse(self, parser: &mut Parser) -> Result; + + // /// Tries to parse the following tokens with this parser. + // fn try_parse(self, parser: Parser) -> Option { + // let initial_pos = cursor.pos(); + // if let Ok(node) = self.parse(cursor) { + // Some(node) + // } else { + // cursor.seek(initial_pos); + // None + // } + // } } /// Boolean representing if the parser should allow a `yield` keyword. @@ -90,22 +95,27 @@ impl From for AllowDefault { } #[derive(Debug)] -pub struct Parser<'a> { - /// Cursor in the parser, the internal structure used to read tokens. - cursor: Cursor<'a>, +pub struct Parser { + /// Lexer used to get tokens for the parser. + lexer: Lexer, } -impl<'a> Parser<'a> { - /// Create a new parser, using `tokens` as input - pub fn new(tokens: &'a [Token]) -> Self { +impl Parser { + pub fn new(reader: R) -> Self { Self { - cursor: Cursor::new(tokens), + lexer: Lexer::new(reader) } } +} + +impl Iterator for Parser +where + R: Read, +{ + type Item = Result; + + fn next(&mut self) -> Option { - /// Parse all expressions in the token array - pub fn parse_all(&mut self) -> Result { - Script.parse(&mut self.cursor) } } @@ -118,12 +128,12 @@ impl<'a> Parser<'a> { #[derive(Debug, Clone, Copy)] pub struct Script; -impl TokenParser for Script { +impl TokenParser for Script { type Output = StatementList; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { if cursor.peek(0).is_some() { - ScriptBody.parse(cursor) + ScriptBody.parse(parser) } else { Ok(StatementList::from(Vec::new())) } @@ -139,10 +149,10 @@ impl TokenParser for Script { #[derive(Debug, Clone, Copy)] pub struct ScriptBody; -impl TokenParser for ScriptBody { +impl TokenParser for ScriptBody { type Output = StatementList; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { - self::statement::StatementList::new(false, false, false, false).parse(cursor) + fn parse(self, parser: &mut Parser) -> Result { + self::statement::StatementList::new(false, false, false, false).parse(parser) } } diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 04afc79fd00..5c5baab1445 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -17,7 +17,7 @@ use crate::{ profiler::BoaProfiler, syntax::{ ast::{node, Punctuator}, - parser::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}, + parser::{AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser}, }, }; @@ -60,10 +60,10 @@ impl Block { } } -impl TokenParser for Block { +impl TokenParser for Block { type Output = node::Block; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Block", "Parsing"); cursor.expect(Punctuator::OpenBlock, "block")?; if let Some(tk) = cursor.peek(0) { diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index 478b8b45973..97f4cc953b4 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -16,7 +16,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Break, Keyword, Punctuator}, - parser::{AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, + parser::{AllowAwait, AllowYield, Parser, ParseError, TokenParser}, }, BoaProfiler, }; @@ -49,10 +49,10 @@ impl BreakStatement { } } -impl TokenParser for BreakStatement { +impl TokenParser for BreakStatement { type Output = Break; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("BreakStatement", "Parsing"); cursor.expect(Keyword::Break, "break statement")?; diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index 88fe239c693..a1d92997bdd 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -15,7 +15,7 @@ use crate::{ syntax::{ ast::{node::Continue, Keyword, Punctuator}, parser::{ - statement::LabelIdentifier, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, + statement::LabelIdentifier, AllowAwait, AllowYield, Parser, ParseError, TokenParser, }, }, BoaProfiler, @@ -49,10 +49,10 @@ impl ContinueStatement { } } -impl TokenParser for ContinueStatement { +impl TokenParser for ContinueStatement { type Output = Continue; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ContinueStatement", "Parsing"); cursor.expect(Keyword::Continue, "continue statement")?; diff --git a/boa/src/syntax/parser/statement/declaration/hoistable.rs b/boa/src/syntax/parser/statement/declaration/hoistable.rs index 37185e6377f..b51692160a2 100644 --- a/boa/src/syntax/parser/statement/declaration/hoistable.rs +++ b/boa/src/syntax/parser/statement/declaration/hoistable.rs @@ -10,7 +10,7 @@ use crate::{ ast::{node::FunctionDecl, Keyword, Node, Punctuator}, parser::{ function::FormalParameters, function::FunctionBody, statement::BindingIdentifier, - AllowAwait, AllowDefault, AllowYield, Cursor, ParseError, ParseResult, TokenParser, + AllowAwait, AllowDefault, AllowYield, Parser, ParseError, ParseResult, TokenParser, }, }, BoaProfiler, @@ -45,10 +45,10 @@ impl HoistableDeclaration { } } -impl TokenParser for HoistableDeclaration { +impl TokenParser for HoistableDeclaration { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("HoistableDeclaration", "Parsing"); // TODO: check for generators and async functions + generators FunctionDeclaration::new(self.allow_yield, self.allow_await, self.is_default) @@ -88,10 +88,10 @@ impl FunctionDeclaration { } } -impl TokenParser for FunctionDeclaration { +impl TokenParser for FunctionDeclaration { type Output = FunctionDecl; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { cursor.expect(Keyword::Function, "function declaration")?; // TODO: If self.is_default, then this can be empty. diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 3fcff93ee33..8554ccda215 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -16,7 +16,7 @@ use crate::{ }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, - Cursor, ParseError, ParseResult, TokenParser, + Parser, ParseError, ParseResult, TokenParser, }, }, BoaProfiler, @@ -51,10 +51,10 @@ impl LexicalDeclaration { } } -impl TokenParser for LexicalDeclaration { +impl TokenParser for LexicalDeclaration { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("LexicalDeclaration", "Parsing"); let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; @@ -106,10 +106,10 @@ impl BindingList { } } -impl TokenParser for BindingList { +impl TokenParser for BindingList { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { // Create vectors to store the variable declarations // Const and Let signatures are slightly different, Const needs definitions, Lets don't let mut let_decls = Vec::new(); @@ -188,10 +188,10 @@ impl LexicalBinding { } } -impl TokenParser for LexicalBinding { +impl TokenParser for LexicalBinding { type Output = (Box, Option); - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let ident = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; let initializer = Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(cursor); diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index 90ed00172ad..d32547c3826 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -18,7 +18,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Keyword, Node}, - parser::{AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, + parser::{AllowAwait, AllowYield, Parser, ParseError, TokenParser}, }, BoaProfiler, }; @@ -48,10 +48,10 @@ impl Declaration { } } -impl TokenParser for Declaration { +impl TokenParser for Declaration { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Declaration", "Parsing"); let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index 7a21ef3465e..bd428435663 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -8,7 +8,7 @@ use crate::{ syntax::{ ast::{node::If, Keyword, Node, Punctuator}, parser::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, + expression::Expression, AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser, }, }, @@ -48,10 +48,10 @@ impl IfStatement { } } -impl TokenParser for IfStatement { +impl TokenParser for IfStatement { type Output = If; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("IfStatement", "Parsing"); cursor.expect(Keyword::If, "if statement")?; cursor.expect(Punctuator::OpenParen, "if statement")?; diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 716a03a996b..b253394f406 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -13,7 +13,7 @@ use crate::{ ast::{node::DoWhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, - Cursor, ParseError, TokenParser, + Parser, ParseError, TokenParser, }, }, BoaProfiler, @@ -54,10 +54,10 @@ impl DoWhileStatement { } } -impl TokenParser for DoWhileStatement { +impl TokenParser for DoWhileStatement { type Output = DoWhileLoop; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("DoWhileStatement", "Parsing"); cursor.expect(Keyword::Do, "do while statement")?; diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index c6dba0a85e0..70f992488ff 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -18,7 +18,7 @@ use crate::{ expression::Expression, statement::declaration::Declaration, statement::{variable::VariableDeclarationList, Statement}, - AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, + AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser, }, }, BoaProfiler, @@ -59,10 +59,10 @@ impl ForStatement { } } -impl TokenParser for ForStatement { +impl TokenParser for ForStatement { type Output = ForLoop; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ForStatement", "Parsing"); cursor.expect(Keyword::For, "for statement")?; cursor.expect(Punctuator::OpenParen, "for statement")?; diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index aaf3802af80..c57d4619966 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -33,7 +33,7 @@ use self::{ variable::VariableStatement, }; use super::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, ParseResult, + expression::Expression, AllowAwait, AllowReturn, AllowYield, Parser, ParseError, ParseResult, TokenParser, }; @@ -91,10 +91,10 @@ impl Statement { } } -impl TokenParser for Statement { +impl TokenParser for Statement { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Statement", "Parsing"); // TODO: add BreakableStatement and divide Whiles, fors and so on to another place. let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; @@ -211,10 +211,10 @@ impl StatementList { } } -impl TokenParser for StatementList { +impl TokenParser for StatementList { type Output = node::StatementList; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("StatementList", "Parsing"); let mut items = Vec::new(); @@ -285,10 +285,10 @@ impl StatementListItem { } } -impl TokenParser for StatementListItem { +impl TokenParser for StatementListItem { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("StatementListItem", "Parsing"); let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; @@ -331,10 +331,10 @@ impl ExpressionStatement { } } -impl TokenParser for ExpressionStatement { +impl TokenParser for ExpressionStatement { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { + fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExpressionStatement", "Parsing"); // TODO: lookahead let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; @@ -381,10 +381,10 @@ impl BindingIdentifier { } } -impl TokenParser for BindingIdentifier { +impl TokenParser for BindingIdentifier { type Output = Box; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("BindingIdentifier", "Parsing"); // TODO: strict mode. diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index ccdb14a9cfb..db5cca20703 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -5,7 +5,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Return, Keyword, Node, Punctuator}, - parser::{expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, + parser::{expression::Expression, AllowAwait, AllowYield, Parser, ParseError, TokenParser}, }, BoaProfiler, }; @@ -38,10 +38,10 @@ impl ReturnStatement { } } -impl TokenParser for ReturnStatement { +impl TokenParser for ReturnStatement { type Output = Return; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ReturnStatement", "Parsing"); cursor.expect(Keyword::Return, "return statement")?; diff --git a/boa/src/syntax/parser/statement/switch/mod.rs b/boa/src/syntax/parser/statement/switch/mod.rs index 8a64086cf12..703efa0f7e4 100644 --- a/boa/src/syntax/parser/statement/switch/mod.rs +++ b/boa/src/syntax/parser/statement/switch/mod.rs @@ -8,7 +8,7 @@ use crate::{ Keyword, Node, Punctuator, }, parser::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, + expression::Expression, AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser, }, }, @@ -46,10 +46,10 @@ impl SwitchStatement { } } -impl TokenParser for SwitchStatement { +impl TokenParser for SwitchStatement { type Output = Switch; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("SwitchStatement", "Parsing"); cursor.expect(Keyword::Switch, "switch statement")?; cursor.expect(Punctuator::OpenParen, "switch statement")?; @@ -94,10 +94,10 @@ impl CaseBlock { } } -impl TokenParser for CaseBlock { +impl TokenParser for CaseBlock { type Output = (Box<[Case]>, Option); - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { cursor.expect(Punctuator::OpenBlock, "switch case block")?; // CaseClauses[?Yield, ?Await, ?Return]opt diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index aa046be358e..29aa3fa2b55 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -5,7 +5,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Throw, Keyword, Punctuator}, - parser::{expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, + parser::{expression::Expression, AllowAwait, AllowYield, Parser, ParseError, TokenParser}, }, BoaProfiler, }; @@ -38,10 +38,10 @@ impl ThrowStatement { } } -impl TokenParser for ThrowStatement { +impl TokenParser for ThrowStatement { type Output = Throw; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ThrowStatement", "Parsing"); cursor.expect(Keyword::Throw, "throw statement")?; diff --git a/boa/src/syntax/parser/statement/try_stm/catch.rs b/boa/src/syntax/parser/statement/try_stm/catch.rs index 1e818400a0b..098febddf24 100644 --- a/boa/src/syntax/parser/statement/try_stm/catch.rs +++ b/boa/src/syntax/parser/statement/try_stm/catch.rs @@ -43,10 +43,10 @@ impl Catch { } } -impl TokenParser for Catch { +impl TokenParser for Catch { type Output = node::Catch; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Catch", "Parsing"); cursor.expect(Keyword::Catch, "try statement")?; let catch_param = if cursor.next_if(Punctuator::OpenParen).is_some() { diff --git a/boa/src/syntax/parser/statement/try_stm/finally.rs b/boa/src/syntax/parser/statement/try_stm/finally.rs index ac4f39a395f..6fdfaa916a0 100644 --- a/boa/src/syntax/parser/statement/try_stm/finally.rs +++ b/boa/src/syntax/parser/statement/try_stm/finally.rs @@ -40,10 +40,10 @@ impl Finally { } } -impl TokenParser for Finally { +impl TokenParser for Finally { type Output = node::Finally; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Finally", "Parsing"); cursor.expect(Keyword::Finally, "try statement")?; Ok( diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index afcb4519f99..e33af12c832 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -11,7 +11,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Try, Keyword}, - parser::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}, + parser::{AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser}, }, BoaProfiler, }; @@ -47,10 +47,10 @@ impl TryStatement { } } -impl TokenParser for TryStatement { +impl TokenParser for TryStatement { type Output = Try; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("TryStatement", "Parsing"); // TRY cursor.expect(Keyword::Try, "try statement")?; diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index ddc86c0c6d6..31152bcbb60 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -8,7 +8,7 @@ use crate::{ }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, - Cursor, ParseError, TokenParser, + Parser, ParseError, TokenParser, }, }, BoaProfiler, @@ -44,10 +44,10 @@ impl VariableStatement { } } -impl TokenParser for VariableStatement { +impl TokenParser for VariableStatement { type Output = VarDeclList; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("VariableStatement", "Parsing"); cursor.expect(Keyword::Var, "variable statement")?; @@ -95,10 +95,10 @@ impl VariableDeclarationList { } } -impl TokenParser for VariableDeclarationList { +impl TokenParser for VariableDeclarationList { type Output = VarDeclList; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let mut list = Vec::new(); loop { @@ -158,10 +158,10 @@ impl VariableDeclaration { } } -impl TokenParser for VariableDeclaration { +impl TokenParser for VariableDeclaration { type Output = VarDecl; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { // TODO: BindingPattern let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; From f4a0c1c20f4d5a561925fa7e8c0ca4c7ac0dc61f Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 21 Jun 2020 22:51:18 +0000 Subject: [PATCH 060/289] Replacing cursor with parser --- .../expression/assignment/arrow_function.rs | 28 ++++----- .../expression/assignment/conditional.rs | 14 ++--- .../expression/assignment/exponentiation.rs | 35 +++++++---- .../parser/expression/assignment/mod.rs | 20 +++--- .../expression/left_hand_side/arguments.rs | 14 ++--- .../parser/expression/left_hand_side/call.rs | 20 +++--- .../expression/left_hand_side/member.rs | 22 +++---- .../parser/expression/left_hand_side/mod.rs | 6 +- boa/src/syntax/parser/expression/mod.rs | 14 ++--- .../primary/array_initializer/mod.rs | 14 ++--- .../expression/primary/function_expression.rs | 14 ++--- .../syntax/parser/expression/primary/mod.rs | 12 ++-- .../primary/object_initializer/mod.rs | 62 +++++++++---------- boa/src/syntax/parser/expression/unary.rs | 20 +++--- boa/src/syntax/parser/expression/update.rs | 18 +++--- boa/src/syntax/parser/function/mod.rs | 26 ++++---- boa/src/syntax/parser/mod.rs | 19 +++--- boa/src/syntax/parser/statement/block/mod.rs | 10 +-- .../syntax/parser/statement/break_stm/mod.rs | 10 +-- .../parser/statement/continue_stm/mod.rs | 10 +-- .../parser/statement/declaration/hoistable.rs | 18 +++--- .../parser/statement/declaration/lexical.rs | 20 +++--- .../parser/statement/declaration/mod.rs | 6 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 16 ++--- .../statement/iteration/do_while_statement.rs | 16 ++--- .../statement/iteration/for_statement.rs | 28 ++++----- .../statement/iteration/while_statement.rs | 16 ++--- boa/src/syntax/parser/statement/mod.rs | 46 +++++++------- .../syntax/parser/statement/return_stm/mod.rs | 10 +-- boa/src/syntax/parser/statement/switch/mod.rs | 12 ++-- boa/src/syntax/parser/statement/throw/mod.rs | 10 +-- .../syntax/parser/statement/try_stm/catch.rs | 18 +++--- .../parser/statement/try_stm/finally.rs | 6 +- .../syntax/parser/statement/try_stm/mod.rs | 12 ++-- boa/src/syntax/parser/statement/variable.rs | 18 +++--- 35 files changed, 326 insertions(+), 314 deletions(-) diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index 7c2a48cf2b4..6c24a36d1a9 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -65,25 +65,25 @@ impl TokenParser for ArrowFunction { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ArrowFunction", "Parsing"); - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = parser.peek(0).ok_or(ParseError::AbruptEnd)?; let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token.kind { // CoverParenthesizedExpressionAndArrowParameterList - cursor.expect(Punctuator::OpenParen, "arrow function")?; - let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect(Punctuator::CloseParen, "arrow function")?; + parser.expect(Punctuator::OpenParen, "arrow function")?; + let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(parser)?; + parser.expect(Punctuator::CloseParen, "arrow function")?; params } else { let param = BindingIdentifier::new(self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .context("arrow function")?; Box::new([FormalParameter::new(param, None, false)]) }; - cursor.peek_expect_no_lineterminator(0)?; + parser.peek_expect_no_lineterminator(0)?; - cursor.expect(Punctuator::Arrow, "arrow function")?; + parser.expect(Punctuator::Arrow, "arrow function")?; - let body = ConciseBody::new(self.allow_in).parse(cursor)?; + let body = ConciseBody::new(self.allow_in).parse(parser)?; Ok(ArrowFunctionDecl::new(params, body)) } @@ -111,15 +111,15 @@ impl TokenParser for ConciseBody { type Output = StatementList; fn parse(self, parser: &mut Parser) -> Result { - match cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind { + match parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Punctuator(Punctuator::OpenBlock) => { - let _ = cursor.next(); - let body = FunctionBody::new(false, false).parse(cursor)?; - cursor.expect(Punctuator::CloseBlock, "arrow function")?; + let _ = parser.next(); + let body = FunctionBody::new(false, false).parse(parser)?; + parser.expect(Punctuator::CloseBlock, "arrow function")?; Ok(body) } _ => Ok(StatementList::from(vec![Return::new( - ExpressionBody::new(self.allow_in, false).parse(cursor)?, + ExpressionBody::new(self.allow_in, false).parse(parser)?, ) .into()])), } @@ -151,6 +151,6 @@ impl TokenParser for ExpressionBody { type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { - AssignmentExpression::new(self.allow_in, false, self.allow_await).parse(cursor) + AssignmentExpression::new(self.allow_in, false, self.allow_await).parse(parser) } } diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index 4442bfb347f..19c7b530903 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -13,7 +13,7 @@ use crate::{ ast::{node::ConditionalOp, Node, Punctuator}, parser::{ expression::{AssignmentExpression, LogicalORExpression}, - AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser, + AllowAwait, AllowIn, AllowYield, Parser, ParseResult, TokenParser, }, }, BoaProfiler, @@ -61,21 +61,21 @@ impl TokenParser for ConditionalExpression { let _timer = BoaProfiler::global().start_event("Conditional", "Parsing"); // TODO: coalesce expression let lhs = LogicalORExpression::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(cursor)?; + .parse(parser)?; - if let Some(tok) = cursor.next() { + if let Some(tok) = parser.next() { if tok.kind == TokenKind::Punctuator(Punctuator::Question) { let then_clause = AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(cursor)?; - cursor.expect(Punctuator::Colon, "conditional expression")?; + .parse(parser)?; + parser.expect(Punctuator::Colon, "conditional expression")?; let else_clause = AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(cursor)?; + .parse(parser)?; return Ok(ConditionalOp::new(lhs, then_clause, else_clause).into()); } else { - cursor.back(); + parser.back(); } } diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 57c102e17f7..5375ac4513c 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -23,6 +23,8 @@ use crate::{ BoaProfiler, }; +use std::io::Read; + /// Parses an exponentiation expression. /// /// More information: @@ -32,12 +34,15 @@ use crate::{ /// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Arithmetic_Operators#Exponentiation /// [spec]: https://tc39.es/ecma262/#prod-ExponentiationExpression #[derive(Debug, Clone, Copy)] -pub(in crate::syntax::parser::expression) struct ExponentiationExpression { +pub(in crate::syntax::parser::expression) struct ExponentiationExpression { allow_yield: AllowYield, allow_await: AllowAwait, } -impl ExponentiationExpression { +impl ExponentiationExpression +where + R: Read +{ /// Creates a new `ExponentiationExpression` parser. pub(in crate::syntax::parser::expression) fn new(allow_yield: Y, allow_await: A) -> Self where @@ -49,12 +54,13 @@ impl ExponentiationExpression { allow_await: allow_await.into(), } } -} -impl ExponentiationExpression { /// Checks by looking at the next token to see whether it's a unary operator or not. - fn is_unary_expression(cursor: &mut Cursor<'_>) -> bool { - if let Some(tok) = cursor.peek(0) { + fn is_unary_expression(parser: &mut Parser) -> bool + where + R: Read + { + if let Some(tok) = parser.peek(0) { match tok.kind { TokenKind::Keyword(Keyword::Delete) | TokenKind::Keyword(Keyword::Void) @@ -71,21 +77,24 @@ impl ExponentiationExpression { } } -impl TokenParser for ExponentiationExpression { +impl TokenParser for ExponentiationExpression +where + R: Read +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExponentiationExpression", "Parsing"); - if Self::is_unary_expression(cursor) { - return UnaryExpression::new(self.allow_yield, self.allow_await).parse(cursor); + if Self::is_unary_expression(parser) { + return UnaryExpression::new(self.allow_yield, self.allow_await).parse(parser); } - let lhs = UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.next() { + let lhs = UpdateExpression::new(self.allow_yield, self.allow_await).parse(parser)?; + if let Some(tok) = parser.next() { if let TokenKind::Punctuator(Punctuator::Exp) = tok.kind { - return Ok(BinOp::new(NumOp::Exp, lhs, self.parse(cursor)?).into()); + return Ok(BinOp::new(NumOp::Exp, lhs, self.parse(parser)?).into()); } else { - cursor.back(); + parser.back(); } } Ok(lhs) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 8763cb0d120..b0d50191f35 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -76,22 +76,22 @@ impl TokenParser for AssignmentExpression { fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); // Arrow function - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = parser.peek(0).ok_or(ParseError::AbruptEnd)?; match next_token.kind { // a=>{} TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield) | TokenKind::Keyword(Keyword::Await) - if cursor.peek_expect_no_lineterminator(1).is_ok() => + if parser.peek_expect_no_lineterminator(1).is_ok() => { - if let Some(tok) = cursor.peek(1) { + if let Some(tok) = parser.peek(1) { if tok.kind == TokenKind::Punctuator(Punctuator::Arrow) { return ArrowFunction::new( self.allow_in, self.allow_yield, self.allow_await, ) - .parse(cursor) + .parse(parser) .map(Node::ArrowFunctionDecl); } } @@ -100,7 +100,7 @@ impl TokenParser for AssignmentExpression { TokenKind::Punctuator(Punctuator::OpenParen) => { if let Some(node) = ArrowFunction::new(self.allow_in, self.allow_yield, self.allow_await) - .try_parse(cursor) + .try_parse(parser) .map(Node::ArrowFunctionDecl) { return Ok(node); @@ -110,20 +110,20 @@ impl TokenParser for AssignmentExpression { } let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(cursor)?; + .parse(parser)?; - if let Some(tok) = cursor.next() { + if let Some(tok) = parser.next() { match tok.kind { TokenKind::Punctuator(Punctuator::Assign) => { - lhs = Assign::new(lhs, self.parse(cursor)?).into(); + lhs = Assign::new(lhs, self.parse(parser)?).into(); } TokenKind::Punctuator(p) if p.as_binop().is_some() => { - let expr = self.parse(cursor)?; + let expr = self.parse(parser)?; let binop = p.as_binop().expect("binop disappeared"); lhs = BinOp::new(binop, lhs, expr).into(); } _ => { - cursor.back(); + parser.back(); } } } diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index 7b7e77f5f89..7c7b180ce37 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -52,10 +52,10 @@ impl TokenParser for Arguments { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Arguments", "Parsing"); - cursor.expect(Punctuator::OpenParen, "arguments")?; + parser.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); loop { - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; + let next_token = parser.next().ok_or(ParseError::AbruptEnd)?; match next_token.kind { TokenKind::Punctuator(Punctuator::CloseParen) => break, TokenKind::Punctuator(Punctuator::Comma) => { @@ -63,7 +63,7 @@ impl TokenParser for Arguments { return Err(ParseError::unexpected(next_token.clone(), None)); } - if cursor.next_if(Punctuator::CloseParen).is_some() { + if parser.next_if(Punctuator::CloseParen).is_some() { break; } } @@ -78,23 +78,23 @@ impl TokenParser for Arguments { "argument list", )); } else { - cursor.back(); + parser.back(); } } } - if cursor.next_if(Punctuator::Spread).is_some() { + if parser.next_if(Punctuator::Spread).is_some() { args.push( Spread::new( AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(cursor)?, + .parse(parser)?, ) .into(), ); } else { args.push( AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(cursor)?, + .parse(parser)?, ); } } diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index ba0fbc541cb..f0dce2b1fd7 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -59,13 +59,13 @@ impl TokenParser for CallExpression { fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("CallExpression", "Parsing"); - let mut lhs = match cursor.peek(0) { + let mut lhs = match parser.peek(0) { Some(tk) if tk.kind == TokenKind::Punctuator(Punctuator::OpenParen) => { - let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; + let args = Arguments::new(self.allow_yield, self.allow_await).parse(parser)?; Node::from(Call::new(self.first_member_expr, args)) } _ => { - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; + let next_token = parser.next().ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::OpenParen)], next_token.clone(), @@ -74,15 +74,15 @@ impl TokenParser for CallExpression { } }; - while let Some(tok) = cursor.peek(0) { + while let Some(tok) = parser.peek(0) { match tok.kind { TokenKind::Punctuator(Punctuator::OpenParen) => { - let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; + let args = Arguments::new(self.allow_yield, self.allow_await).parse(parser)?; lhs = Node::from(Call::new(lhs, args)); } TokenKind::Punctuator(Punctuator::Dot) => { - let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the cursor. - match &cursor.next().ok_or(ParseError::AbruptEnd)?.kind { + let _ = parser.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. + match &parser.next().ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into(); } @@ -99,10 +99,10 @@ impl TokenParser for CallExpression { } } TokenKind::Punctuator(Punctuator::OpenBracket) => { - let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the cursor. + let _ = parser.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. let idx = - Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect(Punctuator::CloseBracket, "call expression")?; + Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + parser.expect(Punctuator::CloseBracket, "call expression")?; lhs = GetField::new(lhs, idx).into(); } _ => break, diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index ffc5d746a2c..7650cca9aea 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -55,23 +55,23 @@ impl TokenParser for MemberExpression { fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("MemberExpression", "Parsing"); - let mut lhs = if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind + let mut lhs = if parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind == TokenKind::Keyword(Keyword::New) { - let _ = cursor.next().expect("keyword disappeared"); - let lhs = self.parse(cursor)?; - let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; + let _ = parser.next().expect("keyword disappeared"); + let lhs = self.parse(parser)?; + let args = Arguments::new(self.allow_yield, self.allow_await).parse(parser)?; let call_node = Call::new(lhs, args); Node::from(New::from(call_node)) } else { - PrimaryExpression::new(self.allow_yield, self.allow_await).parse(cursor)? + PrimaryExpression::new(self.allow_yield, self.allow_await).parse(parser)? }; - while let Some(tok) = cursor.peek(0) { + while let Some(tok) = parser.peek(0) { match &tok.kind { TokenKind::Punctuator(Punctuator::Dot) => { - let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the cursor forward. - match &cursor.next().ok_or(ParseError::AbruptEnd)?.kind { + let _ = parser.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. + match &parser.next().ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into() } @@ -88,10 +88,10 @@ impl TokenParser for MemberExpression { } } TokenKind::Punctuator(Punctuator::OpenBracket) => { - let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the cursor forward. + let _ = parser.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. let idx = - Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect(Punctuator::CloseBracket, "member expression")?; + Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + parser.expect(Punctuator::CloseBracket, "member expression")?; lhs = GetField::new(lhs, idx).into(); } _ => break, diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 62ff592611a..f89931dadca 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -55,10 +55,10 @@ impl TokenParser for LeftHandSideExpression { fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("LeftHandSIdeExpression", "Parsing"); // TODO: Implement NewExpression: new MemberExpression - let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - match cursor.peek(0) { + let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(parser)?; + match parser.peek(0) { Some(ref tok) if tok.kind == TokenKind::Punctuator(Punctuator::OpenParen) => { - CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor) + CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(parser) } _ => Ok(lhs), // TODO: is this correct? } diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 375d84b42d0..15ffb39ad2e 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -51,28 +51,28 @@ impl PartialEq for Keyword { /// /// Those exressions are divided by the punctuators passed as the third parameter. macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $low_param:ident ),*] ) => { - impl TokenParser for $name { + impl TokenParser for $name { type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); - let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; - while let Some(tok) = cursor.peek(0) { + let mut lhs = $lower::new($( self.$low_param ),*).parse(parser)?; + while let Some(tok) = parser.peek(0) { match tok.kind { TokenKind::Punctuator(op) if $( op == $op )||* => { - let _ = cursor.next().expect("token disappeared"); + let _ = parser.next().expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), lhs, - $lower::new($( self.$low_param ),*).parse(cursor)? + $lower::new($( self.$low_param ),*).parse(parser)? ).into(); } TokenKind::Keyword(op) if $( op == $op )||* => { - let _ = cursor.next().expect("token disappeared"); + let _ = parser.next().expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), lhs, - $lower::new($( self.$low_param ),*).parse(cursor)? + $lower::new($( self.$low_param ),*).parse(parser)? ).into(); } _ => break diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index c303aa32931..f1df3689898 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -61,27 +61,27 @@ impl TokenParser for ArrayLiteral { loop { // TODO: Support all features. - while cursor.next_if(Punctuator::Comma).is_some() { + while parser.next_if(Punctuator::Comma).is_some() { elements.push(Node::Const(Const::Undefined)); } - if cursor.next_if(Punctuator::CloseBracket).is_some() { + if parser.next_if(Punctuator::CloseBracket).is_some() { break; } - let _ = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; // Check that there are more tokens to read. + let _ = parser.peek(0).ok_or(ParseError::AbruptEnd)?; // Check that there are more tokens to read. - if cursor.next_if(Punctuator::Spread).is_some() { + if parser.next_if(Punctuator::Spread).is_some() { let node = AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(cursor)?; + .parse(parser)?; elements.push(Spread::new(node).into()); } else { elements.push( AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(cursor)?, + .parse(parser)?, ); } - cursor.next_if(Punctuator::Comma); + parser.next_if(Punctuator::Comma); } Ok(elements.into()) diff --git a/boa/src/syntax/parser/expression/primary/function_expression.rs b/boa/src/syntax/parser/expression/primary/function_expression.rs index 87aad24c38a..542b9702b4d 100644 --- a/boa/src/syntax/parser/expression/primary/function_expression.rs +++ b/boa/src/syntax/parser/expression/primary/function_expression.rs @@ -35,18 +35,18 @@ impl TokenParser for FunctionExpression { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("FunctionExpression", "Parsing"); - let name = BindingIdentifier::new(false, false).try_parse(cursor); + let name = BindingIdentifier::new(false, false).try_parse(parser); - cursor.expect(Punctuator::OpenParen, "function expression")?; + parser.expect(Punctuator::OpenParen, "function expression")?; - let params = FormalParameters::new(false, false).parse(cursor)?; + let params = FormalParameters::new(false, false).parse(parser)?; - cursor.expect(Punctuator::CloseParen, "function expression")?; - cursor.expect(Punctuator::OpenBlock, "function expression")?; + parser.expect(Punctuator::CloseParen, "function expression")?; + parser.expect(Punctuator::OpenBlock, "function expression")?; - let body = FunctionBody::new(false, false).parse(cursor)?; + let body = FunctionBody::new(false, false).parse(parser)?; - cursor.expect(Punctuator::CloseBlock, "function expression")?; + parser.expect(Punctuator::CloseBlock, "function expression")?; Ok(FunctionExpr::new(name, params, body)) } diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 743483c770b..54264ba4a25 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -60,28 +60,28 @@ impl TokenParser for PrimaryExpression { type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { - let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; + let tok = parser.next().ok_or(ParseError::AbruptEnd)?; match &tok.kind { TokenKind::Keyword(Keyword::This) => Ok(Node::This), // TokenKind::Keyword(Keyword::Arguments) => Ok(Node::new(NodeBase::Arguments, tok.pos)), TokenKind::Keyword(Keyword::Function) => { - FunctionExpression.parse(cursor).map(Node::from) + FunctionExpression.parse(parser).map(Node::from) } TokenKind::Punctuator(Punctuator::OpenParen) => { let expr = - Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect(Punctuator::CloseParen, "primary expression")?; + Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + parser.expect(Punctuator::CloseParen, "primary expression")?; Ok(expr) } TokenKind::Punctuator(Punctuator::OpenBracket) => { ArrayLiteral::new(self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .map(Node::ArrayDecl) } TokenKind::Punctuator(Punctuator::OpenBlock) => { Ok(ObjectLiteral::new(self.allow_yield, self.allow_await) - .parse(cursor)? + .parse(parser)? .into()) } TokenKind::BooleanLiteral(boolean) => Ok(Const::from(*boolean).into()), diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 73e82f3c5e5..8e76501720c 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -61,19 +61,19 @@ impl TokenParser for ObjectLiteral { let mut elements = Vec::new(); loop { - if cursor.next_if(Punctuator::CloseBlock).is_some() { + if parser.next_if(Punctuator::CloseBlock).is_some() { break; } elements - .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(cursor)?); + .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(parser)?); - if cursor.next_if(Punctuator::CloseBlock).is_some() { + if parser.next_if(Punctuator::CloseBlock).is_some() { break; } - if cursor.next_if(Punctuator::Comma).is_none() { - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; + if parser.next_if(Punctuator::Comma).is_none() { + let next_token = parser.next().ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![ TokenKind::Punctuator(Punctuator::Comma), @@ -115,36 +115,36 @@ impl PropertyDefinition { } } -impl TokenParser for PropertyDefinition { +impl TokenParser for PropertyDefinition { type Output = node::PropertyDefinition; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { - if cursor.next_if(Punctuator::Spread).is_some() { + fn parse(self, parser: &mut Parser) -> Result { + if parser.next_if(Punctuator::Spread).is_some() { let node = AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(cursor)?; + .parse(parser)?; return Ok(node::PropertyDefinition::SpreadObject(node)); } - let prop_name = cursor + let prop_name = parser .next() .map(Token::to_string) .ok_or(ParseError::AbruptEnd)?; - if cursor.next_if(Punctuator::Colon).is_some() { + if parser.next_if(Punctuator::Colon).is_some() { let val = AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(cursor)?; + .parse(parser)?; return Ok(node::PropertyDefinition::property(prop_name, val)); } - if cursor + if parser .next_if(TokenKind::Punctuator(Punctuator::OpenParen)) .is_some() || ["get", "set"].contains(&prop_name.as_str()) { return MethodDefinition::new(self.allow_yield, self.allow_await, prop_name) - .parse(cursor); + .parse(parser); } - let pos = cursor + let pos = parser .peek(0) .map(|tok| tok.span().start()) .ok_or(ParseError::AbruptEnd)?; @@ -181,23 +181,23 @@ impl MethodDefinition { } } -impl TokenParser for MethodDefinition { +impl TokenParser for MethodDefinition { type Output = node::PropertyDefinition; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let (methodkind, prop_name, params) = match self.identifier.as_str() { idn @ "get" | idn @ "set" => { - let prop_name = cursor + let prop_name = parser .next() .map(Token::to_string) .ok_or(ParseError::AbruptEnd)?; - cursor.expect( + parser.expect( TokenKind::Punctuator(Punctuator::OpenParen), "property method definition", )?; - let first_param = cursor.peek(0).expect("current token disappeared").clone(); - let params = FormalParameters::new(false, false).parse(cursor)?; - cursor.expect(Punctuator::CloseParen, "method definition")?; + let first_param = parser.peek(0).expect("current token disappeared").clone(); + let params = FormalParameters::new(false, false).parse(parser)?; + parser.expect(Punctuator::CloseParen, "method definition")?; if idn == "get" { if !params.is_empty() { return Err(ParseError::unexpected( @@ -217,8 +217,8 @@ impl TokenParser for MethodDefinition { } } prop_name => { - let params = FormalParameters::new(false, false).parse(cursor)?; - cursor.expect(Punctuator::CloseParen, "method definition")?; + let params = FormalParameters::new(false, false).parse(parser)?; + parser.expect(Punctuator::CloseParen, "method definition")?; ( MethodDefinitionKind::Ordinary, prop_name.to_string(), @@ -227,12 +227,12 @@ impl TokenParser for MethodDefinition { } }; - cursor.expect( + parser.expect( TokenKind::Punctuator(Punctuator::OpenBlock), "property method definition", )?; - let body = FunctionBody::new(false, false).parse(cursor)?; - cursor.expect( + let body = FunctionBody::new(false, false).parse(parser)?; + parser.expect( TokenKind::Punctuator(Punctuator::CloseBlock), "property method definition", )?; @@ -278,11 +278,11 @@ impl Initializer { } } -impl TokenParser for Initializer { +impl TokenParser for Initializer { type Output = Node; - fn parse(self, cursor: &mut Cursor<'_>) -> ParseResult { - cursor.expect(TokenKind::Punctuator(Punctuator::Assign), "initializer")?; - AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await).parse(cursor) + fn parse(self, parser: &mut Parser) -> ParseResult { + parser.expect(TokenKind::Punctuator(Punctuator::Assign), "initializer")?; + AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await).parse(parser) } } diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 0b75ade88e5..5e44f15044d 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -52,32 +52,32 @@ impl TokenParser for UnaryExpression { type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { - let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; + let tok = parser.next().ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Delete) => { - Ok(node::UnaryOp::new(UnaryOp::Delete, self.parse(cursor)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Delete, self.parse(parser)?).into()) } TokenKind::Keyword(Keyword::Void) => { - Ok(node::UnaryOp::new(UnaryOp::Void, self.parse(cursor)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Void, self.parse(parser)?).into()) } TokenKind::Keyword(Keyword::TypeOf) => { - Ok(node::UnaryOp::new(UnaryOp::TypeOf, self.parse(cursor)?).into()) + Ok(node::UnaryOp::new(UnaryOp::TypeOf, self.parse(parser)?).into()) } TokenKind::Punctuator(Punctuator::Add) => { - Ok(node::UnaryOp::new(UnaryOp::Plus, self.parse(cursor)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Plus, self.parse(parser)?).into()) } TokenKind::Punctuator(Punctuator::Sub) => { - Ok(node::UnaryOp::new(UnaryOp::Minus, self.parse(cursor)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Minus, self.parse(parser)?).into()) } TokenKind::Punctuator(Punctuator::Neg) => { - Ok(node::UnaryOp::new(UnaryOp::Tilde, self.parse(cursor)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Tilde, self.parse(parser)?).into()) } TokenKind::Punctuator(Punctuator::Not) => { - Ok(node::UnaryOp::new(UnaryOp::Not, self.parse(cursor)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Not, self.parse(parser)?).into()) } _ => { - cursor.back(); - UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor) + parser.back(); + UpdateExpression::new(self.allow_yield, self.allow_await).parse(parser) } } } diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index c54e395a8d7..ab8a404a125 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -42,38 +42,38 @@ impl TokenParser for UpdateExpression { type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = parser.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Punctuator(Punctuator::Inc) => { - cursor.next().expect("token disappeared"); + parser.next().expect("token disappeared"); return Ok(node::UnaryOp::new( UnaryOp::IncrementPre, LeftHandSideExpression::new(self.allow_yield, self.allow_await) - .parse(cursor)?, + .parse(parser)?, ) .into()); } TokenKind::Punctuator(Punctuator::Dec) => { - cursor.next().expect("token disappeared"); + parser.next().expect("token disappeared"); return Ok(node::UnaryOp::new( UnaryOp::DecrementPre, LeftHandSideExpression::new(self.allow_yield, self.allow_await) - .parse(cursor)?, + .parse(parser)?, ) .into()); } _ => {} } - let lhs = LeftHandSideExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.peek(0) { + let lhs = LeftHandSideExpression::new(self.allow_yield, self.allow_await).parse(parser)?; + if let Some(tok) = parser.peek(0) { match tok.kind { TokenKind::Punctuator(Punctuator::Inc) => { - cursor.next().expect("token disappeared"); + parser.next().expect("token disappeared"); return Ok(node::UnaryOp::new(UnaryOp::IncrementPost, lhs).into()); } TokenKind::Punctuator(Punctuator::Dec) => { - cursor.next().expect("token disappeared"); + parser.next().expect("token disappeared"); return Ok(node::UnaryOp::new(UnaryOp::DecrementPost, lhs).into()); } _ => {} diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index bbabf04b4e0..863950e09e2 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -57,7 +57,7 @@ impl TokenParser for FormalParameters { fn parse(self, parser: &mut Parser) -> Result { let mut params = Vec::new(); - if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind + if parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind == TokenKind::Punctuator(Punctuator::CloseParen) { return Ok(params.into_boxed_slice()); @@ -66,14 +66,14 @@ impl TokenParser for FormalParameters { loop { let mut rest_param = false; - params.push(if cursor.next_if(Punctuator::Spread).is_some() { + params.push(if parser.next_if(Punctuator::Spread).is_some() { rest_param = true; - FunctionRestParameter::new(self.allow_yield, self.allow_await).parse(cursor)? + FunctionRestParameter::new(self.allow_yield, self.allow_await).parse(parser)? } else { - FormalParameter::new(self.allow_yield, self.allow_await).parse(cursor)? + FormalParameter::new(self.allow_yield, self.allow_await).parse(parser)? }); - if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind + if parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind == TokenKind::Punctuator(Punctuator::CloseParen) { break; @@ -81,7 +81,7 @@ impl TokenParser for FormalParameters { if rest_param { return Err(ParseError::unexpected( - cursor + parser .peek_prev() .expect("current token disappeared") .clone(), @@ -89,7 +89,7 @@ impl TokenParser for FormalParameters { )); } - cursor.expect(Punctuator::Comma, "parameter list")?; + parser.expect(Punctuator::Comma, "parameter list")?; } Ok(params.into_boxed_slice()) @@ -139,9 +139,9 @@ impl TokenParser for BindingRestElement { fn parse(self, parser: &mut Parser) -> Result { // FIXME: we are reading the spread operator before the rest element. - // cursor.expect(Punctuator::Spread, "rest parameter")?; + // parser.expect(Punctuator::Spread, "rest parameter")?; - let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; + let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; // TODO: BindingPattern Ok(Self::Output::new(param, None, true)) @@ -182,9 +182,9 @@ impl TokenParser for FormalParameter { fn parse(self, parser: &mut Parser) -> Result { // TODO: BindingPattern - let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; + let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; - let init = Initializer::new(true, self.allow_yield, self.allow_await).try_parse(cursor); + let init = Initializer::new(true, self.allow_yield, self.allow_await).try_parse(parser); Ok(Self::Output::new(param, init, false)) } @@ -228,12 +228,12 @@ impl TokenParser for FunctionStatementList { type Output = node::StatementList; fn parse(self, parser: &mut Parser) -> Result { - if let Some(tk) = cursor.peek(0) { + if let Some(tk) = parser.peek(0) { if tk.kind == Punctuator::CloseBlock.into() { return Ok(Vec::new().into()); } } - StatementList::new(self.allow_yield, self.allow_await, true, true).parse(cursor) + StatementList::new(self.allow_yield, self.allow_await, true, true).parse(parser) } } diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 8f29b4d5743..dc55b81619b 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -12,6 +12,7 @@ use crate::syntax::ast::node::StatementList; use crate::syntax::lexer::Token; use crate::syntax::lexer::Lexer; use crate::syntax::ast::Node; +use crate::syntax::lexer::InputElement; use ParseError as Error; @@ -106,16 +107,18 @@ impl Parser { lexer: Lexer::new(reader) } } -} -impl Iterator for Parser -where - R: Read, -{ - type Item = Result; + pub fn parse_all(&mut self) -> Result { + Script.parse(&mut self) + } - fn next(&mut self) -> Option { + pub fn next(&mut self) -> Result { + unimplemented!(); + } + + pub fn peek(&mut self, amount: i32) -> Result { + unimplemented!(); } } @@ -132,7 +135,7 @@ impl TokenParser for Script { type Output = StatementList; fn parse(self, parser: &mut Parser) -> Result { - if cursor.peek(0).is_some() { + if parser.peek().is_some() { ScriptBody.parse(parser) } else { Ok(StatementList::from(Vec::new())) diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 5c5baab1445..1db82b49c22 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -65,19 +65,19 @@ impl TokenParser for Block { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Block", "Parsing"); - cursor.expect(Punctuator::OpenBlock, "block")?; - if let Some(tk) = cursor.peek(0) { + parser.expect(Punctuator::OpenBlock, "block")?; + if let Some(tk) = parser.peek(0) { if tk.kind == TokenKind::Punctuator(Punctuator::CloseBlock) { - cursor.next(); + parser.next(); return Ok(node::Block::from(vec![])); } } let statement_list = StatementList::new(self.allow_yield, self.allow_await, self.allow_return, true) - .parse(cursor) + .parse(parser) .map(node::Block::from)?; - cursor.expect(Punctuator::CloseBlock, "block")?; + parser.expect(Punctuator::CloseBlock, "block")?; Ok(statement_list) } diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index 97f4cc953b4..9cfd96c36c7 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -54,20 +54,20 @@ impl TokenParser for BreakStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("BreakStatement", "Parsing"); - cursor.expect(Keyword::Break, "break statement")?; + parser.expect(Keyword::Break, "break statement")?; - let label = if let (true, tok) = cursor.peek_semicolon(false) { + let label = if let (true, tok) = parser.peek_semicolon(false) { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) => { - let _ = cursor.next(); + let _ = parser.next(); } _ => {} } None } else { - let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect_semicolon(false, "continue statement")?; + let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; + parser.expect_semicolon(false, "continue statement")?; Some(label) }; diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index a1d92997bdd..45314558ef5 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -54,20 +54,20 @@ impl TokenParser for ContinueStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ContinueStatement", "Parsing"); - cursor.expect(Keyword::Continue, "continue statement")?; + parser.expect(Keyword::Continue, "continue statement")?; - let label = if let (true, tok) = cursor.peek_semicolon(false) { + let label = if let (true, tok) = parser.peek_semicolon(false) { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) => { - let _ = cursor.next(); + let _ = parser.next(); } _ => {} } None } else { - let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect_semicolon(false, "continue statement")?; + let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; + parser.expect_semicolon(false, "continue statement")?; Some(label) }; diff --git a/boa/src/syntax/parser/statement/declaration/hoistable.rs b/boa/src/syntax/parser/statement/declaration/hoistable.rs index b51692160a2..4fac1f6a573 100644 --- a/boa/src/syntax/parser/statement/declaration/hoistable.rs +++ b/boa/src/syntax/parser/statement/declaration/hoistable.rs @@ -52,7 +52,7 @@ impl TokenParser for HoistableDeclaration { let _timer = BoaProfiler::global().start_event("HoistableDeclaration", "Parsing"); // TODO: check for generators and async functions + generators FunctionDeclaration::new(self.allow_yield, self.allow_await, self.is_default) - .parse(cursor) + .parse(parser) .map(Node::from) } } @@ -92,21 +92,21 @@ impl TokenParser for FunctionDeclaration { type Output = FunctionDecl; fn parse(self, parser: &mut Parser) -> Result { - cursor.expect(Keyword::Function, "function declaration")?; + parser.expect(Keyword::Function, "function declaration")?; // TODO: If self.is_default, then this can be empty. - let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; + let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect(Punctuator::OpenParen, "function declaration")?; + parser.expect(Punctuator::OpenParen, "function declaration")?; - let params = FormalParameters::new(false, false).parse(cursor)?; + let params = FormalParameters::new(false, false).parse(parser)?; - cursor.expect(Punctuator::CloseParen, "function declaration")?; - cursor.expect(Punctuator::OpenBlock, "function declaration")?; + parser.expect(Punctuator::CloseParen, "function declaration")?; + parser.expect(Punctuator::OpenBlock, "function declaration")?; - let body = FunctionBody::new(self.allow_yield, self.allow_await).parse(cursor)?; + let body = FunctionBody::new(self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect(Punctuator::CloseBlock, "function declaration")?; + parser.expect(Punctuator::CloseBlock, "function declaration")?; Ok(FunctionDecl::new(name, params, body)) } diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 8554ccda215..ae916af72cf 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -56,16 +56,16 @@ impl TokenParser for LexicalDeclaration { fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("LexicalDeclaration", "Parsing"); - let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; + let tok = parser.next().ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Const) => { BindingList::new(self.allow_in, self.allow_yield, self.allow_await, true) - .parse(cursor) + .parse(parser) } TokenKind::Keyword(Keyword::Let) => { BindingList::new(self.allow_in, self.allow_yield, self.allow_await, false) - .parse(cursor) + .parse(parser) } _ => unreachable!("unknown token found"), } @@ -118,7 +118,7 @@ impl TokenParser for BindingList { loop { let (ident, init) = LexicalBinding::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(cursor)?; + .parse(parser)?; if self.is_const { if let Some(init) = init { @@ -126,7 +126,7 @@ impl TokenParser for BindingList { } else { return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::Assign)], - cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), + parser.next().ok_or(ParseError::AbruptEnd)?.clone(), "const declaration", )); } @@ -134,10 +134,10 @@ impl TokenParser for BindingList { let_decls.push(LetDecl::new(ident, init)); } - match cursor.peek_semicolon(false) { + match parser.peek_semicolon(false) { (true, _) => break, (false, Some(tk)) if tk.kind == TokenKind::Punctuator(Punctuator::Comma) => { - let _ = cursor.next(); + let _ = parser.next(); } _ => { return Err(ParseError::expected( @@ -145,7 +145,7 @@ impl TokenParser for BindingList { TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), + parser.next().ok_or(ParseError::AbruptEnd)?.clone(), "lexical declaration", )) } @@ -192,9 +192,9 @@ impl TokenParser for LexicalBinding { type Output = (Box, Option); fn parse(self, parser: &mut Parser) -> Result { - let ident = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; + let ident = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; let initializer = - Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(cursor); + Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(parser); Ok((ident, initializer)) } diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index d32547c3826..38c7508f1c2 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -53,14 +53,14 @@ impl TokenParser for Declaration { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Declaration", "Parsing"); - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = parser.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Function) => { - HoistableDeclaration::new(self.allow_yield, self.allow_await, false).parse(cursor) + HoistableDeclaration::new(self.allow_yield, self.allow_await, false).parse(parser) } TokenKind::Keyword(Keyword::Const) | TokenKind::Keyword(Keyword::Let) => { - LexicalDeclaration::new(true, self.allow_yield, self.allow_await).parse(cursor) + LexicalDeclaration::new(true, self.allow_yield, self.allow_await).parse(parser) } _ => unreachable!("unknown token found"), } diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index bd428435663..47f6fe5be48 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -53,22 +53,22 @@ impl TokenParser for IfStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("IfStatement", "Parsing"); - cursor.expect(Keyword::If, "if statement")?; - cursor.expect(Punctuator::OpenParen, "if statement")?; + parser.expect(Keyword::If, "if statement")?; + parser.expect(Punctuator::OpenParen, "if statement")?; - let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect(Punctuator::CloseParen, "if statement")?; + parser.expect(Punctuator::CloseParen, "if statement")?; let then_stm = - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; - let else_stm = match cursor.peek(0) { + let else_stm = match parser.peek(0) { Some(else_tok) if else_tok.kind == TokenKind::Keyword(Keyword::Else) => { - cursor.next(); + parser.next(); Some( Statement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor)?, + .parse(parser)?, ) } _ => None, diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index b253394f406..2ae635169c0 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -59,12 +59,12 @@ impl TokenParser for DoWhileStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("DoWhileStatement", "Parsing"); - cursor.expect(Keyword::Do, "do while statement")?; + parser.expect(Keyword::Do, "do while statement")?; let body = - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = parser.peek(0).ok_or(ParseError::AbruptEnd)?; if next_token.kind != TokenKind::Keyword(Keyword::While) { return Err(ParseError::expected( @@ -74,13 +74,13 @@ impl TokenParser for DoWhileStatement { )); } - cursor.expect(Keyword::While, "do while statement")?; - cursor.expect(Punctuator::OpenParen, "do while statement")?; + parser.expect(Keyword::While, "do while statement")?; + parser.expect(Punctuator::OpenParen, "do while statement")?; - let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect(Punctuator::CloseParen, "do while statement")?; - cursor.expect_semicolon(true, "do while statement")?; + parser.expect(Punctuator::CloseParen, "do while statement")?; + parser.expect_semicolon(true, "do while statement")?; Ok(DoWhileLoop::new(body, cond)) } diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index 70f992488ff..74d8ba43ab2 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -64,37 +64,37 @@ impl TokenParser for ForStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ForStatement", "Parsing"); - cursor.expect(Keyword::For, "for statement")?; - cursor.expect(Punctuator::OpenParen, "for statement")?; + parser.expect(Keyword::For, "for statement")?; + parser.expect(Punctuator::OpenParen, "for statement")?; - let init = match cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind { + let init = match parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Keyword(Keyword::Var) => Some( VariableDeclarationList::new(false, self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .map(Node::from)?, ), TokenKind::Keyword(Keyword::Let) | TokenKind::Keyword(Keyword::Const) => { - Some(Declaration::new(self.allow_yield, self.allow_await).parse(cursor)?) + Some(Declaration::new(self.allow_yield, self.allow_await).parse(parser)?) } TokenKind::Punctuator(Punctuator::Semicolon) => None, - _ => Some(Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?), + _ => Some(Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?), }; - cursor.expect(Punctuator::Semicolon, "for statement")?; + parser.expect(Punctuator::Semicolon, "for statement")?; - let cond = if cursor.next_if(Punctuator::Semicolon).is_some() { + let cond = if parser.next_if(Punctuator::Semicolon).is_some() { Const::from(true).into() } else { - let step = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect(Punctuator::Semicolon, "for statement")?; + let step = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + parser.expect(Punctuator::Semicolon, "for statement")?; step }; - let step = if cursor.next_if(Punctuator::CloseParen).is_some() { + let step = if parser.next_if(Punctuator::CloseParen).is_some() { None } else { - let step = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect( + let step = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + parser.expect( TokenKind::Punctuator(Punctuator::CloseParen), "for statement", )?; @@ -102,7 +102,7 @@ impl TokenParser for ForStatement { }; let body = - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; // TODO: do not encapsulate the `for` in a block just to have an inner scope. Ok(ForLoop::new(init, cond, step, body)) diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index 72f4d445793..74bb1aa9e43 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -3,7 +3,7 @@ use crate::{ ast::{node::WhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, - Cursor, ParseError, TokenParser, + Parser, ParseError, TokenParser, }, }, BoaProfiler, @@ -44,20 +44,20 @@ impl WhileStatement { } } -impl TokenParser for WhileStatement { +impl TokenParser for WhileStatement { type Output = WhileLoop; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("WhileStatement", "Parsing"); - cursor.expect(Keyword::While, "while statement")?; - cursor.expect(Punctuator::OpenParen, "while statement")?; + parser.expect(Keyword::While, "while statement")?; + parser.expect(Punctuator::OpenParen, "while statement")?; - let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect(Punctuator::CloseParen, "while statement")?; + parser.expect(Punctuator::CloseParen, "while statement")?; let body = - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; Ok(WhileLoop::new(cond, body)) } diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index c57d4619966..c92352aafdf 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -97,38 +97,38 @@ impl TokenParser for Statement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Statement", "Parsing"); // TODO: add BreakableStatement and divide Whiles, fors and so on to another place. - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = parser.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::If) => { IfStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::Var) => { VariableStatement::new(self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::While) => { WhileStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::Do) => { DoWhileStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::For) => { ForStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::Return) => { if self.allow_return.0 { ReturnStatement::new(self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .map(Node::from) } else { Err(ParseError::unexpected(tok.clone(), "statement")) @@ -136,39 +136,39 @@ impl TokenParser for Statement { } TokenKind::Keyword(Keyword::Break) => { BreakStatement::new(self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::Continue) => { ContinueStatement::new(self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::Try) => { TryStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::Throw) => { ThrowStatement::new(self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Keyword(Keyword::Switch) => { SwitchStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor) + .parse(parser) .map(Node::from) } TokenKind::Punctuator(Punctuator::OpenBlock) => { BlockStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor) + .parse(parser) .map(Node::from) } // TODO: https://tc39.es/ecma262/#prod-LabelledStatement // TokenKind::Punctuator(Punctuator::Semicolon) => { // return Ok(Node::new(NodeBase::Nope, tok.pos)) // } - _ => ExpressionStatement::new(self.allow_yield, self.allow_await).parse(cursor), + _ => ExpressionStatement::new(self.allow_yield, self.allow_await).parse(parser), } } } @@ -219,7 +219,7 @@ impl TokenParser for StatementList { let mut items = Vec::new(); loop { - match cursor.peek(0) { + match parser.peek(0) { Some(token) if token.kind == TokenKind::Punctuator(Punctuator::CloseBlock) => { if self.break_when_closingbrase { break; @@ -239,11 +239,11 @@ impl TokenParser for StatementList { let item = StatementListItem::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor)?; + .parse(parser)?; items.push(item); // move the cursor forward for any consecutive semicolon. - while cursor.next_if(Punctuator::Semicolon).is_some() {} + while parser.next_if(Punctuator::Semicolon).is_some() {} } items.sort_by(Node::hoistable_order); @@ -290,16 +290,16 @@ impl TokenParser for StatementListItem { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("StatementListItem", "Parsing"); - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = parser.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Function) | TokenKind::Keyword(Keyword::Const) | TokenKind::Keyword(Keyword::Let) => { - Declaration::new(self.allow_yield, self.allow_await).parse(cursor) + Declaration::new(self.allow_yield, self.allow_await).parse(parser) } _ => { - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor) + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser) } } } @@ -337,9 +337,9 @@ impl TokenParser for ExpressionStatement { fn parse(self, parser: &mut Parser) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExpressionStatement", "Parsing"); // TODO: lookahead - let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect_semicolon(false, "expression statement")?; + parser.expect_semicolon(false, "expression statement")?; Ok(expr) } @@ -388,7 +388,7 @@ impl TokenParser for BindingIdentifier { let _timer = BoaProfiler::global().start_event("BindingIdentifier", "Parsing"); // TODO: strict mode. - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; + let next_token = parser.next().ok_or(ParseError::AbruptEnd)?; match next_token.kind { TokenKind::Identifier(ref s) => Ok(s.clone()), diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index db5cca20703..beebcf8c190 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -43,15 +43,15 @@ impl TokenParser for ReturnStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ReturnStatement", "Parsing"); - cursor.expect(Keyword::Return, "return statement")?; + parser.expect(Keyword::Return, "return statement")?; - if let (true, tok) = cursor.peek_semicolon(false) { + if let (true, tok) = parser.peek_semicolon(false) { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) || tok.kind == TokenKind::LineTerminator => { - let _ = cursor.next(); + let _ = parser.next(); } _ => {} } @@ -59,9 +59,9 @@ impl TokenParser for ReturnStatement { return Ok(Return::new::>(None)); } - let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect_semicolon(false, "return statement")?; + parser.expect_semicolon(false, "return statement")?; Ok(Return::new(expr)) } diff --git a/boa/src/syntax/parser/statement/switch/mod.rs b/boa/src/syntax/parser/statement/switch/mod.rs index 703efa0f7e4..0605a0feea0 100644 --- a/boa/src/syntax/parser/statement/switch/mod.rs +++ b/boa/src/syntax/parser/statement/switch/mod.rs @@ -51,15 +51,15 @@ impl TokenParser for SwitchStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("SwitchStatement", "Parsing"); - cursor.expect(Keyword::Switch, "switch statement")?; - cursor.expect(Punctuator::OpenParen, "switch statement")?; + parser.expect(Keyword::Switch, "switch statement")?; + parser.expect(Punctuator::OpenParen, "switch statement")?; - let condition = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + let condition = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect(Punctuator::CloseParen, "switch statement")?; + parser.expect(Punctuator::CloseParen, "switch statement")?; let (cases, default) = - CaseBlock::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; + CaseBlock::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; Ok(Switch::new(condition, cases, default)) } @@ -98,7 +98,7 @@ impl TokenParser for CaseBlock { type Output = (Box<[Case]>, Option); fn parse(self, parser: &mut Parser) -> Result { - cursor.expect(Punctuator::OpenBlock, "switch case block")?; + parser.expect(Punctuator::OpenBlock, "switch case block")?; // CaseClauses[?Yield, ?Await, ?Return]opt // CaseClauses[?Yield, ?Await, ?Return]optDefaultClause[?Yield, ?Await, ?Return]CaseClauses[?Yield, ?Await, ?Return]opt diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index 29aa3fa2b55..71640337857 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -43,14 +43,14 @@ impl TokenParser for ThrowStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("ThrowStatement", "Parsing"); - cursor.expect(Keyword::Throw, "throw statement")?; + parser.expect(Keyword::Throw, "throw statement")?; - cursor.peek_expect_no_lineterminator(0)?; + parser.peek_expect_no_lineterminator(0)?; - let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.peek(0) { + let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + if let Some(tok) = parser.peek(0) { if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) { - let _ = cursor.next(); + let _ = parser.next(); } } diff --git a/boa/src/syntax/parser/statement/try_stm/catch.rs b/boa/src/syntax/parser/statement/try_stm/catch.rs index 098febddf24..47f140fa7ba 100644 --- a/boa/src/syntax/parser/statement/try_stm/catch.rs +++ b/boa/src/syntax/parser/statement/try_stm/catch.rs @@ -6,7 +6,7 @@ use crate::{ }, parser::{ statement::{block::Block, BindingIdentifier}, - AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, + AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser, }, }, BoaProfiler, @@ -48,11 +48,11 @@ impl TokenParser for Catch { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Catch", "Parsing"); - cursor.expect(Keyword::Catch, "try statement")?; - let catch_param = if cursor.next_if(Punctuator::OpenParen).is_some() { + parser.expect(Keyword::Catch, "try statement")?; + let catch_param = if parser.next_if(Punctuator::OpenParen).is_some() { let catch_param = - CatchParameter::new(self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect(Punctuator::CloseParen, "catch in try statement")?; + CatchParameter::new(self.allow_yield, self.allow_await).parse(parser)?; + parser.expect(Punctuator::CloseParen, "catch in try statement")?; Some(catch_param) } else { None @@ -61,7 +61,7 @@ impl TokenParser for Catch { // Catch block Ok(node::Catch::new::<_, Identifier, _>( catch_param, - Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?, + Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?, )) } } @@ -94,13 +94,13 @@ impl CatchParameter { } } -impl TokenParser for CatchParameter { +impl TokenParser for CatchParameter { type Output = Identifier; - fn parse(self, cursor: &mut Cursor<'_>) -> Result { + fn parse(self, parser: &mut Parser) -> Result { // TODO: should accept BindingPattern BindingIdentifier::new(self.allow_yield, self.allow_await) - .parse(cursor) + .parse(parser) .map(Identifier::from) } } diff --git a/boa/src/syntax/parser/statement/try_stm/finally.rs b/boa/src/syntax/parser/statement/try_stm/finally.rs index 6fdfaa916a0..a3a833068ad 100644 --- a/boa/src/syntax/parser/statement/try_stm/finally.rs +++ b/boa/src/syntax/parser/statement/try_stm/finally.rs @@ -2,7 +2,7 @@ use crate::{ syntax::{ ast::{node, Keyword}, parser::{ - statement::block::Block, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, + statement::block::Block, AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser, }, }, @@ -45,10 +45,10 @@ impl TokenParser for Finally { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("Finally", "Parsing"); - cursor.expect(Keyword::Finally, "try statement")?; + parser.expect(Keyword::Finally, "try statement")?; Ok( Block::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor)? + .parse(parser)? .into(), ) } diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index e33af12c832..5d60d137e11 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -53,12 +53,12 @@ impl TokenParser for TryStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("TryStatement", "Parsing"); // TRY - cursor.expect(Keyword::Try, "try statement")?; + parser.expect(Keyword::Try, "try statement")?; let try_clause = - Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; + Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = parser.peek(0).ok_or(ParseError::AbruptEnd)?; if next_token.kind != TokenKind::Keyword(Keyword::Catch) && next_token.kind != TokenKind::Keyword(Keyword::Finally) @@ -74,17 +74,17 @@ impl TokenParser for TryStatement { } let catch = if next_token.kind == TokenKind::Keyword(Keyword::Catch) { - Some(Catch::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?) + Some(Catch::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?) } else { None }; - let next_token = cursor.peek(0); + let next_token = parser.peek(0); let finally_block = match next_token { Some(token) => match token.kind { TokenKind::Keyword(Keyword::Finally) => Some( Finally::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor)?, + .parse(parser)?, ), _ => None, }, diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 31152bcbb60..7c4115be1f2 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -49,12 +49,12 @@ impl TokenParser for VariableStatement { fn parse(self, parser: &mut Parser) -> Result { let _timer = BoaProfiler::global().start_event("VariableStatement", "Parsing"); - cursor.expect(Keyword::Var, "variable statement")?; + parser.expect(Keyword::Var, "variable statement")?; let decl_list = - VariableDeclarationList::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + VariableDeclarationList::new(true, self.allow_yield, self.allow_await).parse(parser)?; - cursor.expect_semicolon(false, "variable statement")?; + parser.expect_semicolon(false, "variable statement")?; Ok(decl_list) } @@ -104,13 +104,13 @@ impl TokenParser for VariableDeclarationList { loop { list.push( VariableDeclaration::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(cursor)?, + .parse(parser)?, ); - match cursor.peek_semicolon(false) { + match parser.peek_semicolon(false) { (true, _) => break, (false, Some(tk)) if tk.kind == TokenKind::Punctuator(Punctuator::Comma) => { - let _ = cursor.next(); + let _ = parser.next(); } _ => { return Err(ParseError::expected( @@ -118,7 +118,7 @@ impl TokenParser for VariableDeclarationList { TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), + parser.next().ok_or(ParseError::AbruptEnd)?.clone(), "lexical declaration", )) } @@ -164,10 +164,10 @@ impl TokenParser for VariableDeclaration { fn parse(self, parser: &mut Parser) -> Result { // TODO: BindingPattern - let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; + let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; let ident = - Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(cursor); + Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(parser); Ok(VarDecl::new(name, ident)) } From 921a45eb0e69eae6d31bc4ced54c2eede79454fe Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 21 Jun 2020 23:06:24 +0000 Subject: [PATCH 061/289] Updated usage of R as R: read --- .../expression/assignment/arrow_function.rs | 17 +++++-- .../expression/assignment/conditional.rs | 9 +++- .../expression/assignment/exponentiation.rs | 51 +++++++++---------- .../parser/expression/assignment/mod.rs | 9 +++- .../expression/left_hand_side/arguments.rs | 9 +++- .../parser/expression/left_hand_side/call.rs | 9 +++- .../expression/left_hand_side/member.rs | 9 +++- .../parser/expression/left_hand_side/mod.rs | 9 +++- boa/src/syntax/parser/expression/mod.rs | 9 +++- .../primary/array_initializer/mod.rs | 9 +++- .../expression/primary/function_expression.rs | 9 +++- .../syntax/parser/expression/primary/mod.rs | 9 +++- .../primary/object_initializer/mod.rs | 23 +++++++-- boa/src/syntax/parser/expression/unary.rs | 11 ++-- boa/src/syntax/parser/expression/update.rs | 9 +++- boa/src/syntax/parser/function/mod.rs | 24 +++++++-- boa/src/syntax/parser/mod.rs | 25 +++++---- boa/src/syntax/parser/statement/block/mod.rs | 9 +++- .../syntax/parser/statement/break_stm/mod.rs | 9 +++- .../parser/statement/continue_stm/mod.rs | 9 +++- .../parser/statement/declaration/hoistable.rs | 14 +++-- .../parser/statement/declaration/lexical.rs | 19 +++++-- .../parser/statement/declaration/mod.rs | 9 +++- boa/src/syntax/parser/statement/if_stm/mod.rs | 9 +++- .../statement/iteration/do_while_statement.rs | 8 ++- .../statement/iteration/for_statement.rs | 9 +++- .../statement/iteration/while_statement.rs | 9 +++- boa/src/syntax/parser/statement/mod.rs | 29 ++++++++--- .../syntax/parser/statement/return_stm/mod.rs | 9 +++- boa/src/syntax/parser/statement/switch/mod.rs | 14 +++-- boa/src/syntax/parser/statement/throw/mod.rs | 9 +++- .../syntax/parser/statement/try_stm/catch.rs | 14 +++-- .../parser/statement/try_stm/finally.rs | 9 +++- .../syntax/parser/statement/try_stm/mod.rs | 9 +++- boa/src/syntax/parser/statement/variable.rs | 18 +++++-- 35 files changed, 343 insertions(+), 122 deletions(-) diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index 6c24a36d1a9..3b4b46b6fb9 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -25,6 +25,8 @@ use crate::{ BoaProfiler, }; +use std::io::Read; + /// Arrow function parsing. /// /// More information: @@ -60,7 +62,10 @@ impl ArrowFunction { } } -impl TokenParser for ArrowFunction { +impl TokenParser for ArrowFunction +where + R: Read, +{ type Output = ArrowFunctionDecl; fn parse(self, parser: &mut Parser) -> Result { @@ -107,7 +112,10 @@ impl ConciseBody { } } -impl TokenParser for ConciseBody { +impl TokenParser for ConciseBody +where + R: Read, +{ type Output = StatementList; fn parse(self, parser: &mut Parser) -> Result { @@ -147,7 +155,10 @@ impl ExpressionBody { } } -impl TokenParser for ExpressionBody { +impl TokenParser for ExpressionBody +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index 19c7b530903..9a34bc05287 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -13,12 +13,14 @@ use crate::{ ast::{node::ConditionalOp, Node, Punctuator}, parser::{ expression::{AssignmentExpression, LogicalORExpression}, - AllowAwait, AllowIn, AllowYield, Parser, ParseResult, TokenParser, + AllowAwait, AllowIn, AllowYield, ParseResult, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Conditional expression parsing. /// /// More information: @@ -54,7 +56,10 @@ impl ConditionalExpression { } } -impl TokenParser for ConditionalExpression { +impl TokenParser for ConditionalExpression +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 5375ac4513c..3697a579fac 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -17,7 +17,7 @@ use crate::{ }, parser::{ expression::{unary::UnaryExpression, update::UpdateExpression}, - AllowAwait, AllowYield, Parser, ParseResult, TokenParser, + AllowAwait, AllowYield, ParseResult, Parser, TokenParser, }, }, BoaProfiler, @@ -34,15 +34,12 @@ use std::io::Read; /// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Arithmetic_Operators#Exponentiation /// [spec]: https://tc39.es/ecma262/#prod-ExponentiationExpression #[derive(Debug, Clone, Copy)] -pub(in crate::syntax::parser::expression) struct ExponentiationExpression { +pub(in crate::syntax::parser::expression) struct ExponentiationExpression { allow_yield: AllowYield, allow_await: AllowAwait, } -impl ExponentiationExpression -where - R: Read -{ +impl ExponentiationExpression { /// Creates a new `ExponentiationExpression` parser. pub(in crate::syntax::parser::expression) fn new(allow_yield: Y, allow_await: A) -> Self where @@ -54,32 +51,32 @@ where allow_await: allow_await.into(), } } +} - /// Checks by looking at the next token to see whether it's a unary operator or not. - fn is_unary_expression(parser: &mut Parser) -> bool - where - R: Read - { - if let Some(tok) = parser.peek(0) { - match tok.kind { - TokenKind::Keyword(Keyword::Delete) - | TokenKind::Keyword(Keyword::Void) - | TokenKind::Keyword(Keyword::TypeOf) - | TokenKind::Punctuator(Punctuator::Add) - | TokenKind::Punctuator(Punctuator::Sub) - | TokenKind::Punctuator(Punctuator::Not) - | TokenKind::Punctuator(Punctuator::Neg) => true, - _ => false, - } - } else { - false +/// Checks by looking at the next token to see whether it's a unary operator or not. +fn is_unary_expression(parser: &mut Parser) -> bool +where + R: Read, +{ + if let Some(tok) = parser.peek(0) { + match tok.kind { + TokenKind::Keyword(Keyword::Delete) + | TokenKind::Keyword(Keyword::Void) + | TokenKind::Keyword(Keyword::TypeOf) + | TokenKind::Punctuator(Punctuator::Add) + | TokenKind::Punctuator(Punctuator::Sub) + | TokenKind::Punctuator(Punctuator::Not) + | TokenKind::Punctuator(Punctuator::Neg) => true, + _ => false, } + } else { + false } } -impl TokenParser for ExponentiationExpression -where - R: Read +impl TokenParser for ExponentiationExpression +where + R: Read, { type Output = Node; diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index b0d50191f35..6e3c1913963 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -19,12 +19,14 @@ use crate::{ node::{Assign, BinOp, Node}, Keyword, Punctuator, }, - parser::{AllowAwait, AllowIn, AllowYield, Parser, ParseError, ParseResult, TokenParser}, + parser::{AllowAwait, AllowIn, AllowYield, ParseError, ParseResult, Parser, TokenParser}, }, BoaProfiler, }; pub(super) use exponentiation::ExponentiationExpression; +use std::io::Read; + /// Assignment expression parsing. /// /// This can be one of the following: @@ -70,7 +72,10 @@ impl AssignmentExpression { } } -impl TokenParser for AssignmentExpression { +impl TokenParser for AssignmentExpression +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index 7c7b180ce37..e2b6e324d30 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -12,13 +12,15 @@ use crate::{ syntax::{ ast::{node::Spread, Node, Punctuator}, parser::{ - expression::AssignmentExpression, AllowAwait, AllowYield, Parser, ParseError, + expression::AssignmentExpression, AllowAwait, AllowYield, ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Parses a list of arguments. /// /// More information: @@ -47,7 +49,10 @@ impl Arguments { } } -impl TokenParser for Arguments { +impl TokenParser for Arguments +where + R: Read, +{ type Output = Box<[Node]>; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index f0dce2b1fd7..5ac118ecc85 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -19,13 +19,15 @@ use crate::{ Punctuator, }, parser::{ - expression::Expression, AllowAwait, AllowYield, Parser, ParseError, ParseResult, + expression::Expression, AllowAwait, AllowYield, ParseError, ParseResult, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Parses a call expression. /// /// More information: @@ -54,7 +56,10 @@ impl CallExpression { } } -impl TokenParser for CallExpression { +impl TokenParser for CallExpression +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 7650cca9aea..c3437bd568b 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -18,12 +18,14 @@ use crate::{ }, parser::{ expression::{primary::PrimaryExpression, Expression}, - AllowAwait, AllowYield, Parser, ParseError, ParseResult, TokenParser, + AllowAwait, AllowYield, ParseError, ParseResult, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Parses a member expression. /// /// More information: @@ -50,7 +52,10 @@ impl MemberExpression { } } -impl TokenParser for MemberExpression { +impl TokenParser for MemberExpression +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index f89931dadca..b1f41d1423f 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -16,11 +16,13 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Node, Punctuator}, - parser::{AllowAwait, AllowYield, Parser, ParseResult, TokenParser}, + parser::{AllowAwait, AllowYield, ParseResult, Parser, TokenParser}, }, BoaProfiler, }; +use std::io::Read; + /// Parses a left hand side expression. /// /// More information: @@ -49,7 +51,10 @@ impl LeftHandSideExpression { } } -impl TokenParser for LeftHandSideExpression { +impl TokenParser for LeftHandSideExpression +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 15ffb39ad2e..3cb17048ed9 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -17,7 +17,7 @@ mod update; use self::assignment::ExponentiationExpression; pub(super) use self::{assignment::AssignmentExpression, primary::Initializer}; -use super::{AllowAwait, AllowIn, AllowYield, Parser, ParseResult, TokenParser}; +use super::{AllowAwait, AllowIn, AllowYield, ParseResult, Parser, TokenParser}; use crate::syntax::lexer::TokenKind; use crate::{ profiler::BoaProfiler, @@ -27,6 +27,8 @@ use crate::{ }, }; +use std::io::Read; + // For use in the expression! macro to allow for both Punctuator and Keyword parameters. // Always returns false. impl PartialEq for Punctuator { @@ -51,7 +53,10 @@ impl PartialEq for Keyword { /// /// Those exressions are divided by the punctuators passed as the third parameter. macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $low_param:ident ),*] ) => { - impl TokenParser for $name { + impl TokenParser for $name + where + R: Read + { type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index f1df3689898..624c85167d7 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -17,13 +17,15 @@ use crate::{ Const, Punctuator, }, parser::{ - expression::AssignmentExpression, AllowAwait, AllowYield, Parser, ParseError, + expression::AssignmentExpression, AllowAwait, AllowYield, ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Parses an array literal. /// /// More information: @@ -52,7 +54,10 @@ impl ArrayLiteral { } } -impl TokenParser for ArrayLiteral { +impl TokenParser for ArrayLiteral +where + R: Read, +{ type Output = ArrayDecl; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/expression/primary/function_expression.rs b/boa/src/syntax/parser/expression/primary/function_expression.rs index 542b9702b4d..fcb1699743b 100644 --- a/boa/src/syntax/parser/expression/primary/function_expression.rs +++ b/boa/src/syntax/parser/expression/primary/function_expression.rs @@ -13,12 +13,14 @@ use crate::{ parser::{ function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - Parser, ParseError, TokenParser, + ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Function expression parsing. /// /// More information: @@ -30,7 +32,10 @@ use crate::{ #[derive(Debug, Clone, Copy)] pub(super) struct FunctionExpression; -impl TokenParser for FunctionExpression { +impl TokenParser for FunctionExpression +where + R: Read, +{ type Output = FunctionExpr; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 54264ba4a25..30e33094dea 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -24,10 +24,12 @@ use crate::syntax::{ node::{Call, Identifier, New, Node}, Const, Keyword, Punctuator, }, - parser::{AllowAwait, AllowYield, Parser, ParseError, ParseResult, TokenParser}, + parser::{AllowAwait, AllowYield, ParseError, ParseResult, Parser, TokenParser}, }; pub(in crate::syntax::parser) use object_initializer::Initializer; +use std::io::Read; + /// Parses a primary expression. /// /// More information: @@ -56,7 +58,10 @@ impl PrimaryExpression { } } -impl TokenParser for PrimaryExpression { +impl TokenParser for PrimaryExpression +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 8e76501720c..57b02a6c987 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -19,11 +19,12 @@ use crate::{ parser::{ expression::AssignmentExpression, function::{FormalParameters, FunctionBody}, - AllowAwait, AllowIn, AllowYield, Parser, ParseError, ParseResult, TokenParser, + AllowAwait, AllowIn, AllowYield, ParseError, ParseResult, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; /// Parses an object literal. /// @@ -53,7 +54,10 @@ impl ObjectLiteral { } } -impl TokenParser for ObjectLiteral { +impl TokenParser for ObjectLiteral +where + R: Read, +{ type Output = Object; fn parse(self, parser: &mut Parser) -> Result { @@ -115,7 +119,10 @@ impl PropertyDefinition { } } -impl TokenParser for PropertyDefinition { +impl TokenParser for PropertyDefinition +where + R: Read, +{ type Output = node::PropertyDefinition; fn parse(self, parser: &mut Parser) -> Result { @@ -181,7 +188,10 @@ impl MethodDefinition { } } -impl TokenParser for MethodDefinition { +impl TokenParser for MethodDefinition +where + R: Read, +{ type Output = node::PropertyDefinition; fn parse(self, parser: &mut Parser) -> Result { @@ -278,7 +288,10 @@ impl Initializer { } } -impl TokenParser for Initializer { +impl TokenParser for Initializer +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 5e44f15044d..0931eb165a7 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -15,11 +15,13 @@ use crate::syntax::{ Keyword, Punctuator, }, parser::{ - expression::update::UpdateExpression, AllowAwait, AllowYield, Parser, ParseError, - ParseResult, TokenParser, + expression::update::UpdateExpression, AllowAwait, AllowYield, ParseError, ParseResult, + Parser, TokenParser, }, }; +use std::io::Read; + /// Parses a unary expression. /// /// More information: @@ -48,7 +50,10 @@ impl UnaryExpression { } } -impl TokenParser for UnaryExpression { +impl TokenParser for UnaryExpression +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index ab8a404a125..e8f837392a2 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -9,9 +9,11 @@ use super::left_hand_side::LeftHandSideExpression; use crate::syntax::lexer::TokenKind; use crate::syntax::{ ast::{node, op::UnaryOp, Node, Punctuator}, - parser::{AllowAwait, AllowYield, Parser, ParseError, ParseResult, TokenParser}, + parser::{AllowAwait, AllowYield, ParseError, ParseResult, Parser, TokenParser}, }; +use std::io::Read; + /// Parses an update expression. /// /// More information: @@ -38,7 +40,10 @@ impl UpdateExpression { } } -impl TokenParser for UpdateExpression { +impl TokenParser for UpdateExpression +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 863950e09e2..5f6e764d8ee 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -19,10 +19,12 @@ use crate::syntax::{ parser::{ expression::Initializer, statement::{BindingIdentifier, StatementList}, - AllowAwait, AllowYield, Parser, ParseError, TokenParser, + AllowAwait, AllowYield, ParseError, Parser, TokenParser, }, }; +use std::io::Read; + /// Formal parameters parsing. /// /// More information: @@ -51,7 +53,10 @@ impl FormalParameters { } } -impl TokenParser for FormalParameters { +impl TokenParser for FormalParameters +where + R: Read, +{ type Output = Box<[node::FormalParameter]>; fn parse(self, parser: &mut Parser) -> Result { @@ -134,7 +139,10 @@ impl BindingRestElement { } } -impl TokenParser for BindingRestElement { +impl TokenParser for BindingRestElement +where + R: Read, +{ type Output = node::FormalParameter; fn parse(self, parser: &mut Parser) -> Result { @@ -176,7 +184,10 @@ impl FormalParameter { } } -impl TokenParser for FormalParameter { +impl TokenParser for FormalParameter +where + R: Read, +{ type Output = node::FormalParameter; fn parse(self, parser: &mut Parser) -> Result { @@ -224,7 +235,10 @@ impl FunctionStatementList { } } -impl TokenParser for FunctionStatementList { +impl TokenParser for FunctionStatementList +where + R: Read, +{ type Output = node::StatementList; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index dc55b81619b..d7a8e71d5be 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -9,10 +9,10 @@ mod tests; use self::error::{ParseError, ParseResult}; use crate::syntax::ast::node::StatementList; -use crate::syntax::lexer::Token; -use crate::syntax::lexer::Lexer; use crate::syntax::ast::Node; use crate::syntax::lexer::InputElement; +use crate::syntax::lexer::Lexer; +use crate::syntax::lexer::Token; use ParseError as Error; @@ -21,9 +21,9 @@ use std::io::Read; /// Trait implemented by parsers. /// /// This makes it possible to abstract over the underlying implementation of a parser. -trait TokenParser: Sized -where - R: Read +trait TokenParser: Sized +where + R: Read, { /// Output type for the parser. type Output; // = Node; waiting for https://github.com/rust-lang/rust/issues/29661 @@ -96,7 +96,7 @@ impl From for AllowDefault { } #[derive(Debug)] -pub struct Parser { +pub struct Parser { /// Lexer used to get tokens for the parser. lexer: Lexer, } @@ -104,7 +104,7 @@ pub struct Parser { impl Parser { pub fn new(reader: R) -> Self { Self { - lexer: Lexer::new(reader) + lexer: Lexer::new(reader), } } @@ -112,7 +112,6 @@ impl Parser { Script.parse(&mut self) } - pub fn next(&mut self) -> Result { unimplemented!(); } @@ -131,7 +130,10 @@ impl Parser { #[derive(Debug, Clone, Copy)] pub struct Script; -impl TokenParser for Script { +impl TokenParser for Script +where + R: Read, +{ type Output = StatementList; fn parse(self, parser: &mut Parser) -> Result { @@ -152,7 +154,10 @@ impl TokenParser for Script { #[derive(Debug, Clone, Copy)] pub struct ScriptBody; -impl TokenParser for ScriptBody { +impl TokenParser for ScriptBody +where + R: Read, +{ type Output = StatementList; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 1db82b49c22..7b58a395a48 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -17,10 +17,12 @@ use crate::{ profiler::BoaProfiler, syntax::{ ast::{node, Punctuator}, - parser::{AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser}, + parser::{AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser}, }, }; +use std::io::Read; + /// A `BlockStatement` is equivalent to a `Block`. /// /// More information: @@ -60,7 +62,10 @@ impl Block { } } -impl TokenParser for Block { +impl TokenParser for Block +where + R: Read, +{ type Output = node::Block; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index 9cfd96c36c7..fcfcb76c164 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -16,11 +16,13 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Break, Keyword, Punctuator}, - parser::{AllowAwait, AllowYield, Parser, ParseError, TokenParser}, + parser::{AllowAwait, AllowYield, ParseError, Parser, TokenParser}, }, BoaProfiler, }; +use std::io::Read; + /// Break statement parsing /// /// More information: @@ -49,7 +51,10 @@ impl BreakStatement { } } -impl TokenParser for BreakStatement { +impl TokenParser for BreakStatement +where + R: Read, +{ type Output = Break; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index 45314558ef5..6ce16fab00b 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -15,12 +15,14 @@ use crate::{ syntax::{ ast::{node::Continue, Keyword, Punctuator}, parser::{ - statement::LabelIdentifier, AllowAwait, AllowYield, Parser, ParseError, TokenParser, + statement::LabelIdentifier, AllowAwait, AllowYield, ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// For statement parsing /// /// More information: @@ -49,7 +51,10 @@ impl ContinueStatement { } } -impl TokenParser for ContinueStatement { +impl TokenParser for ContinueStatement +where + R: Read, +{ type Output = Continue; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/declaration/hoistable.rs b/boa/src/syntax/parser/statement/declaration/hoistable.rs index 4fac1f6a573..bc9b6e1f092 100644 --- a/boa/src/syntax/parser/statement/declaration/hoistable.rs +++ b/boa/src/syntax/parser/statement/declaration/hoistable.rs @@ -10,12 +10,14 @@ use crate::{ ast::{node::FunctionDecl, Keyword, Node, Punctuator}, parser::{ function::FormalParameters, function::FunctionBody, statement::BindingIdentifier, - AllowAwait, AllowDefault, AllowYield, Parser, ParseError, ParseResult, TokenParser, + AllowAwait, AllowDefault, AllowYield, ParseError, ParseResult, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Hoistable declaration parsing. /// /// More information: @@ -45,7 +47,10 @@ impl HoistableDeclaration { } } -impl TokenParser for HoistableDeclaration { +impl TokenParser for HoistableDeclaration +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { @@ -88,7 +93,10 @@ impl FunctionDeclaration { } } -impl TokenParser for FunctionDeclaration { +impl TokenParser for FunctionDeclaration +where + R: Read, +{ type Output = FunctionDecl; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index ae916af72cf..e01db5ed27f 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -16,12 +16,14 @@ use crate::{ }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, - Parser, ParseError, ParseResult, TokenParser, + ParseError, ParseResult, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Parses a lexical declaration. /// /// More information: @@ -51,7 +53,10 @@ impl LexicalDeclaration { } } -impl TokenParser for LexicalDeclaration { +impl TokenParser for LexicalDeclaration +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { @@ -106,7 +111,10 @@ impl BindingList { } } -impl TokenParser for BindingList { +impl TokenParser for BindingList +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { @@ -188,7 +196,10 @@ impl LexicalBinding { } } -impl TokenParser for LexicalBinding { +impl TokenParser for LexicalBinding +where + R: Read, +{ type Output = (Box, Option); fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index 38c7508f1c2..69257e47391 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -18,11 +18,13 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Keyword, Node}, - parser::{AllowAwait, AllowYield, Parser, ParseError, TokenParser}, + parser::{AllowAwait, AllowYield, ParseError, Parser, TokenParser}, }, BoaProfiler, }; +use std::io::Read; + /// Parses a declaration. /// /// More information: @@ -48,7 +50,10 @@ impl Declaration { } } -impl TokenParser for Declaration { +impl TokenParser for Declaration +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index 47f6fe5be48..236997b2e87 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -8,13 +8,15 @@ use crate::{ syntax::{ ast::{node::If, Keyword, Node, Punctuator}, parser::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, Parser, ParseError, + expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// If statement parsing. /// /// An _If_ statement will have a condition, a block statemet, and an optional _else_ statement. @@ -48,7 +50,10 @@ impl IfStatement { } } -impl TokenParser for IfStatement { +impl TokenParser for IfStatement +where + R: Read, +{ type Output = If; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 2ae635169c0..c0f1042e8d1 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -13,11 +13,12 @@ use crate::{ ast::{node::DoWhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, - Parser, ParseError, TokenParser, + ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; /// Do...while statement parsing /// @@ -54,7 +55,10 @@ impl DoWhileStatement { } } -impl TokenParser for DoWhileStatement { +impl TokenParser for DoWhileStatement +where + R: Read, +{ type Output = DoWhileLoop; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index 74d8ba43ab2..df981443028 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -18,12 +18,14 @@ use crate::{ expression::Expression, statement::declaration::Declaration, statement::{variable::VariableDeclarationList, Statement}, - AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser, + AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// For statement parsing /// /// More information: @@ -59,7 +61,10 @@ impl ForStatement { } } -impl TokenParser for ForStatement { +impl TokenParser for ForStatement +where + R: Read, +{ type Output = ForLoop; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index 74bb1aa9e43..61e5280969a 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -3,12 +3,14 @@ use crate::{ ast::{node::WhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, - Parser, ParseError, TokenParser, + ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// While statement parsing /// /// More information: @@ -44,7 +46,10 @@ impl WhileStatement { } } -impl TokenParser for WhileStatement { +impl TokenParser for WhileStatement +where + R: Read, +{ type Output = WhileLoop; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index c92352aafdf..f8d6a5b6a94 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -33,7 +33,7 @@ use self::{ variable::VariableStatement, }; use super::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, Parser, ParseError, ParseResult, + expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, ParseResult, Parser, TokenParser, }; @@ -43,6 +43,8 @@ use crate::{ BoaProfiler, }; +use std::io::Read; + /// Statement parsing. /// /// This can be one of the following: @@ -91,7 +93,10 @@ impl Statement { } } -impl TokenParser for Statement { +impl TokenParser for Statement +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> Result { @@ -211,7 +216,10 @@ impl StatementList { } } -impl TokenParser for StatementList { +impl TokenParser for StatementList +where + R: Read, +{ type Output = node::StatementList; fn parse(self, parser: &mut Parser) -> Result { @@ -285,7 +293,10 @@ impl StatementListItem { } } -impl TokenParser for StatementListItem { +impl TokenParser for StatementListItem +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> Result { @@ -331,7 +342,10 @@ impl ExpressionStatement { } } -impl TokenParser for ExpressionStatement { +impl TokenParser for ExpressionStatement +where + R: Read, +{ type Output = Node; fn parse(self, parser: &mut Parser) -> ParseResult { @@ -381,7 +395,10 @@ impl BindingIdentifier { } } -impl TokenParser for BindingIdentifier { +impl TokenParser for BindingIdentifier +where + R: Read, +{ type Output = Box; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index beebcf8c190..f18c75bef5e 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -5,11 +5,13 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Return, Keyword, Node, Punctuator}, - parser::{expression::Expression, AllowAwait, AllowYield, Parser, ParseError, TokenParser}, + parser::{expression::Expression, AllowAwait, AllowYield, ParseError, Parser, TokenParser}, }, BoaProfiler, }; +use std::io::Read; + /// Return statement parsing /// /// More information: @@ -38,7 +40,10 @@ impl ReturnStatement { } } -impl TokenParser for ReturnStatement { +impl TokenParser for ReturnStatement +where + R: Read, +{ type Output = Return; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/switch/mod.rs b/boa/src/syntax/parser/statement/switch/mod.rs index 0605a0feea0..c88e14adf69 100644 --- a/boa/src/syntax/parser/statement/switch/mod.rs +++ b/boa/src/syntax/parser/statement/switch/mod.rs @@ -8,13 +8,15 @@ use crate::{ Keyword, Node, Punctuator, }, parser::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, Parser, ParseError, + expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Switch statement parsing. /// /// More information: @@ -46,7 +48,10 @@ impl SwitchStatement { } } -impl TokenParser for SwitchStatement { +impl TokenParser for SwitchStatement +where + R: Read, +{ type Output = Switch; fn parse(self, parser: &mut Parser) -> Result { @@ -94,7 +99,10 @@ impl CaseBlock { } } -impl TokenParser for CaseBlock { +impl TokenParser for CaseBlock +where + R: Read, +{ type Output = (Box<[Case]>, Option); fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index 71640337857..60c9b95339b 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -5,11 +5,13 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Throw, Keyword, Punctuator}, - parser::{expression::Expression, AllowAwait, AllowYield, Parser, ParseError, TokenParser}, + parser::{expression::Expression, AllowAwait, AllowYield, ParseError, Parser, TokenParser}, }, BoaProfiler, }; +use std::io::Read; + /// For statement parsing /// /// More information: @@ -38,7 +40,10 @@ impl ThrowStatement { } } -impl TokenParser for ThrowStatement { +impl TokenParser for ThrowStatement +where + R: Read, +{ type Output = Throw; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/try_stm/catch.rs b/boa/src/syntax/parser/statement/try_stm/catch.rs index 47f140fa7ba..f18b57af0be 100644 --- a/boa/src/syntax/parser/statement/try_stm/catch.rs +++ b/boa/src/syntax/parser/statement/try_stm/catch.rs @@ -6,12 +6,14 @@ use crate::{ }, parser::{ statement::{block::Block, BindingIdentifier}, - AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser, + AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Catch parsing /// /// More information: @@ -43,7 +45,10 @@ impl Catch { } } -impl TokenParser for Catch { +impl TokenParser for Catch +where + R: Read, +{ type Output = node::Catch; fn parse(self, parser: &mut Parser) -> Result { @@ -94,7 +99,10 @@ impl CatchParameter { } } -impl TokenParser for CatchParameter { +impl TokenParser for CatchParameter +where + R: Read, +{ type Output = Identifier; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/try_stm/finally.rs b/boa/src/syntax/parser/statement/try_stm/finally.rs index a3a833068ad..1185222474c 100644 --- a/boa/src/syntax/parser/statement/try_stm/finally.rs +++ b/boa/src/syntax/parser/statement/try_stm/finally.rs @@ -2,13 +2,15 @@ use crate::{ syntax::{ ast::{node, Keyword}, parser::{ - statement::block::Block, AllowAwait, AllowReturn, AllowYield, Parser, ParseError, + statement::block::Block, AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; + /// Finally parsing /// /// More information: @@ -40,7 +42,10 @@ impl Finally { } } -impl TokenParser for Finally { +impl TokenParser for Finally +where + R: Read, +{ type Output = node::Finally; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index 5d60d137e11..faf8e297ee4 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -11,11 +11,13 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Try, Keyword}, - parser::{AllowAwait, AllowReturn, AllowYield, Parser, ParseError, TokenParser}, + parser::{AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser}, }, BoaProfiler, }; +use std::io::Read; + /// Try...catch statement parsing /// /// More information: @@ -47,7 +49,10 @@ impl TryStatement { } } -impl TokenParser for TryStatement { +impl TokenParser for TryStatement +where + R: Read, +{ type Output = Try; fn parse(self, parser: &mut Parser) -> Result { diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 7c4115be1f2..74fd22d3cef 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -8,11 +8,12 @@ use crate::{ }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, - Parser, ParseError, TokenParser, + ParseError, Parser, TokenParser, }, }, BoaProfiler, }; +use std::io::Read; /// Variable statement parsing. /// @@ -44,7 +45,10 @@ impl VariableStatement { } } -impl TokenParser for VariableStatement { +impl TokenParser for VariableStatement +where + R: Read, +{ type Output = VarDeclList; fn parse(self, parser: &mut Parser) -> Result { @@ -95,7 +99,10 @@ impl VariableDeclarationList { } } -impl TokenParser for VariableDeclarationList { +impl TokenParser for VariableDeclarationList +where + R: Read, +{ type Output = VarDeclList; fn parse(self, parser: &mut Parser) -> Result { @@ -158,7 +165,10 @@ impl VariableDeclaration { } } -impl TokenParser for VariableDeclaration { +impl TokenParser for VariableDeclaration +where + R: Read, +{ type Output = VarDecl; fn parse(self, parser: &mut Parser) -> Result { From c012e911b1cfd5abbaad70af4ce131cfa5bdca1b Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 21 Jun 2020 23:19:31 +0000 Subject: [PATCH 062/289] Continuing refractor --- boa/src/lib.rs | 11 +---------- boa/src/syntax/lexer/mod.rs | 5 ++++- boa/src/syntax/parser/mod.rs | 20 ++++++++++++++++---- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/boa/src/lib.rs b/boa/src/lib.rs index accd21f3581..37da335d27c 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -50,16 +50,7 @@ pub use crate::{ }; fn parser_expr(src: &str) -> Result { - let lexer = Lexer::new(src.as_bytes()); - - // Goes through and lexes entire given string before starting any parsing. - let mut tokens = Vec::new(); - - for token in lexer { - tokens.push(token.map_err(|e| format!("Lexing Error: {}", e))?); - } - - Parser::new(&tokens) + Parser::new(src.as_bytes()) .parse_all() .map_err(|e| format!("Parsing Error: {}", e)) } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index c88a12e9d07..3ff581aa3d9 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -90,7 +90,10 @@ impl Lexer { /// Creates a new lexer. #[inline] - pub fn new(reader: R) -> Self { + pub fn new(reader: R) -> Self + where + R: Read + { Self { cursor: Cursor::new(reader), goal_symbol: Default::default(), diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index d7a8e71d5be..fb95e2396b6 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -31,7 +31,7 @@ where /// Parses the token stream using the current parser. /// /// This method needs to be provided by the implementor type. - fn parse(self, parser: &mut Parser) -> Result; + fn parse(self, parser: &mut Parser) -> Result; // /// Tries to parse the following tokens with this parser. // fn try_parse(self, parser: Parser) -> Option { @@ -102,16 +102,24 @@ pub struct Parser { } impl Parser { - pub fn new(reader: R) -> Self { + pub fn new(reader: R) -> Self + where + R: Read + { Self { lexer: Lexer::new(reader), } } - pub fn parse_all(&mut self) -> Result { + pub fn parse_all(&mut self) -> Result + where + R: Read + { Script.parse(&mut self) } + // Note these unimplemented methods may be removed before this parser refractor is finished. + pub fn next(&mut self) -> Result { unimplemented!(); } @@ -119,6 +127,10 @@ impl Parser { pub fn peek(&mut self, amount: i32) -> Result { unimplemented!(); } + + pub fn expect(&mut self, token: Token, context_msg: &str) { + unimplemented!(); + } } /// Parses a full script. @@ -137,7 +149,7 @@ where type Output = StatementList; fn parse(self, parser: &mut Parser) -> Result { - if parser.peek().is_some() { + if parser.peek(0).is_some() { ScriptBody.parse(parser) } else { Ok(StatementList::from(Vec::new())) From ef62ecc8a3274ffdb7c275a3b0715719e2c58e8d Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 13:06:29 +0000 Subject: [PATCH 063/289] Using a parser cursor instead of parser directly for parsing --- boa/src/syntax/parser/cursor.rs | 247 ++++++++++++++++++ boa/src/syntax/parser/function/mod.rs | 18 +- boa/src/syntax/parser/mod.rs | 21 +- boa/src/syntax/parser/statement/block/mod.rs | 14 +- .../syntax/parser/statement/break_stm/mod.rs | 14 +- .../parser/statement/continue_stm/mod.rs | 14 +- .../parser/statement/declaration/hoistable.rs | 24 +- .../parser/statement/declaration/lexical.rs | 28 +- .../parser/statement/declaration/mod.rs | 10 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 20 +- .../statement/iteration/do_while_statement.rs | 20 +- .../statement/iteration/for_statement.rs | 32 +-- .../statement/iteration/while_statement.rs | 14 +- boa/src/syntax/parser/statement/mod.rs | 58 ++-- .../syntax/parser/statement/return_stm/mod.rs | 14 +- boa/src/syntax/parser/statement/switch/mod.rs | 18 +- boa/src/syntax/parser/statement/throw/mod.rs | 14 +- .../syntax/parser/statement/try_stm/catch.rs | 18 +- .../parser/statement/try_stm/finally.rs | 8 +- .../syntax/parser/statement/try_stm/mod.rs | 16 +- boa/src/syntax/parser/statement/variable.rs | 25 +- 21 files changed, 449 insertions(+), 198 deletions(-) create mode 100644 boa/src/syntax/parser/cursor.rs diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs new file mode 100644 index 00000000000..a28e8a54b08 --- /dev/null +++ b/boa/src/syntax/parser/cursor.rs @@ -0,0 +1,247 @@ +//! Cursor implementation for the parser. + +use super::ParseError; +use crate::syntax::ast::Punctuator; +use crate::syntax::lexer::{Token, TokenKind}; +use crate::syntax::lexer::Lexer; + +use std::io::Read; + +/// Token cursor. +/// +/// This internal structure gives basic testable operations to the parser. +#[derive(Debug, Clone, Default)] +pub(super) struct Cursor { + /// The tokens being input. + // tokens: &'a [Token], + lexer: Lexer, + + // The current position within the tokens. + // pos: usize, +} + +impl Cursor +where + R: Read +{ + /// Creates a new cursor. + pub(super) fn new(reader: R) -> Self { + Self { + lexer: Lexer::new(reader) + } + } + + /// Retrieves the current position of the cursor in the token stream. + pub(super) fn pos(&self) -> usize { + self.pos + } + + /// Moves the cursor to the given position. + /// + /// This is intended to be used *always* with `Cursor::pos()`. + pub(super) fn seek(&mut self, pos: usize) { + self.pos = pos + } + + /// Moves the cursor to the next token and returns the token. + pub(super) fn next(&mut self) -> Option { + loop { + let token = self.tokens.get(self.pos); + if let Some(tk) = token { + self.pos += 1; + + if tk.kind != TokenKind::LineTerminator { + break Some(tk); + } + } else { + break None; + } + } + } + + /// Peeks the next token without moving the cursor. + pub(super) fn peek(&self, skip: usize) -> Option<&Token> { + let mut count = 0; + let mut skipped = 0; + loop { + let token = self.tokens.get(self.pos + count); + count += 1; + + if let Some(tk) = token { + if tk.kind != TokenKind::LineTerminator { + if skipped == skip { + break Some(tk); + } + + skipped += 1; + } + } else { + break None; + } + } + } + + /// Moves the cursor to the previous token and returns the token. + pub(super) fn back(&mut self) { + debug_assert!( + self.pos > 0, + "cannot go back in a cursor that is at the beginning of the list of tokens" + ); + + self.pos -= 1; + while self + .tokens + .get(self.pos - 1) + .expect("token disappeared") + .kind + == TokenKind::LineTerminator + && self.pos > 0 + { + self.pos -= 1; + } + } + + /// Peeks the previous token without moving the cursor. + pub(super) fn peek_prev(&self) -> Option<&Token> { + if self.pos == 0 { + None + } else { + let mut back = 1; + let mut tok = self.tokens.get(self.pos - back).expect("token disappeared"); + while self.pos >= back && tok.kind == TokenKind::LineTerminator { + back += 1; + tok = self.tokens.get(self.pos - back).expect("token disappeared"); + } + + if back == self.pos { + None + } else { + Some(tok) + } + } + } + + /// Returns an error if the next token is not of kind `kind`. + /// + /// Note: it will consume the next token. + pub(super) fn expect(&mut self, kind: K, context: &'static str) -> Result<(), ParseError> + where + K: Into, + { + let next_token = self.next().ok_or(ParseError::AbruptEnd)?; + let kind = kind.into(); + + if next_token.kind == kind { + Ok(()) + } else { + Err(ParseError::expected( + vec![kind], + next_token.clone(), + context, + )) + } + } + + /// It will peek for the next token, to see if it's a semicolon. + /// + /// It will automatically insert a semicolon if needed, as specified in the [spec][spec]. + /// + /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion + pub(super) fn peek_semicolon(&self, do_while: bool) -> (bool, Option<&Token>) { + match self.tokens.get(self.pos) { + Some(tk) => match tk.kind { + TokenKind::Punctuator(Punctuator::Semicolon) => (true, Some(tk)), + TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { + (true, Some(tk)) + } + _ => { + if do_while { + debug_assert!( + self.pos != 0, + "cannot be finishing a do-while if we are at the beginning" + ); + + let tok = self + .tokens + .get(self.pos - 1) + .expect("could not find previous token"); + if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { + return (true, Some(tk)); + } + } + + (false, Some(tk)) + } + }, + None => (true, None), + } + } + + /// It will check if the next token is a semicolon. + /// + /// It will automatically insert a semicolon if needed, as specified in the [spec][spec]. + /// + /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion + pub(super) fn expect_semicolon( + &mut self, + do_while: bool, + context: &'static str, + ) -> Result<(), ParseError> { + match self.peek_semicolon(do_while) { + (true, Some(tk)) => match tk.kind { + TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { + self.pos += 1; + Ok(()) + } + _ => Ok(()), + }, + (true, None) => Ok(()), + (false, Some(tk)) => Err(ParseError::expected( + vec![TokenKind::Punctuator(Punctuator::Semicolon)], + tk.clone(), + context, + )), + (false, None) => unreachable!(), + } + } + + /// It will make sure that the next token is not a line terminator. + /// + /// It expects that the token stream does not end here. + pub(super) fn peek_expect_no_lineterminator(&mut self, skip: usize) -> Result<(), ParseError> { + let mut count = 0; + let mut skipped = 0; + loop { + let token = self.tokens.get(self.pos + count); + count += 1; + if let Some(tk) = token { + if skipped == skip && tk.kind == TokenKind::LineTerminator { + break Err(ParseError::unexpected(tk.clone(), None)); + } else if skipped == skip && tk.kind != TokenKind::LineTerminator { + break Ok(()); + } else if tk.kind != TokenKind::LineTerminator { + skipped += 1; + } + } else { + break Err(ParseError::AbruptEnd); + } + } + } + + /// Advance the cursor to the next token and retrieve it, only if it's of `kind` type. + /// + /// When the next token is a `kind` token, get the token, otherwise return `None`. This + /// function skips line terminators. + pub(super) fn next_if(&mut self, kind: K) -> Option + where + K: Into, + { + let next_token = self.peek(0)?; + + if next_token.kind == kind.into() { + self.next() + } else { + None + } + } +} diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 5f6e764d8ee..f52155b0832 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -19,7 +19,7 @@ use crate::syntax::{ parser::{ expression::Initializer, statement::{BindingIdentifier, StatementList}, - AllowAwait, AllowYield, ParseError, Parser, TokenParser, + AllowAwait, AllowYield, ParseError, Cursor, TokenParser, }, }; @@ -59,10 +59,10 @@ where { type Output = Box<[node::FormalParameter]>; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let mut params = Vec::new(); - if parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind + if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind == TokenKind::Punctuator(Punctuator::CloseParen) { return Ok(params.into_boxed_slice()); @@ -71,14 +71,14 @@ where loop { let mut rest_param = false; - params.push(if parser.next_if(Punctuator::Spread).is_some() { + params.push(if cursor.next_if(Punctuator::Spread).is_some() { rest_param = true; - FunctionRestParameter::new(self.allow_yield, self.allow_await).parse(parser)? + FunctionRestParameter::new(self.allow_yield, self.allow_await).parse(cursor)? } else { - FormalParameter::new(self.allow_yield, self.allow_await).parse(parser)? + FormalParameter::new(self.allow_yield, self.allow_await).parse(cursor)? }); - if parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind + if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind == TokenKind::Punctuator(Punctuator::CloseParen) { break; @@ -86,7 +86,7 @@ where if rest_param { return Err(ParseError::unexpected( - parser + cursor .peek_prev() .expect("current token disappeared") .clone(), @@ -94,7 +94,7 @@ where )); } - parser.expect(Punctuator::Comma, "parameter list")?; + cursor.expect(Punctuator::Comma, "parameter list")?; } Ok(params.into_boxed_slice()) diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index fb95e2396b6..9d6fb2b88a0 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -1,6 +1,7 @@ //! Boa parser implementation. pub mod error; +mod cursor; mod expression; mod function; mod statement; @@ -16,6 +17,8 @@ use crate::syntax::lexer::Token; use ParseError as Error; +use cursor::Cursor; + use std::io::Read; /// Trait implemented by parsers. @@ -31,7 +34,7 @@ where /// Parses the token stream using the current parser. /// /// This method needs to be provided by the implementor type. - fn parse(self, parser: &mut Parser) -> Result; + fn parse(self, cursor: &mut Cursor) -> Result; // /// Tries to parse the following tokens with this parser. // fn try_parse(self, parser: Parser) -> Option { @@ -98,7 +101,7 @@ impl From for AllowDefault { #[derive(Debug)] pub struct Parser { /// Lexer used to get tokens for the parser. - lexer: Lexer, + cursor: Cursor, } impl Parser { @@ -107,7 +110,7 @@ impl Parser { R: Read { Self { - lexer: Lexer::new(reader), + cursor: Cursor::new(reader) } } @@ -115,7 +118,7 @@ impl Parser { where R: Read { - Script.parse(&mut self) + Script.parse(&mut self.cursor) } // Note these unimplemented methods may be removed before this parser refractor is finished. @@ -148,9 +151,9 @@ where { type Output = StatementList; - fn parse(self, parser: &mut Parser) -> Result { - if parser.peek(0).is_some() { - ScriptBody.parse(parser) + fn parse(self, cursor: &mut Cursor) -> Result { + if cursor.peek(0).is_some() { + ScriptBody.parse(cursor) } else { Ok(StatementList::from(Vec::new())) } @@ -172,7 +175,7 @@ where { type Output = StatementList; - fn parse(self, parser: &mut Parser) -> Result { - self::statement::StatementList::new(false, false, false, false).parse(parser) + fn parse(self, cursor: &mut Cursor) -> Result { + self::statement::StatementList::new(false, false, false, false).parse(cursor) } } diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 7b58a395a48..7cb972c06c4 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -17,7 +17,7 @@ use crate::{ profiler::BoaProfiler, syntax::{ ast::{node, Punctuator}, - parser::{AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser}, + parser::{AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser}, }, }; @@ -68,21 +68,21 @@ where { type Output = node::Block; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Block", "Parsing"); - parser.expect(Punctuator::OpenBlock, "block")?; - if let Some(tk) = parser.peek(0) { + cursor.expect(Punctuator::OpenBlock, "block")?; + if let Some(tk) = cursor.peek(0) { if tk.kind == TokenKind::Punctuator(Punctuator::CloseBlock) { - parser.next(); + cursor.next(); return Ok(node::Block::from(vec![])); } } let statement_list = StatementList::new(self.allow_yield, self.allow_await, self.allow_return, true) - .parse(parser) + .parse(cursor) .map(node::Block::from)?; - parser.expect(Punctuator::CloseBlock, "block")?; + cursor.expect(Punctuator::CloseBlock, "block")?; Ok(statement_list) } diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index fcfcb76c164..05a77eddc45 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -16,7 +16,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Break, Keyword, Punctuator}, - parser::{AllowAwait, AllowYield, ParseError, Parser, TokenParser}, + parser::{AllowAwait, AllowYield, ParseError, Cursor, TokenParser}, }, BoaProfiler, }; @@ -57,22 +57,22 @@ where { type Output = Break; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("BreakStatement", "Parsing"); - parser.expect(Keyword::Break, "break statement")?; + cursor.expect(Keyword::Break, "break statement")?; - let label = if let (true, tok) = parser.peek_semicolon(false) { + let label = if let (true, tok) = cursor.peek_semicolon(false) { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) => { - let _ = parser.next(); + let _ = cursor.next(); } _ => {} } None } else { - let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; - parser.expect_semicolon(false, "continue statement")?; + let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect_semicolon(false, "continue statement")?; Some(label) }; diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index 6ce16fab00b..55c7b7493e8 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -15,7 +15,7 @@ use crate::{ syntax::{ ast::{node::Continue, Keyword, Punctuator}, parser::{ - statement::LabelIdentifier, AllowAwait, AllowYield, ParseError, Parser, TokenParser, + statement::LabelIdentifier, AllowAwait, AllowYield, ParseError, Cursor, TokenParser, }, }, BoaProfiler, @@ -57,22 +57,22 @@ where { type Output = Continue; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ContinueStatement", "Parsing"); - parser.expect(Keyword::Continue, "continue statement")?; + cursor.expect(Keyword::Continue, "continue statement")?; - let label = if let (true, tok) = parser.peek_semicolon(false) { + let label = if let (true, tok) = cursor.peek_semicolon(false) { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) => { - let _ = parser.next(); + let _ = cursor.next(); } _ => {} } None } else { - let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; - parser.expect_semicolon(false, "continue statement")?; + let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect_semicolon(false, "continue statement")?; Some(label) }; diff --git a/boa/src/syntax/parser/statement/declaration/hoistable.rs b/boa/src/syntax/parser/statement/declaration/hoistable.rs index bc9b6e1f092..0da58c61fdf 100644 --- a/boa/src/syntax/parser/statement/declaration/hoistable.rs +++ b/boa/src/syntax/parser/statement/declaration/hoistable.rs @@ -10,7 +10,7 @@ use crate::{ ast::{node::FunctionDecl, Keyword, Node, Punctuator}, parser::{ function::FormalParameters, function::FunctionBody, statement::BindingIdentifier, - AllowAwait, AllowDefault, AllowYield, ParseError, ParseResult, Parser, TokenParser, + AllowAwait, AllowDefault, AllowYield, ParseError, ParseResult, Cursor, TokenParser, }, }, BoaProfiler, @@ -53,11 +53,11 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("HoistableDeclaration", "Parsing"); // TODO: check for generators and async functions + generators FunctionDeclaration::new(self.allow_yield, self.allow_await, self.is_default) - .parse(parser) + .parse(cursor) .map(Node::from) } } @@ -99,22 +99,22 @@ where { type Output = FunctionDecl; - fn parse(self, parser: &mut Parser) -> Result { - parser.expect(Keyword::Function, "function declaration")?; + fn parse(self, cursor: &mut Cursor) -> Result { + cursor.expect(Keyword::Function, "function declaration")?; // TODO: If self.is_default, then this can be empty. - let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; + let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect(Punctuator::OpenParen, "function declaration")?; + cursor.expect(Punctuator::OpenParen, "function declaration")?; - let params = FormalParameters::new(false, false).parse(parser)?; + let params = FormalParameters::new(false, false).parse(cursor)?; - parser.expect(Punctuator::CloseParen, "function declaration")?; - parser.expect(Punctuator::OpenBlock, "function declaration")?; + cursor.expect(Punctuator::CloseParen, "function declaration")?; + cursor.expect(Punctuator::OpenBlock, "function declaration")?; - let body = FunctionBody::new(self.allow_yield, self.allow_await).parse(parser)?; + let body = FunctionBody::new(self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect(Punctuator::CloseBlock, "function declaration")?; + cursor.expect(Punctuator::CloseBlock, "function declaration")?; Ok(FunctionDecl::new(name, params, body)) } diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index e01db5ed27f..74522e71489 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -16,7 +16,7 @@ use crate::{ }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, - ParseError, ParseResult, Parser, TokenParser, + ParseError, ParseResult, Cursor, TokenParser, }, }, BoaProfiler, @@ -59,18 +59,18 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("LexicalDeclaration", "Parsing"); - let tok = parser.next().ok_or(ParseError::AbruptEnd)?; + let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Const) => { BindingList::new(self.allow_in, self.allow_yield, self.allow_await, true) - .parse(parser) + .parse(cursor) } TokenKind::Keyword(Keyword::Let) => { BindingList::new(self.allow_in, self.allow_yield, self.allow_await, false) - .parse(parser) + .parse(cursor) } _ => unreachable!("unknown token found"), } @@ -117,7 +117,7 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { // Create vectors to store the variable declarations // Const and Let signatures are slightly different, Const needs definitions, Lets don't let mut let_decls = Vec::new(); @@ -126,7 +126,7 @@ where loop { let (ident, init) = LexicalBinding::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(parser)?; + .parse(cursor)?; if self.is_const { if let Some(init) = init { @@ -134,7 +134,7 @@ where } else { return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::Assign)], - parser.next().ok_or(ParseError::AbruptEnd)?.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), "const declaration", )); } @@ -142,10 +142,10 @@ where let_decls.push(LetDecl::new(ident, init)); } - match parser.peek_semicolon(false) { + match cursor.peek_semicolon(false) { (true, _) => break, (false, Some(tk)) if tk.kind == TokenKind::Punctuator(Punctuator::Comma) => { - let _ = parser.next(); + let _ = cursor.next(); } _ => { return Err(ParseError::expected( @@ -153,7 +153,7 @@ where TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - parser.next().ok_or(ParseError::AbruptEnd)?.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), "lexical declaration", )) } @@ -202,10 +202,10 @@ where { type Output = (Box, Option); - fn parse(self, parser: &mut Parser) -> Result { - let ident = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; + fn parse(self, cursor: &mut Cursor) -> Result { + let ident = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; let initializer = - Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(parser); + Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(cursor); Ok((ident, initializer)) } diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index 69257e47391..9934efb5b43 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -18,7 +18,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Keyword, Node}, - parser::{AllowAwait, AllowYield, ParseError, Parser, TokenParser}, + parser::{AllowAwait, AllowYield, ParseError, Cursor, TokenParser}, }, BoaProfiler, }; @@ -56,16 +56,16 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Declaration", "Parsing"); - let tok = parser.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Function) => { - HoistableDeclaration::new(self.allow_yield, self.allow_await, false).parse(parser) + HoistableDeclaration::new(self.allow_yield, self.allow_await, false).parse(cursor) } TokenKind::Keyword(Keyword::Const) | TokenKind::Keyword(Keyword::Let) => { - LexicalDeclaration::new(true, self.allow_yield, self.allow_await).parse(parser) + LexicalDeclaration::new(true, self.allow_yield, self.allow_await).parse(cursor) } _ => unreachable!("unknown token found"), } diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index 236997b2e87..c225e677140 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -8,7 +8,7 @@ use crate::{ syntax::{ ast::{node::If, Keyword, Node, Punctuator}, parser::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, Parser, + expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser, }, }, @@ -56,24 +56,24 @@ where { type Output = If; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("IfStatement", "Parsing"); - parser.expect(Keyword::If, "if statement")?; - parser.expect(Punctuator::OpenParen, "if statement")?; + cursor.expect(Keyword::If, "if statement")?; + cursor.expect(Punctuator::OpenParen, "if statement")?; - let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect(Punctuator::CloseParen, "if statement")?; + cursor.expect(Punctuator::CloseParen, "if statement")?; let then_stm = - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let else_stm = match parser.peek(0) { + let else_stm = match cursor.peek(0) { Some(else_tok) if else_tok.kind == TokenKind::Keyword(Keyword::Else) => { - parser.next(); + cursor.next(); Some( Statement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser)?, + .parse(cursor)?, ) } _ => None, diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index c0f1042e8d1..3c9e1f010c9 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -13,7 +13,7 @@ use crate::{ ast::{node::DoWhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, - ParseError, Parser, TokenParser, + ParseError, Cursor, TokenParser, }, }, BoaProfiler, @@ -61,14 +61,14 @@ where { type Output = DoWhileLoop; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("DoWhileStatement", "Parsing"); - parser.expect(Keyword::Do, "do while statement")?; + cursor.expect(Keyword::Do, "do while statement")?; let body = - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let next_token = parser.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; if next_token.kind != TokenKind::Keyword(Keyword::While) { return Err(ParseError::expected( @@ -78,13 +78,13 @@ where )); } - parser.expect(Keyword::While, "do while statement")?; - parser.expect(Punctuator::OpenParen, "do while statement")?; + cursor.expect(Keyword::While, "do while statement")?; + cursor.expect(Punctuator::OpenParen, "do while statement")?; - let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect(Punctuator::CloseParen, "do while statement")?; - parser.expect_semicolon(true, "do while statement")?; + cursor.expect(Punctuator::CloseParen, "do while statement")?; + cursor.expect_semicolon(true, "do while statement")?; Ok(DoWhileLoop::new(body, cond)) } diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index df981443028..79a7671861a 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -18,7 +18,7 @@ use crate::{ expression::Expression, statement::declaration::Declaration, statement::{variable::VariableDeclarationList, Statement}, - AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser, + AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser, }, }, BoaProfiler, @@ -67,39 +67,39 @@ where { type Output = ForLoop; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ForStatement", "Parsing"); - parser.expect(Keyword::For, "for statement")?; - parser.expect(Punctuator::OpenParen, "for statement")?; + cursor.expect(Keyword::For, "for statement")?; + cursor.expect(Punctuator::OpenParen, "for statement")?; - let init = match parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind { + let init = match cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Keyword(Keyword::Var) => Some( VariableDeclarationList::new(false, self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .map(Node::from)?, ), TokenKind::Keyword(Keyword::Let) | TokenKind::Keyword(Keyword::Const) => { - Some(Declaration::new(self.allow_yield, self.allow_await).parse(parser)?) + Some(Declaration::new(self.allow_yield, self.allow_await).parse(cursor)?) } TokenKind::Punctuator(Punctuator::Semicolon) => None, - _ => Some(Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?), + _ => Some(Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?), }; - parser.expect(Punctuator::Semicolon, "for statement")?; + cursor.expect(Punctuator::Semicolon, "for statement")?; - let cond = if parser.next_if(Punctuator::Semicolon).is_some() { + let cond = if cursor.next_if(Punctuator::Semicolon).is_some() { Const::from(true).into() } else { - let step = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - parser.expect(Punctuator::Semicolon, "for statement")?; + let step = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect(Punctuator::Semicolon, "for statement")?; step }; - let step = if parser.next_if(Punctuator::CloseParen).is_some() { + let step = if cursor.next_if(Punctuator::CloseParen).is_some() { None } else { - let step = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - parser.expect( + let step = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect( TokenKind::Punctuator(Punctuator::CloseParen), "for statement", )?; @@ -107,7 +107,7 @@ where }; let body = - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; // TODO: do not encapsulate the `for` in a block just to have an inner scope. Ok(ForLoop::new(init, cond, step, body)) diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index 61e5280969a..6cf7317cc28 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -3,7 +3,7 @@ use crate::{ ast::{node::WhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, - ParseError, Parser, TokenParser, + ParseError, Cursor, TokenParser, }, }, BoaProfiler, @@ -52,17 +52,17 @@ where { type Output = WhileLoop; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("WhileStatement", "Parsing"); - parser.expect(Keyword::While, "while statement")?; - parser.expect(Punctuator::OpenParen, "while statement")?; + cursor.expect(Keyword::While, "while statement")?; + cursor.expect(Punctuator::OpenParen, "while statement")?; - let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect(Punctuator::CloseParen, "while statement")?; + cursor.expect(Punctuator::CloseParen, "while statement")?; let body = - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; Ok(WhileLoop::new(cond, body)) } diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index f8d6a5b6a94..96686aceb64 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -33,7 +33,7 @@ use self::{ variable::VariableStatement, }; use super::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, ParseResult, Parser, + expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, ParseResult, Cursor, TokenParser, }; @@ -99,41 +99,41 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Statement", "Parsing"); // TODO: add BreakableStatement and divide Whiles, fors and so on to another place. - let tok = parser.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::If) => { IfStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::Var) => { VariableStatement::new(self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::While) => { WhileStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::Do) => { DoWhileStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::For) => { ForStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::Return) => { if self.allow_return.0 { ReturnStatement::new(self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .map(Node::from) } else { Err(ParseError::unexpected(tok.clone(), "statement")) @@ -141,39 +141,39 @@ where } TokenKind::Keyword(Keyword::Break) => { BreakStatement::new(self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::Continue) => { ContinueStatement::new(self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::Try) => { TryStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::Throw) => { ThrowStatement::new(self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Keyword(Keyword::Switch) => { SwitchStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser) + .parse(cursor) .map(Node::from) } TokenKind::Punctuator(Punctuator::OpenBlock) => { BlockStatement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser) + .parse(cursor) .map(Node::from) } // TODO: https://tc39.es/ecma262/#prod-LabelledStatement // TokenKind::Punctuator(Punctuator::Semicolon) => { // return Ok(Node::new(NodeBase::Nope, tok.pos)) // } - _ => ExpressionStatement::new(self.allow_yield, self.allow_await).parse(parser), + _ => ExpressionStatement::new(self.allow_yield, self.allow_await).parse(cursor), } } } @@ -222,12 +222,12 @@ where { type Output = node::StatementList; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("StatementList", "Parsing"); let mut items = Vec::new(); loop { - match parser.peek(0) { + match cursor.peek(0) { Some(token) if token.kind == TokenKind::Punctuator(Punctuator::CloseBlock) => { if self.break_when_closingbrase { break; @@ -247,11 +247,11 @@ where let item = StatementListItem::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser)?; + .parse(cursor)?; items.push(item); // move the cursor forward for any consecutive semicolon. - while parser.next_if(Punctuator::Semicolon).is_some() {} + while cursor.next_if(Punctuator::Semicolon).is_some() {} } items.sort_by(Node::hoistable_order); @@ -299,18 +299,18 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("StatementListItem", "Parsing"); - let tok = parser.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Function) | TokenKind::Keyword(Keyword::Const) | TokenKind::Keyword(Keyword::Let) => { - Declaration::new(self.allow_yield, self.allow_await).parse(parser) + Declaration::new(self.allow_yield, self.allow_await).parse(cursor) } _ => { - Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser) + Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor) } } } @@ -348,12 +348,12 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExpressionStatement", "Parsing"); // TODO: lookahead - let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect_semicolon(false, "expression statement")?; + cursor.expect_semicolon(false, "expression statement")?; Ok(expr) } @@ -401,11 +401,11 @@ where { type Output = Box; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("BindingIdentifier", "Parsing"); // TODO: strict mode. - let next_token = parser.next().ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; match next_token.kind { TokenKind::Identifier(ref s) => Ok(s.clone()), diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index f18c75bef5e..77377d4dd4d 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -5,7 +5,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Return, Keyword, Node, Punctuator}, - parser::{expression::Expression, AllowAwait, AllowYield, ParseError, Parser, TokenParser}, + parser::{expression::Expression, AllowAwait, AllowYield, ParseError, Cursor, TokenParser}, }, BoaProfiler, }; @@ -46,17 +46,17 @@ where { type Output = Return; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ReturnStatement", "Parsing"); - parser.expect(Keyword::Return, "return statement")?; + cursor.expect(Keyword::Return, "return statement")?; - if let (true, tok) = parser.peek_semicolon(false) { + if let (true, tok) = cursor.peek_semicolon(false) { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) || tok.kind == TokenKind::LineTerminator => { - let _ = parser.next(); + let _ = cursor.next(); } _ => {} } @@ -64,9 +64,9 @@ where return Ok(Return::new::>(None)); } - let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect_semicolon(false, "return statement")?; + cursor.expect_semicolon(false, "return statement")?; Ok(Return::new(expr)) } diff --git a/boa/src/syntax/parser/statement/switch/mod.rs b/boa/src/syntax/parser/statement/switch/mod.rs index c88e14adf69..7635ed097f0 100644 --- a/boa/src/syntax/parser/statement/switch/mod.rs +++ b/boa/src/syntax/parser/statement/switch/mod.rs @@ -8,7 +8,7 @@ use crate::{ Keyword, Node, Punctuator, }, parser::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, Parser, + expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser, }, }, @@ -54,17 +54,17 @@ where { type Output = Switch; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("SwitchStatement", "Parsing"); - parser.expect(Keyword::Switch, "switch statement")?; - parser.expect(Punctuator::OpenParen, "switch statement")?; + cursor.expect(Keyword::Switch, "switch statement")?; + cursor.expect(Punctuator::OpenParen, "switch statement")?; - let condition = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; + let condition = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect(Punctuator::CloseParen, "switch statement")?; + cursor.expect(Punctuator::CloseParen, "switch statement")?; let (cases, default) = - CaseBlock::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; + CaseBlock::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; Ok(Switch::new(condition, cases, default)) } @@ -105,8 +105,8 @@ where { type Output = (Box<[Case]>, Option); - fn parse(self, parser: &mut Parser) -> Result { - parser.expect(Punctuator::OpenBlock, "switch case block")?; + fn parse(self, cursor: &mut Cursor) -> Result { + cursor.expect(Punctuator::OpenBlock, "switch case block")?; // CaseClauses[?Yield, ?Await, ?Return]opt // CaseClauses[?Yield, ?Await, ?Return]optDefaultClause[?Yield, ?Await, ?Return]CaseClauses[?Yield, ?Await, ?Return]opt diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index 60c9b95339b..41b1ec51d23 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -5,7 +5,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Throw, Keyword, Punctuator}, - parser::{expression::Expression, AllowAwait, AllowYield, ParseError, Parser, TokenParser}, + parser::{expression::Expression, AllowAwait, AllowYield, ParseError, Cursor, TokenParser}, }, BoaProfiler, }; @@ -46,16 +46,16 @@ where { type Output = Throw; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ThrowStatement", "Parsing"); - parser.expect(Keyword::Throw, "throw statement")?; + cursor.expect(Keyword::Throw, "throw statement")?; - parser.peek_expect_no_lineterminator(0)?; + cursor.peek_expect_no_lineterminator(0)?; - let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - if let Some(tok) = parser.peek(0) { + let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + if let Some(tok) = cursor.peek(0) { if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) { - let _ = parser.next(); + let _ = cursor.next(); } } diff --git a/boa/src/syntax/parser/statement/try_stm/catch.rs b/boa/src/syntax/parser/statement/try_stm/catch.rs index f18b57af0be..8f25d0f477c 100644 --- a/boa/src/syntax/parser/statement/try_stm/catch.rs +++ b/boa/src/syntax/parser/statement/try_stm/catch.rs @@ -6,7 +6,7 @@ use crate::{ }, parser::{ statement::{block::Block, BindingIdentifier}, - AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser, + AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser, }, }, BoaProfiler, @@ -51,13 +51,13 @@ where { type Output = node::Catch; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Catch", "Parsing"); - parser.expect(Keyword::Catch, "try statement")?; - let catch_param = if parser.next_if(Punctuator::OpenParen).is_some() { + cursor.expect(Keyword::Catch, "try statement")?; + let catch_param = if cursor.next_if(Punctuator::OpenParen).is_some() { let catch_param = - CatchParameter::new(self.allow_yield, self.allow_await).parse(parser)?; - parser.expect(Punctuator::CloseParen, "catch in try statement")?; + CatchParameter::new(self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect(Punctuator::CloseParen, "catch in try statement")?; Some(catch_param) } else { None @@ -66,7 +66,7 @@ where // Catch block Ok(node::Catch::new::<_, Identifier, _>( catch_param, - Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?, + Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?, )) } } @@ -105,10 +105,10 @@ where { type Output = Identifier; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { // TODO: should accept BindingPattern BindingIdentifier::new(self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .map(Identifier::from) } } diff --git a/boa/src/syntax/parser/statement/try_stm/finally.rs b/boa/src/syntax/parser/statement/try_stm/finally.rs index 1185222474c..65c8e14db80 100644 --- a/boa/src/syntax/parser/statement/try_stm/finally.rs +++ b/boa/src/syntax/parser/statement/try_stm/finally.rs @@ -2,7 +2,7 @@ use crate::{ syntax::{ ast::{node, Keyword}, parser::{ - statement::block::Block, AllowAwait, AllowReturn, AllowYield, ParseError, Parser, + statement::block::Block, AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser, }, }, @@ -48,12 +48,12 @@ where { type Output = node::Finally; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Finally", "Parsing"); - parser.expect(Keyword::Finally, "try statement")?; + cursor.expect(Keyword::Finally, "try statement")?; Ok( Block::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser)? + .parse(cursor)? .into(), ) } diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index faf8e297ee4..5111ccaf685 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -11,7 +11,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Try, Keyword}, - parser::{AllowAwait, AllowReturn, AllowYield, ParseError, Parser, TokenParser}, + parser::{AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser}, }, BoaProfiler, }; @@ -55,15 +55,15 @@ where { type Output = Try; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("TryStatement", "Parsing"); // TRY - parser.expect(Keyword::Try, "try statement")?; + cursor.expect(Keyword::Try, "try statement")?; let try_clause = - Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?; + Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let next_token = parser.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; if next_token.kind != TokenKind::Keyword(Keyword::Catch) && next_token.kind != TokenKind::Keyword(Keyword::Finally) @@ -79,17 +79,17 @@ where } let catch = if next_token.kind == TokenKind::Keyword(Keyword::Catch) { - Some(Catch::new(self.allow_yield, self.allow_await, self.allow_return).parse(parser)?) + Some(Catch::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?) } else { None }; - let next_token = parser.peek(0); + let next_token = cursor.peek(0); let finally_block = match next_token { Some(token) => match token.kind { TokenKind::Keyword(Keyword::Finally) => Some( Finally::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(parser)?, + .parse(cursor)?, ), _ => None, }, diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 74fd22d3cef..fb9df47a737 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -14,6 +14,7 @@ use crate::{ BoaProfiler, }; use std::io::Read; +use crate::syntax::parser::Cursor; /// Variable statement parsing. /// @@ -51,14 +52,14 @@ where { type Output = VarDeclList; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("VariableStatement", "Parsing"); - parser.expect(Keyword::Var, "variable statement")?; + cursor.expect(Keyword::Var, "variable statement")?; let decl_list = - VariableDeclarationList::new(true, self.allow_yield, self.allow_await).parse(parser)?; + VariableDeclarationList::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - parser.expect_semicolon(false, "variable statement")?; + cursor.expect_semicolon(false, "variable statement")?; Ok(decl_list) } @@ -105,19 +106,19 @@ where { type Output = VarDeclList; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let mut list = Vec::new(); loop { list.push( VariableDeclaration::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(parser)?, + .parse(cursor)?, ); - match parser.peek_semicolon(false) { + match cursor.peek_semicolon(false) { (true, _) => break, (false, Some(tk)) if tk.kind == TokenKind::Punctuator(Punctuator::Comma) => { - let _ = parser.next(); + let _ = cursor.next(); } _ => { return Err(ParseError::expected( @@ -125,7 +126,7 @@ where TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - parser.next().ok_or(ParseError::AbruptEnd)?.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), "lexical declaration", )) } @@ -171,13 +172,13 @@ where { type Output = VarDecl; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { // TODO: BindingPattern - let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; + let name = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; let ident = - Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(parser); + Initializer::new(self.allow_in, self.allow_yield, self.allow_await).try_parse(cursor); Ok(VarDecl::new(name, ident)) } From 5e9598222c998ce99ce96b7c005fd20c2888a219 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 13:22:45 +0000 Subject: [PATCH 064/289] Switching parser back to cursor --- .../expression/assignment/arrow_function.rs | 36 ++++++------- .../expression/assignment/conditional.rs | 16 +++--- .../expression/assignment/exponentiation.rs | 16 +++--- .../parser/expression/assignment/mod.rs | 20 ++++---- .../expression/left_hand_side/arguments.rs | 16 +++--- .../parser/expression/left_hand_side/call.rs | 16 +++--- .../expression/left_hand_side/member.rs | 22 ++++---- .../parser/expression/left_hand_side/mod.rs | 10 ++-- boa/src/syntax/parser/expression/mod.rs | 10 ++-- .../primary/array_initializer/mod.rs | 18 +++---- .../expression/primary/function_expression.rs | 18 +++---- .../syntax/parser/expression/primary/mod.rs | 16 +++--- .../primary/object_initializer/mod.rs | 50 +++++++++---------- boa/src/syntax/parser/function/mod.rs | 14 +++--- 14 files changed, 139 insertions(+), 139 deletions(-) diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index 3b4b46b6fb9..1c3557365d9 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -19,7 +19,7 @@ use crate::{ error::{ErrorContext, ParseError, ParseResult}, function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - AllowAwait, AllowIn, AllowYield, Parser, TokenParser, + AllowAwait, AllowIn, AllowYield, Cursor, TokenParser, }, }, BoaProfiler, @@ -68,27 +68,27 @@ where { type Output = ArrowFunctionDecl; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ArrowFunction", "Parsing"); - let next_token = parser.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token.kind { // CoverParenthesizedExpressionAndArrowParameterList - parser.expect(Punctuator::OpenParen, "arrow function")?; - let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(parser)?; - parser.expect(Punctuator::CloseParen, "arrow function")?; + cursor.expect(Punctuator::OpenParen, "arrow function")?; + let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect(Punctuator::CloseParen, "arrow function")?; params } else { let param = BindingIdentifier::new(self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .context("arrow function")?; Box::new([FormalParameter::new(param, None, false)]) }; - parser.peek_expect_no_lineterminator(0)?; + cursor.peek_expect_no_lineterminator(0)?; - parser.expect(Punctuator::Arrow, "arrow function")?; + cursor.expect(Punctuator::Arrow, "arrow function")?; - let body = ConciseBody::new(self.allow_in).parse(parser)?; + let body = ConciseBody::new(self.allow_in).parse(cursor)?; Ok(ArrowFunctionDecl::new(params, body)) } @@ -118,16 +118,16 @@ where { type Output = StatementList; - fn parse(self, parser: &mut Parser) -> Result { - match parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind { + fn parse(self, cursor: &mut Cursor) -> Result { + match cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Punctuator(Punctuator::OpenBlock) => { - let _ = parser.next(); - let body = FunctionBody::new(false, false).parse(parser)?; - parser.expect(Punctuator::CloseBlock, "arrow function")?; + let _ = cursor.next(); + let body = FunctionBody::new(false, false).parse(cursor)?; + cursor.expect(Punctuator::CloseBlock, "arrow function")?; Ok(body) } _ => Ok(StatementList::from(vec![Return::new( - ExpressionBody::new(self.allow_in, false).parse(parser)?, + ExpressionBody::new(self.allow_in, false).parse(cursor)?, ) .into()])), } @@ -161,7 +161,7 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { - AssignmentExpression::new(self.allow_in, false, self.allow_await).parse(parser) + fn parse(self, cursor: &mut Cursor) -> ParseResult { + AssignmentExpression::new(self.allow_in, false, self.allow_await).parse(cursor) } } diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index 9a34bc05287..e711bc404eb 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -13,7 +13,7 @@ use crate::{ ast::{node::ConditionalOp, Node, Punctuator}, parser::{ expression::{AssignmentExpression, LogicalORExpression}, - AllowAwait, AllowIn, AllowYield, ParseResult, Parser, TokenParser, + AllowAwait, AllowIn, AllowYield, ParseResult, Cursor, TokenParser, }, }, BoaProfiler, @@ -62,25 +62,25 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Conditional", "Parsing"); // TODO: coalesce expression let lhs = LogicalORExpression::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(parser)?; + .parse(cursor)?; - if let Some(tok) = parser.next() { + if let Some(tok) = cursor.next() { if tok.kind == TokenKind::Punctuator(Punctuator::Question) { let then_clause = AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(parser)?; - parser.expect(Punctuator::Colon, "conditional expression")?; + .parse(cursor)?; + cursor.expect(Punctuator::Colon, "conditional expression")?; let else_clause = AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(parser)?; + .parse(cursor)?; return Ok(ConditionalOp::new(lhs, then_clause, else_clause).into()); } else { - parser.back(); + cursor.back(); } } diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 3697a579fac..e24b0a377f9 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -17,7 +17,7 @@ use crate::{ }, parser::{ expression::{unary::UnaryExpression, update::UpdateExpression}, - AllowAwait, AllowYield, ParseResult, Parser, TokenParser, + AllowAwait, AllowYield, ParseResult, Cursor, TokenParser, }, }, BoaProfiler, @@ -80,18 +80,18 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExponentiationExpression", "Parsing"); - if Self::is_unary_expression(parser) { - return UnaryExpression::new(self.allow_yield, self.allow_await).parse(parser); + if Self::is_unary_expression(cursor) { + return UnaryExpression::new(self.allow_yield, self.allow_await).parse(cursor); } - let lhs = UpdateExpression::new(self.allow_yield, self.allow_await).parse(parser)?; - if let Some(tok) = parser.next() { + let lhs = UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; + if let Some(tok) = cursor.next() { if let TokenKind::Punctuator(Punctuator::Exp) = tok.kind { - return Ok(BinOp::new(NumOp::Exp, lhs, self.parse(parser)?).into()); + return Ok(BinOp::new(NumOp::Exp, lhs, self.parse(cursor)?).into()); } else { - parser.back(); + cursor.back(); } } Ok(lhs) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 6e3c1913963..1fcd7403b91 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -19,7 +19,7 @@ use crate::{ node::{Assign, BinOp, Node}, Keyword, Punctuator, }, - parser::{AllowAwait, AllowIn, AllowYield, ParseError, ParseResult, Parser, TokenParser}, + parser::{AllowAwait, AllowIn, AllowYield, ParseError, ParseResult, Cursor, TokenParser}, }, BoaProfiler, }; @@ -78,18 +78,18 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); // Arrow function - let next_token = parser.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; match next_token.kind { // a=>{} TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield) | TokenKind::Keyword(Keyword::Await) - if parser.peek_expect_no_lineterminator(1).is_ok() => + if cursor.peek_expect_no_lineterminator(1).is_ok() => { - if let Some(tok) = parser.peek(1) { + if let Some(tok) = cursor.peek(1) { if tok.kind == TokenKind::Punctuator(Punctuator::Arrow) { return ArrowFunction::new( self.allow_in, @@ -115,20 +115,20 @@ where } let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) - .parse(parser)?; + .parse(cursor)?; - if let Some(tok) = parser.next() { + if let Some(tok) = cursor.next() { match tok.kind { TokenKind::Punctuator(Punctuator::Assign) => { - lhs = Assign::new(lhs, self.parse(parser)?).into(); + lhs = Assign::new(lhs, self.parse(cursor)?).into(); } TokenKind::Punctuator(p) if p.as_binop().is_some() => { - let expr = self.parse(parser)?; + let expr = self.parse(cursor)?; let binop = p.as_binop().expect("binop disappeared"); lhs = BinOp::new(binop, lhs, expr).into(); } _ => { - parser.back(); + cursor.back(); } } } diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index e2b6e324d30..e243f17de41 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -55,12 +55,12 @@ where { type Output = Box<[Node]>; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Arguments", "Parsing"); - parser.expect(Punctuator::OpenParen, "arguments")?; + cursor.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); loop { - let next_token = parser.next().ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; match next_token.kind { TokenKind::Punctuator(Punctuator::CloseParen) => break, TokenKind::Punctuator(Punctuator::Comma) => { @@ -68,7 +68,7 @@ where return Err(ParseError::unexpected(next_token.clone(), None)); } - if parser.next_if(Punctuator::CloseParen).is_some() { + if cursor.next_if(Punctuator::CloseParen).is_some() { break; } } @@ -83,23 +83,23 @@ where "argument list", )); } else { - parser.back(); + cursor.back(); } } } - if parser.next_if(Punctuator::Spread).is_some() { + if cursor.next_if(Punctuator::Spread).is_some() { args.push( Spread::new( AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(parser)?, + .parse(cursor)?, ) .into(), ); } else { args.push( AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(parser)?, + .parse(cursor)?, ); } } diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 5ac118ecc85..fb28e7ef7df 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -19,7 +19,7 @@ use crate::{ Punctuator, }, parser::{ - expression::Expression, AllowAwait, AllowYield, ParseError, ParseResult, Parser, + expression::Expression, AllowAwait, AllowYield, ParseError, ParseResult, Cursor, TokenParser, }, }, @@ -62,7 +62,7 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("CallExpression", "Parsing"); let mut lhs = match parser.peek(0) { Some(tk) if tk.kind == TokenKind::Punctuator(Punctuator::OpenParen) => { @@ -82,12 +82,12 @@ where while let Some(tok) = parser.peek(0) { match tok.kind { TokenKind::Punctuator(Punctuator::OpenParen) => { - let args = Arguments::new(self.allow_yield, self.allow_await).parse(parser)?; + let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; lhs = Node::from(Call::new(lhs, args)); } TokenKind::Punctuator(Punctuator::Dot) => { - let _ = parser.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. - match &parser.next().ok_or(ParseError::AbruptEnd)?.kind { + let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. + match &cursor.next().ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into(); } @@ -104,10 +104,10 @@ where } } TokenKind::Punctuator(Punctuator::OpenBracket) => { - let _ = parser.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. + let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. let idx = - Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - parser.expect(Punctuator::CloseBracket, "call expression")?; + Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect(Punctuator::CloseBracket, "call expression")?; lhs = GetField::new(lhs, idx).into(); } _ => break, diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index c3437bd568b..50c5d0205b5 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -18,7 +18,7 @@ use crate::{ }, parser::{ expression::{primary::PrimaryExpression, Expression}, - AllowAwait, AllowYield, ParseError, ParseResult, Parser, TokenParser, + AllowAwait, AllowYield, ParseError, ParseResult, Cursor, TokenParser, }, }, BoaProfiler, @@ -58,25 +58,25 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("MemberExpression", "Parsing"); let mut lhs = if parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind == TokenKind::Keyword(Keyword::New) { - let _ = parser.next().expect("keyword disappeared"); + let _ = cursor.next().expect("keyword disappeared"); let lhs = self.parse(parser)?; - let args = Arguments::new(self.allow_yield, self.allow_await).parse(parser)?; + let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; let call_node = Call::new(lhs, args); Node::from(New::from(call_node)) } else { - PrimaryExpression::new(self.allow_yield, self.allow_await).parse(parser)? + PrimaryExpression::new(self.allow_yield, self.allow_await).parse(cursor)? }; - while let Some(tok) = parser.peek(0) { + while let Some(tok) = cursor.peek(0) { match &tok.kind { TokenKind::Punctuator(Punctuator::Dot) => { - let _ = parser.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. - match &parser.next().ok_or(ParseError::AbruptEnd)?.kind { + let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. + match &cursor.next().ok_or(ParseError::AbruptEnd)?.kind { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into() } @@ -93,10 +93,10 @@ where } } TokenKind::Punctuator(Punctuator::OpenBracket) => { - let _ = parser.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. + let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. let idx = - Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - parser.expect(Punctuator::CloseBracket, "member expression")?; + Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect(Punctuator::CloseBracket, "member expression")?; lhs = GetField::new(lhs, idx).into(); } _ => break, diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index b1f41d1423f..1f45a21c803 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -16,7 +16,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Node, Punctuator}, - parser::{AllowAwait, AllowYield, ParseResult, Parser, TokenParser}, + parser::{AllowAwait, AllowYield, ParseResult, Cursor, TokenParser}, }, BoaProfiler, }; @@ -57,13 +57,13 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("LeftHandSIdeExpression", "Parsing"); // TODO: Implement NewExpression: new MemberExpression - let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(parser)?; - match parser.peek(0) { + let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; + match cursor.peek(0) { Some(ref tok) if tok.kind == TokenKind::Punctuator(Punctuator::OpenParen) => { - CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(parser) + CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor) } _ => Ok(lhs), // TODO: is this correct? } diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 3cb17048ed9..dbb2bea32c0 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -17,7 +17,7 @@ mod update; use self::assignment::ExponentiationExpression; pub(super) use self::{assignment::AssignmentExpression, primary::Initializer}; -use super::{AllowAwait, AllowIn, AllowYield, ParseResult, Parser, TokenParser}; +use super::{AllowAwait, AllowIn, AllowYield, ParseResult, Cursor, TokenParser}; use crate::syntax::lexer::TokenKind; use crate::{ profiler::BoaProfiler, @@ -59,9 +59,9 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); - let mut lhs = $lower::new($( self.$low_param ),*).parse(parser)?; + let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; while let Some(tok) = parser.peek(0) { match tok.kind { TokenKind::Punctuator(op) if $( op == $op )||* => { @@ -69,7 +69,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), lhs, - $lower::new($( self.$low_param ),*).parse(parser)? + $lower::new($( self.$low_param ),*).parse(cursor)? ).into(); } TokenKind::Keyword(op) if $( op == $op )||* => { @@ -77,7 +77,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), lhs, - $lower::new($( self.$low_param ),*).parse(parser)? + $lower::new($( self.$low_param ),*).parse(cursor)? ).into(); } _ => break diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index 624c85167d7..cdbafa4e5dd 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -17,7 +17,7 @@ use crate::{ Const, Punctuator, }, parser::{ - expression::AssignmentExpression, AllowAwait, AllowYield, ParseError, Parser, + expression::AssignmentExpression, AllowAwait, AllowYield, ParseError, Cursor, TokenParser, }, }, @@ -60,33 +60,33 @@ where { type Output = ArrayDecl; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ArrayLiteral", "Parsing"); let mut elements = Vec::new(); loop { // TODO: Support all features. - while parser.next_if(Punctuator::Comma).is_some() { + while cursor.next_if(Punctuator::Comma).is_some() { elements.push(Node::Const(Const::Undefined)); } - if parser.next_if(Punctuator::CloseBracket).is_some() { + if cursor.next_if(Punctuator::CloseBracket).is_some() { break; } - let _ = parser.peek(0).ok_or(ParseError::AbruptEnd)?; // Check that there are more tokens to read. + let _ = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; // Check that there are more tokens to read. - if parser.next_if(Punctuator::Spread).is_some() { + if cursor.next_if(Punctuator::Spread).is_some() { let node = AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(parser)?; + .parse(cursor)?; elements.push(Spread::new(node).into()); } else { elements.push( AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(parser)?, + .parse(cursor)?, ); } - parser.next_if(Punctuator::Comma); + cursor.next_if(Punctuator::Comma); } Ok(elements.into()) diff --git a/boa/src/syntax/parser/expression/primary/function_expression.rs b/boa/src/syntax/parser/expression/primary/function_expression.rs index fcb1699743b..113c9500313 100644 --- a/boa/src/syntax/parser/expression/primary/function_expression.rs +++ b/boa/src/syntax/parser/expression/primary/function_expression.rs @@ -13,7 +13,7 @@ use crate::{ parser::{ function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - ParseError, Parser, TokenParser, + ParseError, Cursor, TokenParser, }, }, BoaProfiler, @@ -38,20 +38,20 @@ where { type Output = FunctionExpr; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("FunctionExpression", "Parsing"); - let name = BindingIdentifier::new(false, false).try_parse(parser); + let name = BindingIdentifier::new(false, false).try_parse(cursor); - parser.expect(Punctuator::OpenParen, "function expression")?; + cursor.expect(Punctuator::OpenParen, "function expression")?; - let params = FormalParameters::new(false, false).parse(parser)?; + let params = FormalParameters::new(false, false).parse(cursor)?; - parser.expect(Punctuator::CloseParen, "function expression")?; - parser.expect(Punctuator::OpenBlock, "function expression")?; + cursor.expect(Punctuator::CloseParen, "function expression")?; + cursor.expect(Punctuator::OpenBlock, "function expression")?; - let body = FunctionBody::new(false, false).parse(parser)?; + let body = FunctionBody::new(false, false).parse(cursor)?; - parser.expect(Punctuator::CloseBlock, "function expression")?; + cursor.expect(Punctuator::CloseBlock, "function expression")?; Ok(FunctionExpr::new(name, params, body)) } diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 30e33094dea..7e489f8810c 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -24,7 +24,7 @@ use crate::syntax::{ node::{Call, Identifier, New, Node}, Const, Keyword, Punctuator, }, - parser::{AllowAwait, AllowYield, ParseError, ParseResult, Parser, TokenParser}, + parser::{AllowAwait, AllowYield, ParseError, ParseResult, Cursor, TokenParser}, }; pub(in crate::syntax::parser) use object_initializer::Initializer; @@ -64,29 +64,29 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { - let tok = parser.next().ok_or(ParseError::AbruptEnd)?; + fn parse(self, cursor: &mut Cursor) -> ParseResult { + let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; match &tok.kind { TokenKind::Keyword(Keyword::This) => Ok(Node::This), // TokenKind::Keyword(Keyword::Arguments) => Ok(Node::new(NodeBase::Arguments, tok.pos)), TokenKind::Keyword(Keyword::Function) => { - FunctionExpression.parse(parser).map(Node::from) + FunctionExpression.parse(cursor).map(Node::from) } TokenKind::Punctuator(Punctuator::OpenParen) => { let expr = - Expression::new(true, self.allow_yield, self.allow_await).parse(parser)?; - parser.expect(Punctuator::CloseParen, "primary expression")?; + Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + cursor.expect(Punctuator::CloseParen, "primary expression")?; Ok(expr) } TokenKind::Punctuator(Punctuator::OpenBracket) => { ArrayLiteral::new(self.allow_yield, self.allow_await) - .parse(parser) + .parse(cursor) .map(Node::ArrayDecl) } TokenKind::Punctuator(Punctuator::OpenBlock) => { Ok(ObjectLiteral::new(self.allow_yield, self.allow_await) - .parse(parser)? + .parse(cursor)? .into()) } TokenKind::BooleanLiteral(boolean) => Ok(Const::from(*boolean).into()), diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 57b02a6c987..b478f1db1f3 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -19,7 +19,7 @@ use crate::{ parser::{ expression::AssignmentExpression, function::{FormalParameters, FunctionBody}, - AllowAwait, AllowIn, AllowYield, ParseError, ParseResult, Parser, TokenParser, + AllowAwait, AllowIn, AllowYield, ParseError, ParseResult, Cursor, TokenParser, }, }, BoaProfiler, @@ -60,24 +60,24 @@ where { type Output = Object; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ObjectLiteral", "Parsing"); let mut elements = Vec::new(); loop { - if parser.next_if(Punctuator::CloseBlock).is_some() { + if cursor.next_if(Punctuator::CloseBlock).is_some() { break; } elements - .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(parser)?); + .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(cursor)?); - if parser.next_if(Punctuator::CloseBlock).is_some() { + if cursor.next_if(Punctuator::CloseBlock).is_some() { break; } - if parser.next_if(Punctuator::Comma).is_none() { - let next_token = parser.next().ok_or(ParseError::AbruptEnd)?; + if cursor.next_if(Punctuator::Comma).is_none() { + let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![ TokenKind::Punctuator(Punctuator::Comma), @@ -125,33 +125,33 @@ where { type Output = node::PropertyDefinition; - fn parse(self, parser: &mut Parser) -> Result { - if parser.next_if(Punctuator::Spread).is_some() { + fn parse(self, cursor: &mut Cursor) -> Result { + if cursor.next_if(Punctuator::Spread).is_some() { let node = AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(parser)?; + .parse(cursor)?; return Ok(node::PropertyDefinition::SpreadObject(node)); } - let prop_name = parser + let prop_name = cursor .next() .map(Token::to_string) .ok_or(ParseError::AbruptEnd)?; - if parser.next_if(Punctuator::Colon).is_some() { + if cursor.next_if(Punctuator::Colon).is_some() { let val = AssignmentExpression::new(true, self.allow_yield, self.allow_await) - .parse(parser)?; + .parse(cursor)?; return Ok(node::PropertyDefinition::property(prop_name, val)); } - if parser + if cursor .next_if(TokenKind::Punctuator(Punctuator::OpenParen)) .is_some() || ["get", "set"].contains(&prop_name.as_str()) { return MethodDefinition::new(self.allow_yield, self.allow_await, prop_name) - .parse(parser); + .parse(cursor); } - let pos = parser + let pos = cursor .peek(0) .map(|tok| tok.span().start()) .ok_or(ParseError::AbruptEnd)?; @@ -194,20 +194,20 @@ where { type Output = node::PropertyDefinition; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { let (methodkind, prop_name, params) = match self.identifier.as_str() { idn @ "get" | idn @ "set" => { let prop_name = parser .next() .map(Token::to_string) .ok_or(ParseError::AbruptEnd)?; - parser.expect( + cursor.expect( TokenKind::Punctuator(Punctuator::OpenParen), "property method definition", )?; - let first_param = parser.peek(0).expect("current token disappeared").clone(); - let params = FormalParameters::new(false, false).parse(parser)?; - parser.expect(Punctuator::CloseParen, "method definition")?; + let first_param = cursor.peek(0).expect("current token disappeared").clone(); + let params = FormalParameters::new(false, false).parse(cursor)?; + cursor.expect(Punctuator::CloseParen, "method definition")?; if idn == "get" { if !params.is_empty() { return Err(ParseError::unexpected( @@ -227,7 +227,7 @@ where } } prop_name => { - let params = FormalParameters::new(false, false).parse(parser)?; + let params = FormalParameters::new(false, false).parse(cursor)?; parser.expect(Punctuator::CloseParen, "method definition")?; ( MethodDefinitionKind::Ordinary, @@ -294,8 +294,8 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { - parser.expect(TokenKind::Punctuator(Punctuator::Assign), "initializer")?; - AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await).parse(parser) + fn parse(self, cursor: &mut Cursor) -> ParseResult { + cursor.expect(TokenKind::Punctuator(Punctuator::Assign), "initializer")?; + AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await).parse(cursor) } } diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index f52155b0832..c0af1ac6b46 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -145,11 +145,11 @@ where { type Output = node::FormalParameter; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { // FIXME: we are reading the spread operator before the rest element. // parser.expect(Punctuator::Spread, "rest parameter")?; - let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; + let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; // TODO: BindingPattern Ok(Self::Output::new(param, None, true)) @@ -190,12 +190,12 @@ where { type Output = node::FormalParameter; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { // TODO: BindingPattern - let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(parser)?; + let param = BindingIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; - let init = Initializer::new(true, self.allow_yield, self.allow_await).try_parse(parser); + let init = Initializer::new(true, self.allow_yield, self.allow_await).try_parse(cursor); Ok(Self::Output::new(param, init, false)) } @@ -241,13 +241,13 @@ where { type Output = node::StatementList; - fn parse(self, parser: &mut Parser) -> Result { + fn parse(self, cursor: &mut Cursor) -> Result { if let Some(tk) = parser.peek(0) { if tk.kind == Punctuator::CloseBlock.into() { return Ok(Vec::new().into()); } } - StatementList::new(self.allow_yield, self.allow_await, true, true).parse(parser) + StatementList::new(self.allow_yield, self.allow_await, true, true).parse(cursor) } } From 71fa22700c354e763493d46cadbef58cd815848c Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 13:44:45 +0000 Subject: [PATCH 065/289] Marked all parser cursor methods as unimplemented --- boa/src/syntax/parser/cursor.rs | 297 +++++++++--------- boa/src/syntax/parser/cursor_old.rs | 241 -------------- .../expression/assignment/exponentiation.rs | 4 +- .../parser/expression/assignment/mod.rs | 4 +- .../expression/left_hand_side/arguments.rs | 2 +- .../parser/expression/left_hand_side/call.rs | 8 +- .../expression/left_hand_side/member.rs | 4 +- boa/src/syntax/parser/expression/mod.rs | 6 +- .../primary/object_initializer/mod.rs | 10 +- boa/src/syntax/parser/expression/unary.rs | 24 +- boa/src/syntax/parser/expression/update.rs | 22 +- boa/src/syntax/parser/function/mod.rs | 2 +- boa/src/syntax/parser/mod.rs | 4 + boa/src/syntax/parser/tests.rs | 14 +- 14 files changed, 204 insertions(+), 438 deletions(-) delete mode 100644 boa/src/syntax/parser/cursor_old.rs diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index a28e8a54b08..f76ddf349c4 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -33,92 +33,99 @@ where /// Retrieves the current position of the cursor in the token stream. pub(super) fn pos(&self) -> usize { - self.pos + unimplemented!(); + // self.pos } /// Moves the cursor to the given position. /// /// This is intended to be used *always* with `Cursor::pos()`. pub(super) fn seek(&mut self, pos: usize) { - self.pos = pos + unimplemented!(); + // self.pos = pos } /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option { - loop { - let token = self.tokens.get(self.pos); - if let Some(tk) = token { - self.pos += 1; - - if tk.kind != TokenKind::LineTerminator { - break Some(tk); - } - } else { - break None; - } - } + unimplemented!(); + // loop { + // let token = self.tokens.get(self.pos); + // if let Some(tk) = token { + // self.pos += 1; + + // if tk.kind != TokenKind::LineTerminator { + // break Some(tk); + // } + // } else { + // break None; + // } + // } } /// Peeks the next token without moving the cursor. pub(super) fn peek(&self, skip: usize) -> Option<&Token> { - let mut count = 0; - let mut skipped = 0; - loop { - let token = self.tokens.get(self.pos + count); - count += 1; - - if let Some(tk) = token { - if tk.kind != TokenKind::LineTerminator { - if skipped == skip { - break Some(tk); - } - - skipped += 1; - } - } else { - break None; - } - } + unimplemented!(); + // let mut count = 0; + // let mut skipped = 0; + // loop { + // let token = self.tokens.get(self.pos + count); + // count += 1; + + // if let Some(tk) = token { + // if tk.kind != TokenKind::LineTerminator { + // if skipped == skip { + // break Some(tk); + // } + + // skipped += 1; + // } + // } else { + // break None; + // } + // } } /// Moves the cursor to the previous token and returns the token. pub(super) fn back(&mut self) { - debug_assert!( - self.pos > 0, - "cannot go back in a cursor that is at the beginning of the list of tokens" - ); - - self.pos -= 1; - while self - .tokens - .get(self.pos - 1) - .expect("token disappeared") - .kind - == TokenKind::LineTerminator - && self.pos > 0 - { - self.pos -= 1; - } + unimplemented!(); + + // debug_assert!( + // self.pos > 0, + // "cannot go back in a cursor that is at the beginning of the list of tokens" + // ); + + // self.pos -= 1; + // while self + // .tokens + // .get(self.pos - 1) + // .expect("token disappeared") + // .kind + // == TokenKind::LineTerminator + // && self.pos > 0 + // { + // self.pos -= 1; + // } } /// Peeks the previous token without moving the cursor. pub(super) fn peek_prev(&self) -> Option<&Token> { - if self.pos == 0 { - None - } else { - let mut back = 1; - let mut tok = self.tokens.get(self.pos - back).expect("token disappeared"); - while self.pos >= back && tok.kind == TokenKind::LineTerminator { - back += 1; - tok = self.tokens.get(self.pos - back).expect("token disappeared"); - } - - if back == self.pos { - None - } else { - Some(tok) - } - } + unimplemented!(); + // if self.pos == 0 { + // None + // } else { + // let mut back = 1; + // let mut tok = self.tokens.get(self.pos - back).expect("token disappeared"); + // while self.pos >= back && tok.kind == TokenKind::LineTerminator { + // back += 1; + // tok = self.tokens.get(self.pos - back).expect("token disappeared"); + // } + + // if back == self.pos { + // None + // } else { + // Some(tok) + // } + // } } /// Returns an error if the next token is not of kind `kind`. @@ -128,18 +135,19 @@ where where K: Into, { - let next_token = self.next().ok_or(ParseError::AbruptEnd)?; - let kind = kind.into(); - - if next_token.kind == kind { - Ok(()) - } else { - Err(ParseError::expected( - vec![kind], - next_token.clone(), - context, - )) - } + unimplemented!(); + // let next_token = self.next().ok_or(ParseError::AbruptEnd)?; + // let kind = kind.into(); + + // if next_token.kind == kind { + // Ok(()) + // } else { + // Err(ParseError::expected( + // vec![kind], + // next_token.clone(), + // context, + // )) + // } } /// It will peek for the next token, to see if it's a semicolon. @@ -148,33 +156,34 @@ where /// /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion pub(super) fn peek_semicolon(&self, do_while: bool) -> (bool, Option<&Token>) { - match self.tokens.get(self.pos) { - Some(tk) => match tk.kind { - TokenKind::Punctuator(Punctuator::Semicolon) => (true, Some(tk)), - TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { - (true, Some(tk)) - } - _ => { - if do_while { - debug_assert!( - self.pos != 0, - "cannot be finishing a do-while if we are at the beginning" - ); - - let tok = self - .tokens - .get(self.pos - 1) - .expect("could not find previous token"); - if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { - return (true, Some(tk)); - } - } - - (false, Some(tk)) - } - }, - None => (true, None), - } + unimplemented!(); + // match self.tokens.get(self.pos) { + // Some(tk) => match tk.kind { + // TokenKind::Punctuator(Punctuator::Semicolon) => (true, Some(tk)), + // TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { + // (true, Some(tk)) + // } + // _ => { + // if do_while { + // debug_assert!( + // self.pos != 0, + // "cannot be finishing a do-while if we are at the beginning" + // ); + + // let tok = self + // .tokens + // .get(self.pos - 1) + // .expect("could not find previous token"); + // if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { + // return (true, Some(tk)); + // } + // } + + // (false, Some(tk)) + // } + // }, + // None => (true, None), + // } } /// It will check if the next token is a semicolon. @@ -187,45 +196,48 @@ where do_while: bool, context: &'static str, ) -> Result<(), ParseError> { - match self.peek_semicolon(do_while) { - (true, Some(tk)) => match tk.kind { - TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { - self.pos += 1; - Ok(()) - } - _ => Ok(()), - }, - (true, None) => Ok(()), - (false, Some(tk)) => Err(ParseError::expected( - vec![TokenKind::Punctuator(Punctuator::Semicolon)], - tk.clone(), - context, - )), - (false, None) => unreachable!(), - } + unimplemented!(); + + // match self.peek_semicolon(do_while) { + // (true, Some(tk)) => match tk.kind { + // TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { + // self.pos += 1; + // Ok(()) + // } + // _ => Ok(()), + // }, + // (true, None) => Ok(()), + // (false, Some(tk)) => Err(ParseError::expected( + // vec![TokenKind::Punctuator(Punctuator::Semicolon)], + // tk.clone(), + // context, + // )), + // (false, None) => unreachable!(), + // } } /// It will make sure that the next token is not a line terminator. /// /// It expects that the token stream does not end here. pub(super) fn peek_expect_no_lineterminator(&mut self, skip: usize) -> Result<(), ParseError> { - let mut count = 0; - let mut skipped = 0; - loop { - let token = self.tokens.get(self.pos + count); - count += 1; - if let Some(tk) = token { - if skipped == skip && tk.kind == TokenKind::LineTerminator { - break Err(ParseError::unexpected(tk.clone(), None)); - } else if skipped == skip && tk.kind != TokenKind::LineTerminator { - break Ok(()); - } else if tk.kind != TokenKind::LineTerminator { - skipped += 1; - } - } else { - break Err(ParseError::AbruptEnd); - } - } + unimplemented!(); + // let mut count = 0; + // let mut skipped = 0; + // loop { + // let token = self.tokens.get(self.pos + count); + // count += 1; + // if let Some(tk) = token { + // if skipped == skip && tk.kind == TokenKind::LineTerminator { + // break Err(ParseError::unexpected(tk.clone(), None)); + // } else if skipped == skip && tk.kind != TokenKind::LineTerminator { + // break Ok(()); + // } else if tk.kind != TokenKind::LineTerminator { + // skipped += 1; + // } + // } else { + // break Err(ParseError::AbruptEnd); + // } + // } } /// Advance the cursor to the next token and retrieve it, only if it's of `kind` type. @@ -236,12 +248,13 @@ where where K: Into, { - let next_token = self.peek(0)?; + unimplemented!(); + // let next_token = self.peek(0)?; - if next_token.kind == kind.into() { - self.next() - } else { - None - } + // if next_token.kind == kind.into() { + // self.next() + // } else { + // None + // } } } diff --git a/boa/src/syntax/parser/cursor_old.rs b/boa/src/syntax/parser/cursor_old.rs deleted file mode 100644 index dfa29db3b5f..00000000000 --- a/boa/src/syntax/parser/cursor_old.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! Cursor implementation for the parser. - -use super::ParseError; -use crate::syntax::ast::Punctuator; -use crate::syntax::lexer::{Token, TokenKind}; - -/// Token cursor. -/// -/// This internal structure gives basic testable operations to the parser. -#[derive(Debug, Clone, Default)] -pub(super) struct Cursor { - /// The tokens being input. - // tokens: &'a [Token], - lexer: crate::syntax::lexer::Lexer, - /// The current position within the tokens. - pos: usize, -} - -impl<'a> Cursor { - /// Creates a new cursor. - pub(super) fn new(tokens: &'a [Token]) -> Self { - Self { - tokens, - ..Self::default() - } - } - - /// Retrieves the current position of the cursor in the token stream. - pub(super) fn pos(&self) -> usize { - self.pos - } - - /// Moves the cursor to the given position. - /// - /// This is intended to be used *always* with `Cursor::pos()`. - pub(super) fn seek(&mut self, pos: usize) { - self.pos = pos - } - - /// Moves the cursor to the next token and returns the token. - pub(super) fn next(&mut self) -> Option<&'a Token> { - loop { - let token = self.tokens.get(self.pos); - if let Some(tk) = token { - self.pos += 1; - - if tk.kind != TokenKind::LineTerminator { - break Some(tk); - } - } else { - break None; - } - } - } - - /// Peeks the next token without moving the cursor. - pub(super) fn peek(&self, skip: usize) -> Option<&'a Token> { - let mut count = 0; - let mut skipped = 0; - loop { - let token = self.tokens.get(self.pos + count); - count += 1; - - if let Some(tk) = token { - if tk.kind != TokenKind::LineTerminator { - if skipped == skip { - break Some(tk); - } - - skipped += 1; - } - } else { - break None; - } - } - } - - /// Moves the cursor to the previous token and returns the token. - pub(super) fn back(&mut self) { - debug_assert!( - self.pos > 0, - "cannot go back in a cursor that is at the beginning of the list of tokens" - ); - - self.pos -= 1; - while self - .tokens - .get(self.pos - 1) - .expect("token disappeared") - .kind - == TokenKind::LineTerminator - && self.pos > 0 - { - self.pos -= 1; - } - } - - /// Peeks the previous token without moving the cursor. - pub(super) fn peek_prev(&self) -> Option<&'a Token> { - if self.pos == 0 { - None - } else { - let mut back = 1; - let mut tok = self.tokens.get(self.pos - back).expect("token disappeared"); - while self.pos >= back && tok.kind == TokenKind::LineTerminator { - back += 1; - tok = self.tokens.get(self.pos - back).expect("token disappeared"); - } - - if back == self.pos { - None - } else { - Some(tok) - } - } - } - - /// Returns an error if the next token is not of kind `kind`. - /// - /// Note: it will consume the next token. - pub(super) fn expect(&mut self, kind: K, context: &'static str) -> Result<(), ParseError> - where - K: Into, - { - let next_token = self.next().ok_or(ParseError::AbruptEnd)?; - let kind = kind.into(); - - if next_token.kind == kind { - Ok(()) - } else { - Err(ParseError::expected( - vec![kind], - next_token.clone(), - context, - )) - } - } - - /// It will peek for the next token, to see if it's a semicolon. - /// - /// It will automatically insert a semicolon if needed, as specified in the [spec][spec]. - /// - /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion - pub(super) fn peek_semicolon(&self, do_while: bool) -> (bool, Option<&Token>) { - match self.tokens.get(self.pos) { - Some(tk) => match tk.kind { - TokenKind::Punctuator(Punctuator::Semicolon) => (true, Some(tk)), - TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { - (true, Some(tk)) - } - _ => { - if do_while { - debug_assert!( - self.pos != 0, - "cannot be finishing a do-while if we are at the beginning" - ); - - let tok = self - .tokens - .get(self.pos - 1) - .expect("could not find previous token"); - if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { - return (true, Some(tk)); - } - } - - (false, Some(tk)) - } - }, - None => (true, None), - } - } - - /// It will check if the next token is a semicolon. - /// - /// It will automatically insert a semicolon if needed, as specified in the [spec][spec]. - /// - /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion - pub(super) fn expect_semicolon( - &mut self, - do_while: bool, - context: &'static str, - ) -> Result<(), ParseError> { - match self.peek_semicolon(do_while) { - (true, Some(tk)) => match tk.kind { - TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { - self.pos += 1; - Ok(()) - } - _ => Ok(()), - }, - (true, None) => Ok(()), - (false, Some(tk)) => Err(ParseError::expected( - vec![TokenKind::Punctuator(Punctuator::Semicolon)], - tk.clone(), - context, - )), - (false, None) => unreachable!(), - } - } - - /// It will make sure that the next token is not a line terminator. - /// - /// It expects that the token stream does not end here. - pub(super) fn peek_expect_no_lineterminator(&mut self, skip: usize) -> Result<(), ParseError> { - let mut count = 0; - let mut skipped = 0; - loop { - let token = self.tokens.get(self.pos + count); - count += 1; - if let Some(tk) = token { - if skipped == skip && tk.kind == TokenKind::LineTerminator { - break Err(ParseError::unexpected(tk.clone(), None)); - } else if skipped == skip && tk.kind != TokenKind::LineTerminator { - break Ok(()); - } else if tk.kind != TokenKind::LineTerminator { - skipped += 1; - } - } else { - break Err(ParseError::AbruptEnd); - } - } - } - - /// Advance the cursor to the next token and retrieve it, only if it's of `kind` type. - /// - /// When the next token is a `kind` token, get the token, otherwise return `None`. This - /// function skips line terminators. - pub(super) fn next_if(&mut self, kind: K) -> Option<&'a Token> - where - K: Into, - { - let next_token = self.peek(0)?; - - if next_token.kind == kind.into() { - self.next() - } else { - None - } - } -} diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index e24b0a377f9..2123f48a1f0 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -54,11 +54,11 @@ impl ExponentiationExpression { } /// Checks by looking at the next token to see whether it's a unary operator or not. -fn is_unary_expression(parser: &mut Parser) -> bool +fn is_unary_expression(cursor: &mut Cursor) -> bool where R: Read, { - if let Some(tok) = parser.peek(0) { + if let Some(tok) = cursor.peek(0) { match tok.kind { TokenKind::Keyword(Keyword::Delete) | TokenKind::Keyword(Keyword::Void) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 1fcd7403b91..99617bc5610 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -96,7 +96,7 @@ where self.allow_yield, self.allow_await, ) - .parse(parser) + .parse(cursor) .map(Node::ArrowFunctionDecl); } } @@ -105,7 +105,7 @@ where TokenKind::Punctuator(Punctuator::OpenParen) => { if let Some(node) = ArrowFunction::new(self.allow_in, self.allow_yield, self.allow_await) - .try_parse(parser) + .try_parse(cursor) .map(Node::ArrowFunctionDecl) { return Ok(node); diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index e243f17de41..a207dc95354 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -12,7 +12,7 @@ use crate::{ syntax::{ ast::{node::Spread, Node, Punctuator}, parser::{ - expression::AssignmentExpression, AllowAwait, AllowYield, ParseError, Parser, + expression::AssignmentExpression, AllowAwait, AllowYield, ParseError, Cursor, TokenParser, }, }, diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index fb28e7ef7df..1ea5754d806 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -64,13 +64,13 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("CallExpression", "Parsing"); - let mut lhs = match parser.peek(0) { + let mut lhs = match cursor.peek(0) { Some(tk) if tk.kind == TokenKind::Punctuator(Punctuator::OpenParen) => { - let args = Arguments::new(self.allow_yield, self.allow_await).parse(parser)?; + let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; Node::from(Call::new(self.first_member_expr, args)) } _ => { - let next_token = parser.next().ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::OpenParen)], next_token.clone(), @@ -79,7 +79,7 @@ where } }; - while let Some(tok) = parser.peek(0) { + while let Some(tok) = cursor.peek(0) { match tok.kind { TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 50c5d0205b5..7ad1daf4f55 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -60,11 +60,11 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("MemberExpression", "Parsing"); - let mut lhs = if parser.peek(0).ok_or(ParseError::AbruptEnd)?.kind + let mut lhs = if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind == TokenKind::Keyword(Keyword::New) { let _ = cursor.next().expect("keyword disappeared"); - let lhs = self.parse(parser)?; + let lhs = self.parse(cursor)?; let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; let call_node = Call::new(lhs, args); diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index dbb2bea32c0..861ff251071 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -62,10 +62,10 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; - while let Some(tok) = parser.peek(0) { + while let Some(tok) = cursor.peek(0) { match tok.kind { TokenKind::Punctuator(op) if $( op == $op )||* => { - let _ = parser.next().expect("token disappeared"); + let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), lhs, @@ -73,7 +73,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo ).into(); } TokenKind::Keyword(op) if $( op == $op )||* => { - let _ = parser.next().expect("token disappeared"); + let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), lhs, diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index b478f1db1f3..d790e1df8d2 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -197,7 +197,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let (methodkind, prop_name, params) = match self.identifier.as_str() { idn @ "get" | idn @ "set" => { - let prop_name = parser + let prop_name = cursor .next() .map(Token::to_string) .ok_or(ParseError::AbruptEnd)?; @@ -228,7 +228,7 @@ where } prop_name => { let params = FormalParameters::new(false, false).parse(cursor)?; - parser.expect(Punctuator::CloseParen, "method definition")?; + cursor.expect(Punctuator::CloseParen, "method definition")?; ( MethodDefinitionKind::Ordinary, prop_name.to_string(), @@ -237,12 +237,12 @@ where } }; - parser.expect( + cursor.expect( TokenKind::Punctuator(Punctuator::OpenBlock), "property method definition", )?; - let body = FunctionBody::new(false, false).parse(parser)?; - parser.expect( + let body = FunctionBody::new(false, false).parse(cursor)?; + cursor.expect( TokenKind::Punctuator(Punctuator::CloseBlock), "property method definition", )?; diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 0931eb165a7..cd24b0a7a0a 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -16,7 +16,7 @@ use crate::syntax::{ }, parser::{ expression::update::UpdateExpression, AllowAwait, AllowYield, ParseError, ParseResult, - Parser, TokenParser, + Cursor, TokenParser, }, }; @@ -56,33 +56,33 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { - let tok = parser.next().ok_or(ParseError::AbruptEnd)?; + fn parse(self, cursor: &mut Cursor) -> ParseResult { + let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Delete) => { - Ok(node::UnaryOp::new(UnaryOp::Delete, self.parse(parser)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Delete, self.parse(cursor)?).into()) } TokenKind::Keyword(Keyword::Void) => { - Ok(node::UnaryOp::new(UnaryOp::Void, self.parse(parser)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Void, self.parse(cursor)?).into()) } TokenKind::Keyword(Keyword::TypeOf) => { - Ok(node::UnaryOp::new(UnaryOp::TypeOf, self.parse(parser)?).into()) + Ok(node::UnaryOp::new(UnaryOp::TypeOf, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Add) => { - Ok(node::UnaryOp::new(UnaryOp::Plus, self.parse(parser)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Plus, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Sub) => { - Ok(node::UnaryOp::new(UnaryOp::Minus, self.parse(parser)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Minus, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Neg) => { - Ok(node::UnaryOp::new(UnaryOp::Tilde, self.parse(parser)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Tilde, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Not) => { - Ok(node::UnaryOp::new(UnaryOp::Not, self.parse(parser)?).into()) + Ok(node::UnaryOp::new(UnaryOp::Not, self.parse(cursor)?).into()) } _ => { - parser.back(); - UpdateExpression::new(self.allow_yield, self.allow_await).parse(parser) + cursor.back(); + UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor) } } } diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index e8f837392a2..07609bc0e1f 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -9,7 +9,7 @@ use super::left_hand_side::LeftHandSideExpression; use crate::syntax::lexer::TokenKind; use crate::syntax::{ ast::{node, op::UnaryOp, Node, Punctuator}, - parser::{AllowAwait, AllowYield, ParseError, ParseResult, Parser, TokenParser}, + parser::{AllowAwait, AllowYield, ParseError, ParseResult, Cursor, TokenParser}, }; use std::io::Read; @@ -46,39 +46,39 @@ where { type Output = Node; - fn parse(self, parser: &mut Parser) -> ParseResult { - let tok = parser.peek(0).ok_or(ParseError::AbruptEnd)?; + fn parse(self, cursor: &mut Cursor) -> ParseResult { + let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Punctuator(Punctuator::Inc) => { - parser.next().expect("token disappeared"); + cursor.next().expect("token disappeared"); return Ok(node::UnaryOp::new( UnaryOp::IncrementPre, LeftHandSideExpression::new(self.allow_yield, self.allow_await) - .parse(parser)?, + .parse(cursor)?, ) .into()); } TokenKind::Punctuator(Punctuator::Dec) => { - parser.next().expect("token disappeared"); + cursor.next().expect("token disappeared"); return Ok(node::UnaryOp::new( UnaryOp::DecrementPre, LeftHandSideExpression::new(self.allow_yield, self.allow_await) - .parse(parser)?, + .parse(cursor)?, ) .into()); } _ => {} } - let lhs = LeftHandSideExpression::new(self.allow_yield, self.allow_await).parse(parser)?; - if let Some(tok) = parser.peek(0) { + let lhs = LeftHandSideExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; + if let Some(tok) = cursor.peek(0) { match tok.kind { TokenKind::Punctuator(Punctuator::Inc) => { - parser.next().expect("token disappeared"); + cursor.next().expect("token disappeared"); return Ok(node::UnaryOp::new(UnaryOp::IncrementPost, lhs).into()); } TokenKind::Punctuator(Punctuator::Dec) => { - parser.next().expect("token disappeared"); + cursor.next().expect("token disappeared"); return Ok(node::UnaryOp::new(UnaryOp::DecrementPost, lhs).into()); } _ => {} diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index c0af1ac6b46..b6c251b47e4 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -242,7 +242,7 @@ where type Output = node::StatementList; fn parse(self, cursor: &mut Cursor) -> Result { - if let Some(tk) = parser.peek(0) { + if let Some(tk) = cursor.peek(0) { if tk.kind == Punctuator::CloseBlock.into() { return Ok(Vec::new().into()); } diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 9d6fb2b88a0..77fb150f47b 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -36,6 +36,10 @@ where /// This method needs to be provided by the implementor type. fn parse(self, cursor: &mut Cursor) -> Result; + fn try_parse(self, cursor: &mut Cursor) -> Option { + unimplemented!(); + } + // /// Tries to parse the following tokens with this parser. // fn try_parse(self, parser: Parser) -> Option { // let initial_pos = cursor.pos(); diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 47fba3fdd10..c5cf4707225 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -20,13 +20,8 @@ pub(super) fn check_parser(js: &str, expr: L) where L: Into>, { - let lexer = Lexer::new(js.as_bytes()); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - assert_eq!( - Parser::new(&tokens).parse_all().expect("failed to parse"), + Parser::new(js.as_bytes()).parse_all().expect("failed to parse"), StatementList::from(expr) ); } @@ -34,12 +29,7 @@ where /// Checks that the given javascript string creates a parse error. // TODO: #[track_caller]: https://github.com/rust-lang/rust/issues/47809 pub(super) fn check_invalid(js: &str) { - let lexer = Lexer::new(js.as_bytes()); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - - assert!(Parser::new(&tokens).parse_all().is_err()); + assert!(Parser::new(js.as_bytes()).parse_all().is_err()); } /// Should be parsed as `new Class().method()` instead of `new (Class().method())` From 50aa88b6571275296ca89c00d3d90bff5e0d37e3 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 15:18:45 +0100 Subject: [PATCH 066/289] ParserCursor updated the return types of the new methods --- boa/src/syntax/lexer/mod.rs | 5 ++++- boa/src/syntax/parser/cursor.rs | 16 ++++++++-------- .../expression/assignment/exponentiation.rs | 2 +- .../expression/primary/object_initializer/mod.rs | 10 ++++++---- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 3ff581aa3d9..c0045daae58 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -107,7 +107,10 @@ impl Lexer { // that means it could be multiple different tokens depending on the input token. // // As per https://tc39.es/ecma262/#sec-ecmascript-language-lexical-grammar - fn lex_slash_token(&mut self, start: Position) -> Result { + fn lex_slash_token(&mut self, start: Position) -> Result + where + R: Read + { if let Some(c) = self.cursor.peek() { match c { Err(e) => { diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index f76ddf349c4..13981031164 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -10,7 +10,7 @@ use std::io::Read; /// Token cursor. /// /// This internal structure gives basic testable operations to the parser. -#[derive(Debug, Clone, Default)] +#[derive(Debug)] pub(super) struct Cursor { /// The tokens being input. // tokens: &'a [Token], @@ -46,7 +46,7 @@ where } /// Moves the cursor to the next token and returns the token. - pub(super) fn next(&mut self) -> Option { + pub(super) fn next(&mut self) -> Option> { unimplemented!(); // loop { // let token = self.tokens.get(self.pos); @@ -63,7 +63,7 @@ where } /// Peeks the next token without moving the cursor. - pub(super) fn peek(&self, skip: usize) -> Option<&Token> { + pub(super) fn peek(&self, skip: usize) -> Option<&Result> { unimplemented!(); // let mut count = 0; // let mut skipped = 0; @@ -86,7 +86,7 @@ where } /// Moves the cursor to the previous token and returns the token. - pub(super) fn back(&mut self) { + pub(super) fn back(&mut self) -> Option> { unimplemented!(); // debug_assert!( @@ -108,7 +108,7 @@ where } /// Peeks the previous token without moving the cursor. - pub(super) fn peek_prev(&self) -> Option<&Token> { + pub(super) fn peek_prev(&self) -> Option<&Result> { unimplemented!(); // if self.pos == 0 { // None @@ -131,7 +131,7 @@ where /// Returns an error if the next token is not of kind `kind`. /// /// Note: it will consume the next token. - pub(super) fn expect(&mut self, kind: K, context: &'static str) -> Result<(), ParseError> + pub(super) fn expect(&mut self, kind: K, context: &'static str) -> Result where K: Into, { @@ -195,7 +195,7 @@ where &mut self, do_while: bool, context: &'static str, - ) -> Result<(), ParseError> { + ) -> Result { unimplemented!(); // match self.peek_semicolon(do_while) { @@ -244,7 +244,7 @@ where /// /// When the next token is a `kind` token, get the token, otherwise return `None`. This /// function skips line terminators. - pub(super) fn next_if(&mut self, kind: K) -> Option + pub(super) fn next_if(&mut self, kind: K) -> Option> where K: Into, { diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 2123f48a1f0..b5faf48fdac 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -82,7 +82,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExponentiationExpression", "Parsing"); - if Self::is_unary_expression(cursor) { + if is_unary_expression(cursor) { return UnaryExpression::new(self.allow_yield, self.allow_await).parse(cursor); } diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index d790e1df8d2..b58efa61134 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -132,10 +132,12 @@ where return Ok(node::PropertyDefinition::SpreadObject(node)); } - let prop_name = cursor - .next() - .map(Token::to_string) - .ok_or(ParseError::AbruptEnd)?; + let prop_name = cursor.next().ok_or(ParseError::AbruptEnd)?.map(Token::to_string).ok_or(ParseError::AbruptEnd)?; + + // let prop_name = cursor + // .next() + // .map(Token::to_string) + // .ok_or(ParseError::AbruptEnd)?; if cursor.next_if(Punctuator::Colon).is_some() { let val = AssignmentExpression::new(true, self.allow_yield, self.allow_await) .parse(cursor)?; From 67edb646973280edd2663477d065122bc33bf20c Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 16:16:17 +0100 Subject: [PATCH 067/289] Continued refractor of parser with support for ParseErrors --- boa/src/syntax/parser/cursor.rs | 4 +-- .../expression/assignment/arrow_function.rs | 4 +-- .../expression/assignment/conditional.rs | 2 +- .../expression/assignment/exponentiation.rs | 35 ++++++++++--------- .../parser/expression/assignment/mod.rs | 6 ++-- .../expression/left_hand_side/arguments.rs | 4 +-- .../parser/expression/left_hand_side/call.rs | 11 +++--- .../expression/left_hand_side/member.rs | 15 ++++---- .../parser/expression/left_hand_side/mod.rs | 11 ++++-- boa/src/syntax/parser/expression/mod.rs | 6 ++-- .../syntax/parser/expression/primary/mod.rs | 4 +-- .../primary/object_initializer/mod.rs | 20 ++++------- boa/src/syntax/parser/expression/unary.rs | 2 +- boa/src/syntax/parser/expression/update.rs | 4 +-- boa/src/syntax/parser/function/mod.rs | 12 +++---- boa/src/syntax/parser/statement/block/mod.rs | 2 +- .../parser/statement/declaration/lexical.rs | 6 ++-- .../parser/statement/declaration/mod.rs | 2 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 2 +- .../statement/iteration/do_while_statement.rs | 4 +-- .../statement/iteration/for_statement.rs | 2 +- boa/src/syntax/parser/statement/mod.rs | 14 ++++---- boa/src/syntax/parser/statement/throw/mod.rs | 2 +- .../syntax/parser/statement/try_stm/mod.rs | 8 ++--- boa/src/syntax/parser/statement/variable.rs | 2 +- 25 files changed, 93 insertions(+), 91 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 13981031164..5929476f154 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -63,7 +63,7 @@ where } /// Peeks the next token without moving the cursor. - pub(super) fn peek(&self, skip: usize) -> Option<&Result> { + pub(super) fn peek(&self, skip: usize) -> Option> { unimplemented!(); // let mut count = 0; // let mut skipped = 0; @@ -108,7 +108,7 @@ where } /// Peeks the previous token without moving the cursor. - pub(super) fn peek_prev(&self) -> Option<&Result> { + pub(super) fn peek_prev(&self) -> Option> { unimplemented!(); // if self.pos == 0 { // None diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index 1c3557365d9..bb7c81ec59b 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -71,7 +71,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ArrowFunction", "Parsing"); let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; - let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token.kind { + let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token?.kind() { // CoverParenthesizedExpressionAndArrowParameterList cursor.expect(Punctuator::OpenParen, "arrow function")?; let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(cursor)?; @@ -119,7 +119,7 @@ where type Output = StatementList; fn parse(self, cursor: &mut Cursor) -> Result { - match cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind { + match cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() { TokenKind::Punctuator(Punctuator::OpenBlock) => { let _ = cursor.next(); let body = FunctionBody::new(false, false).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index e711bc404eb..14f12ae8da9 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -69,7 +69,7 @@ where .parse(cursor)?; if let Some(tok) = cursor.next() { - if tok.kind == TokenKind::Punctuator(Punctuator::Question) { + if tok?.kind() == &TokenKind::Punctuator(Punctuator::Question) { let then_clause = AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index b5faf48fdac..412d66b6125 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -8,6 +8,7 @@ //! [spec]: https://tc39.es/ecma262/#sec-exp-operator use crate::syntax::lexer::TokenKind; +use super::ParseError; use crate::{ syntax::{ ast::{ @@ -54,24 +55,26 @@ impl ExponentiationExpression { } /// Checks by looking at the next token to see whether it's a unary operator or not. -fn is_unary_expression(cursor: &mut Cursor) -> bool +fn is_unary_expression(cursor: &mut Cursor) -> Result where R: Read, { - if let Some(tok) = cursor.peek(0) { - match tok.kind { - TokenKind::Keyword(Keyword::Delete) - | TokenKind::Keyword(Keyword::Void) - | TokenKind::Keyword(Keyword::TypeOf) - | TokenKind::Punctuator(Punctuator::Add) - | TokenKind::Punctuator(Punctuator::Sub) - | TokenKind::Punctuator(Punctuator::Not) - | TokenKind::Punctuator(Punctuator::Neg) => true, - _ => false, + Ok( + if let Some(tok) = cursor.peek(0) { + match tok?.kind() { + TokenKind::Keyword(Keyword::Delete) + | TokenKind::Keyword(Keyword::Void) + | TokenKind::Keyword(Keyword::TypeOf) + | TokenKind::Punctuator(Punctuator::Add) + | TokenKind::Punctuator(Punctuator::Sub) + | TokenKind::Punctuator(Punctuator::Not) + | TokenKind::Punctuator(Punctuator::Neg) => true, + _ => false, + } + } else { + false } - } else { - false - } + ) } impl TokenParser for ExponentiationExpression @@ -82,13 +85,13 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExponentiationExpression", "Parsing"); - if is_unary_expression(cursor) { + if is_unary_expression(cursor)? { return UnaryExpression::new(self.allow_yield, self.allow_await).parse(cursor); } let lhs = UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; if let Some(tok) = cursor.next() { - if let TokenKind::Punctuator(Punctuator::Exp) = tok.kind { + if let TokenKind::Punctuator(Punctuator::Exp) = tok?.kind() { return Ok(BinOp::new(NumOp::Exp, lhs, self.parse(cursor)?).into()); } else { cursor.back(); diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 99617bc5610..f1e40456bf0 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -82,7 +82,7 @@ where let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); // Arrow function let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; - match next_token.kind { + match next_token?.kind() { // a=>{} TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield) @@ -90,7 +90,7 @@ where if cursor.peek_expect_no_lineterminator(1).is_ok() => { if let Some(tok) = cursor.peek(1) { - if tok.kind == TokenKind::Punctuator(Punctuator::Arrow) { + if tok?.kind() == &TokenKind::Punctuator(Punctuator::Arrow) { return ArrowFunction::new( self.allow_in, self.allow_yield, @@ -118,7 +118,7 @@ where .parse(cursor)?; if let Some(tok) = cursor.next() { - match tok.kind { + match tok?.kind() { TokenKind::Punctuator(Punctuator::Assign) => { lhs = Assign::new(lhs, self.parse(cursor)?).into(); } diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index a207dc95354..ea6d7664369 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -60,8 +60,8 @@ where cursor.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); loop { - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; - match next_token.kind { + let next_token = cursor.next().ok_or(ParseError::AbruptEnd)??; + match next_token.kind() { TokenKind::Punctuator(Punctuator::CloseParen) => break, TokenKind::Punctuator(Punctuator::Comma) => { if args.is_empty() { diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 1ea5754d806..2bbfd61273e 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -65,7 +65,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("CallExpression", "Parsing"); let mut lhs = match cursor.peek(0) { - Some(tk) if tk.kind == TokenKind::Punctuator(Punctuator::OpenParen) => { + Some(tk) if tk?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; Node::from(Call::new(self.first_member_expr, args)) } @@ -73,21 +73,22 @@ where let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::OpenParen)], - next_token.clone(), + next_token?.clone(), "call expression", )); } }; while let Some(tok) = cursor.peek(0) { - match tok.kind { + let token = tok?; + match token.kind() { TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; lhs = Node::from(Call::new(lhs, args)); } TokenKind::Punctuator(Punctuator::Dot) => { let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. - match &cursor.next().ok_or(ParseError::AbruptEnd)?.kind { + match &cursor.next().ok_or(ParseError::AbruptEnd)??.kind() { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into(); } @@ -97,7 +98,7 @@ where _ => { return Err(ParseError::expected( vec![TokenKind::identifier("identifier")], - tok.clone(), + token.clone(), "call expression", )); } diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 7ad1daf4f55..014bd187d50 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -60,8 +60,8 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("MemberExpression", "Parsing"); - let mut lhs = if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind - == TokenKind::Keyword(Keyword::New) + let mut lhs = if cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() + == &TokenKind::Keyword(Keyword::New) { let _ = cursor.next().expect("keyword disappeared"); let lhs = self.parse(cursor)?; @@ -73,10 +73,11 @@ where PrimaryExpression::new(self.allow_yield, self.allow_await).parse(cursor)? }; while let Some(tok) = cursor.peek(0) { - match &tok.kind { - TokenKind::Punctuator(Punctuator::Dot) => { + let token = tok?; + match token.kind() { + &TokenKind::Punctuator(Punctuator::Dot) => { let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. - match &cursor.next().ok_or(ParseError::AbruptEnd)?.kind { + match &cursor.next().ok_or(ParseError::AbruptEnd)??.kind() { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into() } @@ -86,13 +87,13 @@ where _ => { return Err(ParseError::expected( vec![TokenKind::identifier("identifier")], - tok.clone(), + token.clone(), "member expression", )); } } } - TokenKind::Punctuator(Punctuator::OpenBracket) => { + &TokenKind::Punctuator(Punctuator::OpenBracket) => { let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. let idx = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 1f45a21c803..167fd7211f8 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -20,6 +20,7 @@ use crate::{ }, BoaProfiler, }; +use super::super::ParseError; use std::io::Read; @@ -57,13 +58,17 @@ where { type Output = Node; - fn parse(self, cursor: &mut Cursor) -> ParseResult { + fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("LeftHandSIdeExpression", "Parsing"); // TODO: Implement NewExpression: new MemberExpression let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; match cursor.peek(0) { - Some(ref tok) if tok.kind == TokenKind::Punctuator(Punctuator::OpenParen) => { - CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor) + Some(tok) => { + if tok?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) { + CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor) + } else { + Ok(lhs) + } } _ => Ok(lhs), // TODO: is this correct? } diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 861ff251071..18724b2c664 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -63,8 +63,8 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; while let Some(tok) = cursor.peek(0) { - match tok.kind { - TokenKind::Punctuator(op) if $( op == $op )||* => { + match tok?.kind() { + &TokenKind::Punctuator(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), @@ -72,7 +72,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo $lower::new($( self.$low_param ),*).parse(cursor)? ).into(); } - TokenKind::Keyword(op) if $( op == $op )||* => { + &TokenKind::Keyword(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 7e489f8810c..38b2da36ecf 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -65,9 +65,9 @@ where type Output = Node; fn parse(self, cursor: &mut Cursor) -> ParseResult { - let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; + let tok = cursor.next().ok_or(ParseError::AbruptEnd)??; - match &tok.kind { + match tok.kind() { TokenKind::Keyword(Keyword::This) => Ok(Node::This), // TokenKind::Keyword(Keyword::Arguments) => Ok(Node::new(NodeBase::Arguments, tok.pos)), TokenKind::Keyword(Keyword::Function) => { diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index b58efa61134..3daa572d3e0 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -77,7 +77,7 @@ where } if cursor.next_if(Punctuator::Comma).is_none() { - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.next().ok_or(ParseError::AbruptEnd)??; return Err(ParseError::expected( vec![ TokenKind::Punctuator(Punctuator::Comma), @@ -132,12 +132,7 @@ where return Ok(node::PropertyDefinition::SpreadObject(node)); } - let prop_name = cursor.next().ok_or(ParseError::AbruptEnd)?.map(Token::to_string).ok_or(ParseError::AbruptEnd)?; - - // let prop_name = cursor - // .next() - // .map(Token::to_string) - // .ok_or(ParseError::AbruptEnd)?; + let prop_name = cursor.next().ok_or(ParseError::AbruptEnd)??.to_string(); if cursor.next_if(Punctuator::Colon).is_some() { let val = AssignmentExpression::new(true, self.allow_yield, self.allow_await) .parse(cursor)?; @@ -155,8 +150,8 @@ where let pos = cursor .peek(0) - .map(|tok| tok.span().start()) - .ok_or(ParseError::AbruptEnd)?; + .ok_or(ParseError::AbruptEnd)?? + .span().start(); Err(ParseError::general("expected property definition", pos)) } } @@ -199,15 +194,12 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let (methodkind, prop_name, params) = match self.identifier.as_str() { idn @ "get" | idn @ "set" => { - let prop_name = cursor - .next() - .map(Token::to_string) - .ok_or(ParseError::AbruptEnd)?; + let prop_name = cursor.next().ok_or(ParseError::AbruptEnd)??.to_string(); cursor.expect( TokenKind::Punctuator(Punctuator::OpenParen), "property method definition", )?; - let first_param = cursor.peek(0).expect("current token disappeared").clone(); + let first_param = cursor.peek(0).expect("current token disappeared")?.clone(); let params = FormalParameters::new(false, false).parse(cursor)?; cursor.expect(Punctuator::CloseParen, "method definition")?; if idn == "get" { diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index cd24b0a7a0a..3dd61810947 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -58,7 +58,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; - match tok.kind { + match tok?.kind() { TokenKind::Keyword(Keyword::Delete) => { Ok(node::UnaryOp::new(UnaryOp::Delete, self.parse(cursor)?).into()) } diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index 07609bc0e1f..bb5fa163b63 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -48,7 +48,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; - match tok.kind { + match tok?.kind() { TokenKind::Punctuator(Punctuator::Inc) => { cursor.next().expect("token disappeared"); return Ok(node::UnaryOp::new( @@ -72,7 +72,7 @@ where let lhs = LeftHandSideExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; if let Some(tok) = cursor.peek(0) { - match tok.kind { + match tok?.kind() { TokenKind::Punctuator(Punctuator::Inc) => { cursor.next().expect("token disappeared"); return Ok(node::UnaryOp::new(UnaryOp::IncrementPost, lhs).into()); diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index b6c251b47e4..73314071409 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -62,8 +62,8 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let mut params = Vec::new(); - if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind - == TokenKind::Punctuator(Punctuator::CloseParen) + if cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() + == &TokenKind::Punctuator(Punctuator::CloseParen) { return Ok(params.into_boxed_slice()); } @@ -78,8 +78,8 @@ where FormalParameter::new(self.allow_yield, self.allow_await).parse(cursor)? }); - if cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind - == TokenKind::Punctuator(Punctuator::CloseParen) + if cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() + == &TokenKind::Punctuator(Punctuator::CloseParen) { break; } @@ -88,7 +88,7 @@ where return Err(ParseError::unexpected( cursor .peek_prev() - .expect("current token disappeared") + .expect("current token disappeared")? .clone(), "rest parameter must be the last formal parameter", )); @@ -243,7 +243,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { if let Some(tk) = cursor.peek(0) { - if tk.kind == Punctuator::CloseBlock.into() { + if tk?.kind() == &Punctuator::CloseBlock.into() { return Ok(Vec::new().into()); } } diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 7cb972c06c4..bb3d3a89fd0 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -72,7 +72,7 @@ where let _timer = BoaProfiler::global().start_event("Block", "Parsing"); cursor.expect(Punctuator::OpenBlock, "block")?; if let Some(tk) = cursor.peek(0) { - if tk.kind == TokenKind::Punctuator(Punctuator::CloseBlock) { + if tk?.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) { cursor.next(); return Ok(node::Block::from(vec![])); } diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 74522e71489..462e5f06629 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -63,7 +63,7 @@ where let _timer = BoaProfiler::global().start_event("LexicalDeclaration", "Parsing"); let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; - match tok.kind { + match tok?.kind() { TokenKind::Keyword(Keyword::Const) => { BindingList::new(self.allow_in, self.allow_yield, self.allow_await, true) .parse(cursor) @@ -134,7 +134,7 @@ where } else { return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::Assign)], - cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)??.clone(), "const declaration", )); } @@ -153,7 +153,7 @@ where TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)??.clone(), "lexical declaration", )) } diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index 9934efb5b43..5e6aadcc434 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -60,7 +60,7 @@ where let _timer = BoaProfiler::global().start_event("Declaration", "Parsing"); let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; - match tok.kind { + match tok?.kind() { TokenKind::Keyword(Keyword::Function) => { HoistableDeclaration::new(self.allow_yield, self.allow_await, false).parse(cursor) } diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index c225e677140..4af10cc2573 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -69,7 +69,7 @@ where Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; let else_stm = match cursor.peek(0) { - Some(else_tok) if else_tok.kind == TokenKind::Keyword(Keyword::Else) => { + Some(else_tok) if else_tok?.kind() == &TokenKind::Keyword(Keyword::Else) => { cursor.next(); Some( Statement::new(self.allow_yield, self.allow_await, self.allow_return) diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 3c9e1f010c9..57fc7e73b5f 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -68,9 +68,9 @@ where let body = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)??; - if next_token.kind != TokenKind::Keyword(Keyword::While) { + if next_token.kind() != &TokenKind::Keyword(Keyword::While) { return Err(ParseError::expected( vec![TokenKind::Keyword(Keyword::While)], next_token.clone(), diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index 79a7671861a..67fa2c74190 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -72,7 +72,7 @@ where cursor.expect(Keyword::For, "for statement")?; cursor.expect(Punctuator::OpenParen, "for statement")?; - let init = match cursor.peek(0).ok_or(ParseError::AbruptEnd)?.kind { + let init = match cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() { TokenKind::Keyword(Keyword::Var) => Some( VariableDeclarationList::new(false, self.allow_yield, self.allow_await) .parse(cursor) diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 96686aceb64..9bd88049d38 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -102,9 +102,9 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Statement", "Parsing"); // TODO: add BreakableStatement and divide Whiles, fors and so on to another place. - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)??; - match tok.kind { + match tok.kind() { TokenKind::Keyword(Keyword::If) => { IfStatement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor) @@ -228,11 +228,11 @@ where loop { match cursor.peek(0) { - Some(token) if token.kind == TokenKind::Punctuator(Punctuator::CloseBlock) => { + Some(token) if token?.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => { if self.break_when_closingbrase { break; } else { - return Err(ParseError::unexpected(token.clone(), None)); + return Err(ParseError::unexpected(token?.clone(), None)); } } None => { @@ -301,7 +301,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("StatementListItem", "Parsing"); - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)??; match tok.kind { TokenKind::Keyword(Keyword::Function) @@ -405,9 +405,9 @@ where let _timer = BoaProfiler::global().start_event("BindingIdentifier", "Parsing"); // TODO: strict mode. - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.next().ok_or(ParseError::AbruptEnd)??; - match next_token.kind { + match next_token.kind() { TokenKind::Identifier(ref s) => Ok(s.clone()), TokenKind::Keyword(k @ Keyword::Yield) if !self.allow_yield.0 => Ok(k.as_str().into()), TokenKind::Keyword(k @ Keyword::Await) if !self.allow_await.0 => Ok(k.as_str().into()), diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index 41b1ec51d23..690051a6f39 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -54,7 +54,7 @@ where let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; if let Some(tok) = cursor.peek(0) { - if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) { + if tok?.kind() == &TokenKind::Punctuator(Punctuator::Semicolon) { let _ = cursor.next(); } } diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index 5111ccaf685..21eb2e9341a 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -63,10 +63,10 @@ where let try_clause = Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)??; - if next_token.kind != TokenKind::Keyword(Keyword::Catch) - && next_token.kind != TokenKind::Keyword(Keyword::Finally) + if next_token.kind() != &TokenKind::Keyword(Keyword::Catch) + && next_token.kind() != &TokenKind::Keyword(Keyword::Finally) { return Err(ParseError::expected( vec![ @@ -86,7 +86,7 @@ where let next_token = cursor.peek(0); let finally_block = match next_token { - Some(token) => match token.kind { + Some(token) => match token?.kind() { TokenKind::Keyword(Keyword::Finally) => Some( Finally::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor)?, diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index fb9df47a737..508c6205133 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -126,7 +126,7 @@ where TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - cursor.next().ok_or(ParseError::AbruptEnd)?.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)??.clone(), "lexical declaration", )) } From 910ba1a937f03943226e732f5bcc1264bb4bf84b Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 16:35:41 +0100 Subject: [PATCH 068/289] Continuing to adapt parser to new cursor returns --- boa/src/syntax/lexer/mod.rs | 6 ++-- boa/src/syntax/parser/cursor.rs | 11 +++---- .../expression/assignment/conditional.rs | 2 +- .../expression/assignment/exponentiation.rs | 32 +++++++++---------- .../parser/expression/assignment/mod.rs | 5 ++- .../expression/left_hand_side/arguments.rs | 2 +- .../parser/expression/left_hand_side/call.rs | 9 ++++-- .../expression/left_hand_side/member.rs | 2 +- .../parser/expression/left_hand_side/mod.rs | 4 +-- boa/src/syntax/parser/expression/mod.rs | 2 +- .../primary/array_initializer/mod.rs | 2 +- .../expression/primary/function_expression.rs | 2 +- .../syntax/parser/expression/primary/mod.rs | 2 +- .../primary/object_initializer/mod.rs | 7 ++-- boa/src/syntax/parser/expression/unary.rs | 4 +-- boa/src/syntax/parser/expression/update.rs | 2 +- boa/src/syntax/parser/function/mod.rs | 2 +- boa/src/syntax/parser/mod.rs | 12 +++---- boa/src/syntax/parser/statement/block/mod.rs | 2 +- .../syntax/parser/statement/break_stm/mod.rs | 2 +- .../parser/statement/continue_stm/mod.rs | 2 +- .../parser/statement/declaration/hoistable.rs | 2 +- .../parser/statement/declaration/lexical.rs | 2 +- .../parser/statement/declaration/mod.rs | 2 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 8 +++-- .../statement/iteration/do_while_statement.rs | 2 +- .../statement/iteration/for_statement.rs | 2 +- .../statement/iteration/while_statement.rs | 2 +- boa/src/syntax/parser/statement/mod.rs | 11 +++++-- .../syntax/parser/statement/return_stm/mod.rs | 2 +- boa/src/syntax/parser/statement/switch/mod.rs | 2 +- boa/src/syntax/parser/statement/throw/mod.rs | 2 +- .../syntax/parser/statement/try_stm/catch.rs | 2 +- .../parser/statement/try_stm/finally.rs | 2 +- .../syntax/parser/statement/try_stm/mod.rs | 2 +- boa/src/syntax/parser/statement/variable.rs | 2 +- boa/src/syntax/parser/tests.rs | 4 ++- 37 files changed, 84 insertions(+), 79 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index c0045daae58..1abe7427e52 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -90,9 +90,9 @@ impl Lexer { /// Creates a new lexer. #[inline] - pub fn new(reader: R) -> Self + pub fn new(reader: R) -> Self where - R: Read + R: Read, { Self { cursor: Cursor::new(reader), @@ -109,7 +109,7 @@ impl Lexer { // As per https://tc39.es/ecma262/#sec-ecmascript-language-lexical-grammar fn lex_slash_token(&mut self, start: Position) -> Result where - R: Read + R: Read, { if let Some(c) = self.cursor.peek() { match c { diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 5929476f154..51daae6ac55 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -2,8 +2,8 @@ use super::ParseError; use crate::syntax::ast::Punctuator; -use crate::syntax::lexer::{Token, TokenKind}; use crate::syntax::lexer::Lexer; +use crate::syntax::lexer::{Token, TokenKind}; use std::io::Read; @@ -15,19 +15,18 @@ pub(super) struct Cursor { /// The tokens being input. // tokens: &'a [Token], lexer: Lexer, - // The current position within the tokens. // pos: usize, } -impl Cursor -where - R: Read +impl Cursor +where + R: Read, { /// Creates a new cursor. pub(super) fn new(reader: R) -> Self { Self { - lexer: Lexer::new(reader) + lexer: Lexer::new(reader), } } diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index 14f12ae8da9..fed241d03a4 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -13,7 +13,7 @@ use crate::{ ast::{node::ConditionalOp, Node, Punctuator}, parser::{ expression::{AssignmentExpression, LogicalORExpression}, - AllowAwait, AllowIn, AllowYield, ParseResult, Cursor, TokenParser, + AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 412d66b6125..864c6fcaa8f 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -7,8 +7,8 @@ //! [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Arithmetic_Operators#Exponentiation //! [spec]: https://tc39.es/ecma262/#sec-exp-operator -use crate::syntax::lexer::TokenKind; use super::ParseError; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ @@ -18,7 +18,7 @@ use crate::{ }, parser::{ expression::{unary::UnaryExpression, update::UpdateExpression}, - AllowAwait, AllowYield, ParseResult, Cursor, TokenParser, + AllowAwait, AllowYield, Cursor, ParseResult, TokenParser, }, }, BoaProfiler, @@ -59,22 +59,20 @@ fn is_unary_expression(cursor: &mut Cursor) -> Result where R: Read, { - Ok( - if let Some(tok) = cursor.peek(0) { - match tok?.kind() { - TokenKind::Keyword(Keyword::Delete) - | TokenKind::Keyword(Keyword::Void) - | TokenKind::Keyword(Keyword::TypeOf) - | TokenKind::Punctuator(Punctuator::Add) - | TokenKind::Punctuator(Punctuator::Sub) - | TokenKind::Punctuator(Punctuator::Not) - | TokenKind::Punctuator(Punctuator::Neg) => true, - _ => false, - } - } else { - false + Ok(if let Some(tok) = cursor.peek(0) { + match tok?.kind() { + TokenKind::Keyword(Keyword::Delete) + | TokenKind::Keyword(Keyword::Void) + | TokenKind::Keyword(Keyword::TypeOf) + | TokenKind::Punctuator(Punctuator::Add) + | TokenKind::Punctuator(Punctuator::Sub) + | TokenKind::Punctuator(Punctuator::Not) + | TokenKind::Punctuator(Punctuator::Neg) => true, + _ => false, } - ) + } else { + false + }) } impl TokenParser for ExponentiationExpression diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index f1e40456bf0..097c6063ee6 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -19,7 +19,7 @@ use crate::{ node::{Assign, BinOp, Node}, Keyword, Punctuator, }, - parser::{AllowAwait, AllowIn, AllowYield, ParseError, ParseResult, Cursor, TokenParser}, + parser::{AllowAwait, AllowIn, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }, BoaProfiler, }; @@ -81,8 +81,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); // Arrow function - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; - match next_token?.kind() { + match cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() { // a=>{} TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield) diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index ea6d7664369..8490cbea45c 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -12,7 +12,7 @@ use crate::{ syntax::{ ast::{node::Spread, Node, Punctuator}, parser::{ - expression::AssignmentExpression, AllowAwait, AllowYield, ParseError, Cursor, + expression::AssignmentExpression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, }, }, diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 2bbfd61273e..235f7d757a9 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -19,7 +19,7 @@ use crate::{ Punctuator, }, parser::{ - expression::Expression, AllowAwait, AllowYield, ParseError, ParseResult, Cursor, + expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser, }, }, @@ -64,8 +64,11 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("CallExpression", "Parsing"); - let mut lhs = match cursor.peek(0) { - Some(tk) if tk?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { + + let tk = cursor.peek(0); + + let mut lhs = match tk { + Some(_) if tk.unwrap()?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; Node::from(Call::new(self.first_member_expr, args)) } diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 014bd187d50..f3fde540b9f 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -18,7 +18,7 @@ use crate::{ }, parser::{ expression::{primary::PrimaryExpression, Expression}, - AllowAwait, AllowYield, ParseError, ParseResult, Cursor, TokenParser, + AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 167fd7211f8..40c89772421 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -12,15 +12,15 @@ mod call; mod member; use self::{call::CallExpression, member::MemberExpression}; +use super::super::ParseError; use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Node, Punctuator}, - parser::{AllowAwait, AllowYield, ParseResult, Cursor, TokenParser}, + parser::{AllowAwait, AllowYield, Cursor, ParseResult, TokenParser}, }, BoaProfiler, }; -use super::super::ParseError; use std::io::Read; diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 18724b2c664..f814e88cffe 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -17,7 +17,7 @@ mod update; use self::assignment::ExponentiationExpression; pub(super) use self::{assignment::AssignmentExpression, primary::Initializer}; -use super::{AllowAwait, AllowIn, AllowYield, ParseResult, Cursor, TokenParser}; +use super::{AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser}; use crate::syntax::lexer::TokenKind; use crate::{ profiler::BoaProfiler, diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index cdbafa4e5dd..13c6fd7e86b 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -17,7 +17,7 @@ use crate::{ Const, Punctuator, }, parser::{ - expression::AssignmentExpression, AllowAwait, AllowYield, ParseError, Cursor, + expression::AssignmentExpression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, }, }, diff --git a/boa/src/syntax/parser/expression/primary/function_expression.rs b/boa/src/syntax/parser/expression/primary/function_expression.rs index 113c9500313..b239e8c295f 100644 --- a/boa/src/syntax/parser/expression/primary/function_expression.rs +++ b/boa/src/syntax/parser/expression/primary/function_expression.rs @@ -13,7 +13,7 @@ use crate::{ parser::{ function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - ParseError, Cursor, TokenParser, + Cursor, ParseError, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 38b2da36ecf..7f2128f8373 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -24,7 +24,7 @@ use crate::syntax::{ node::{Call, Identifier, New, Node}, Const, Keyword, Punctuator, }, - parser::{AllowAwait, AllowYield, ParseError, ParseResult, Cursor, TokenParser}, + parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }; pub(in crate::syntax::parser) use object_initializer::Initializer; diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 3daa572d3e0..0ccafddb0a8 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -19,7 +19,7 @@ use crate::{ parser::{ expression::AssignmentExpression, function::{FormalParameters, FunctionBody}, - AllowAwait, AllowIn, AllowYield, ParseError, ParseResult, Cursor, TokenParser, + AllowAwait, AllowIn, AllowYield, Cursor, ParseError, ParseResult, TokenParser, }, }, BoaProfiler, @@ -148,10 +148,7 @@ where .parse(cursor); } - let pos = cursor - .peek(0) - .ok_or(ParseError::AbruptEnd)?? - .span().start(); + let pos = cursor.peek(0).ok_or(ParseError::AbruptEnd)??.span().start(); Err(ParseError::general("expected property definition", pos)) } } diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 3dd61810947..055ae2927fd 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -15,8 +15,8 @@ use crate::syntax::{ Keyword, Punctuator, }, parser::{ - expression::update::UpdateExpression, AllowAwait, AllowYield, ParseError, ParseResult, - Cursor, TokenParser, + expression::update::UpdateExpression, AllowAwait, AllowYield, Cursor, ParseError, + ParseResult, TokenParser, }, }; diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index bb5fa163b63..7b25eb9bd3e 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -9,7 +9,7 @@ use super::left_hand_side::LeftHandSideExpression; use crate::syntax::lexer::TokenKind; use crate::syntax::{ ast::{node, op::UnaryOp, Node, Punctuator}, - parser::{AllowAwait, AllowYield, ParseError, ParseResult, Cursor, TokenParser}, + parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }; use std::io::Read; diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 73314071409..ff303ed72f0 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -19,7 +19,7 @@ use crate::syntax::{ parser::{ expression::Initializer, statement::{BindingIdentifier, StatementList}, - AllowAwait, AllowYield, ParseError, Cursor, TokenParser, + AllowAwait, AllowYield, Cursor, ParseError, TokenParser, }, }; diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 77fb150f47b..36501428518 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -1,7 +1,7 @@ //! Boa parser implementation. -pub mod error; mod cursor; +pub mod error; mod expression; mod function; mod statement; @@ -109,18 +109,18 @@ pub struct Parser { } impl Parser { - pub fn new(reader: R) -> Self + pub fn new(reader: R) -> Self where - R: Read + R: Read, { Self { - cursor: Cursor::new(reader) + cursor: Cursor::new(reader), } } - pub fn parse_all(&mut self) -> Result + pub fn parse_all(&mut self) -> Result where - R: Read + R: Read, { Script.parse(&mut self.cursor) } diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index bb3d3a89fd0..3da3313e40a 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -17,7 +17,7 @@ use crate::{ profiler::BoaProfiler, syntax::{ ast::{node, Punctuator}, - parser::{AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser}, + parser::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}, }, }; diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index 05a77eddc45..1230545eb41 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -16,7 +16,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Break, Keyword, Punctuator}, - parser::{AllowAwait, AllowYield, ParseError, Cursor, TokenParser}, + parser::{AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, }; diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index 55c7b7493e8..f1ec954db74 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -15,7 +15,7 @@ use crate::{ syntax::{ ast::{node::Continue, Keyword, Punctuator}, parser::{ - statement::LabelIdentifier, AllowAwait, AllowYield, ParseError, Cursor, TokenParser, + statement::LabelIdentifier, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/declaration/hoistable.rs b/boa/src/syntax/parser/statement/declaration/hoistable.rs index 0da58c61fdf..1f18c099762 100644 --- a/boa/src/syntax/parser/statement/declaration/hoistable.rs +++ b/boa/src/syntax/parser/statement/declaration/hoistable.rs @@ -10,7 +10,7 @@ use crate::{ ast::{node::FunctionDecl, Keyword, Node, Punctuator}, parser::{ function::FormalParameters, function::FunctionBody, statement::BindingIdentifier, - AllowAwait, AllowDefault, AllowYield, ParseError, ParseResult, Cursor, TokenParser, + AllowAwait, AllowDefault, AllowYield, Cursor, ParseError, ParseResult, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 462e5f06629..f08d91d6755 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -16,7 +16,7 @@ use crate::{ }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, - ParseError, ParseResult, Cursor, TokenParser, + Cursor, ParseError, ParseResult, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index 5e6aadcc434..89d68decc6c 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -18,7 +18,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Keyword, Node}, - parser::{AllowAwait, AllowYield, ParseError, Cursor, TokenParser}, + parser::{AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, }; diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index 4af10cc2573..8288003ec7a 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -8,7 +8,7 @@ use crate::{ syntax::{ ast::{node::If, Keyword, Node, Punctuator}, parser::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, + expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, }, }, @@ -68,8 +68,10 @@ where let then_stm = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let else_stm = match cursor.peek(0) { - Some(else_tok) if else_tok?.kind() == &TokenKind::Keyword(Keyword::Else) => { + let else_tok = cursor.peek(0); + + let else_stm = match else_tok { + Some(_) if else_tok.unwrap()?.kind() == &TokenKind::Keyword(Keyword::Else) => { cursor.next(); Some( Statement::new(self.allow_yield, self.allow_await, self.allow_return) diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 57fc7e73b5f..1aab2091a1b 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -13,7 +13,7 @@ use crate::{ ast::{node::DoWhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, - ParseError, Cursor, TokenParser, + Cursor, ParseError, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index 67fa2c74190..063a03499c0 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -18,7 +18,7 @@ use crate::{ expression::Expression, statement::declaration::Declaration, statement::{variable::VariableDeclarationList, Statement}, - AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser, + AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index 6cf7317cc28..71239c0cfc1 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -3,7 +3,7 @@ use crate::{ ast::{node::WhileLoop, Keyword, Punctuator}, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, - ParseError, Cursor, TokenParser, + Cursor, ParseError, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 9bd88049d38..976a4d227af 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -33,7 +33,7 @@ use self::{ variable::VariableStatement, }; use super::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, ParseResult, Cursor, + expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, ParseResult, TokenParser, }; @@ -228,13 +228,18 @@ where loop { match cursor.peek(0) { - Some(token) if token?.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => { + Some(Ok(token)) + if token.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => + { if self.break_when_closingbrase { break; } else { - return Err(ParseError::unexpected(token?.clone(), None)); + return Err(ParseError::unexpected(token.clone(), None)); } } + Some(Err(e)) => { + return Err(e); + } None => { if self.break_when_closingbrase { return Err(ParseError::AbruptEnd); diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index 77377d4dd4d..a17b17ed375 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -5,7 +5,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Return, Keyword, Node, Punctuator}, - parser::{expression::Expression, AllowAwait, AllowYield, ParseError, Cursor, TokenParser}, + parser::{expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, }; diff --git a/boa/src/syntax/parser/statement/switch/mod.rs b/boa/src/syntax/parser/statement/switch/mod.rs index 7635ed097f0..fad9d21d70c 100644 --- a/boa/src/syntax/parser/statement/switch/mod.rs +++ b/boa/src/syntax/parser/statement/switch/mod.rs @@ -8,7 +8,7 @@ use crate::{ Keyword, Node, Punctuator, }, parser::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, + expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, }, }, diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index 690051a6f39..2cd03464bbb 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -5,7 +5,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Throw, Keyword, Punctuator}, - parser::{expression::Expression, AllowAwait, AllowYield, ParseError, Cursor, TokenParser}, + parser::{expression::Expression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, }; diff --git a/boa/src/syntax/parser/statement/try_stm/catch.rs b/boa/src/syntax/parser/statement/try_stm/catch.rs index 8f25d0f477c..701a1b52a98 100644 --- a/boa/src/syntax/parser/statement/try_stm/catch.rs +++ b/boa/src/syntax/parser/statement/try_stm/catch.rs @@ -6,7 +6,7 @@ use crate::{ }, parser::{ statement::{block::Block, BindingIdentifier}, - AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser, + AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/statement/try_stm/finally.rs b/boa/src/syntax/parser/statement/try_stm/finally.rs index 65c8e14db80..125d57df9dd 100644 --- a/boa/src/syntax/parser/statement/try_stm/finally.rs +++ b/boa/src/syntax/parser/statement/try_stm/finally.rs @@ -2,7 +2,7 @@ use crate::{ syntax::{ ast::{node, Keyword}, parser::{ - statement::block::Block, AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, + statement::block::Block, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, }, }, diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index 21eb2e9341a..31b6d1c7570 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -11,7 +11,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Try, Keyword}, - parser::{AllowAwait, AllowReturn, AllowYield, ParseError, Cursor, TokenParser}, + parser::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}, }, BoaProfiler, }; diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 508c6205133..5ecb1f56642 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -1,5 +1,6 @@ // use super::lexical_declaration_continuation; use crate::syntax::lexer::TokenKind; +use crate::syntax::parser::Cursor; use crate::{ syntax::{ ast::{ @@ -14,7 +15,6 @@ use crate::{ BoaProfiler, }; use std::io::Read; -use crate::syntax::parser::Cursor; /// Variable statement parsing. /// diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index c5cf4707225..5ae51aa7eb9 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -21,7 +21,9 @@ where L: Into>, { assert_eq!( - Parser::new(js.as_bytes()).parse_all().expect("failed to parse"), + Parser::new(js.as_bytes()) + .parse_all() + .expect("failed to parse"), StatementList::from(expr) ); } From 61b1c68872ac94271462fbb04e6164e075a82779 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 16:44:20 +0100 Subject: [PATCH 069/289] Fix multiple mutable cursor reference --- .../syntax/parser/expression/left_hand_side/member.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index f3fde540b9f..b05cac83149 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -73,11 +73,12 @@ where PrimaryExpression::new(self.allow_yield, self.allow_await).parse(cursor)? }; while let Some(tok) = cursor.peek(0) { - let token = tok?; + let token = tok?.clone(); match token.kind() { &TokenKind::Punctuator(Punctuator::Dot) => { - let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. - match &cursor.next().ok_or(ParseError::AbruptEnd)??.kind() { + cursor.next().ok_or(ParseError::AbruptEnd)??; // We move the parser forward. + + match cursor.next().ok_or(ParseError::AbruptEnd)??.kind() { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into() } @@ -87,7 +88,7 @@ where _ => { return Err(ParseError::expected( vec![TokenKind::identifier("identifier")], - token.clone(), + token, "member expression", )); } From 354cf1a56f1ca1820f47b7857e3ae7afdb606cb2 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 16:52:37 +0100 Subject: [PATCH 070/289] Updated benchmarks lexer/parser usage --- boa/benches/exec.rs | 100 ++++++------------ boa/benches/parser.rs | 34 +----- .../parser/expression/assignment/mod.rs | 24 ++--- .../parser/expression/left_hand_side/call.rs | 7 +- 4 files changed, 50 insertions(+), 115 deletions(-) diff --git a/boa/benches/exec.rs b/boa/benches/exec.rs index 74d97ff6148..040e1ebdac7 100644 --- a/boa/benches/exec.rs +++ b/boa/benches/exec.rs @@ -25,14 +25,10 @@ fn symbol_creation(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(SYMBOL_CREATION.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(SYMBOL_CREATION.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Symbols (Execution)", move |b| { @@ -58,14 +54,10 @@ fn for_loop_execution(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(FOR_LOOP.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(FOR_LOOP.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("For loop (Execution)", move |b| { @@ -91,14 +83,10 @@ fn fibonacci(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(FIBONACCI.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(FIBONACCI.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Fibonacci (Execution)", move |b| { @@ -122,14 +110,10 @@ fn object_creation(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(OBJECT_CREATION.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(OBJECT_CREATION.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Object Creation (Execution)", move |b| { @@ -153,14 +137,10 @@ fn object_prop_access_const(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(OBJECT_PROP_ACCESS_CONST.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(OBJECT_PROP_ACCESS_CONST.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Static Object Property Access (Execution)", move |b| { @@ -184,14 +164,10 @@ fn object_prop_access_dyn(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(OBJECT_PROP_ACCESS_DYN.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(OBJECT_PROP_ACCESS_DYN.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("Dynamic Object Property Access (Execution)", move |b| { @@ -212,14 +188,10 @@ fn regexp_literal_creation(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(REGEXP_LITERAL_CREATION.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(REGEXP_LITERAL_CREATION.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("RegExp Literal Creation (Execution)", move |b| { @@ -240,14 +212,10 @@ fn regexp_creation(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(REGEXP_CREATION.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(REGEXP_CREATION.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("RegExp (Execution)", move |b| { @@ -268,14 +236,10 @@ fn regexp_literal(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(REGEXP_LITERAL.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(REGEXP_LITERAL.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("RegExp Literal (Execution)", move |b| { @@ -296,14 +260,10 @@ fn regexp(c: &mut Criterion) { let realm = Realm::create(); let mut engine = Interpreter::new(realm); - // Lex all the tokens. - let lexer = Lexer::new(black_box(REGEXP.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - // Parse the AST nodes. - let nodes = Parser::new(&black_box(tokens)).parse_all().unwrap(); + let nodes = Parser::new(black_box(REGEXP.as_bytes())) + .parse_all() + .unwrap(); // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler c.bench_function("RegExp (Execution)", move |b| { diff --git a/boa/benches/parser.rs b/boa/benches/parser.rs index 057b9a135e9..49111673b06 100644 --- a/boa/benches/parser.rs +++ b/boa/benches/parser.rs @@ -18,14 +18,7 @@ fn expression_parser(c: &mut Criterion) { // We include the lexing in the benchmarks, since they will get together soon, anyways. c.bench_function("Expression (Parser)", move |b| { - b.iter(|| { - let lexer = Lexer::new(black_box(EXPRESSION.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - - Parser::new(&black_box(tokens)).parse_all() - }) + b.iter(|| Parser::new(black_box(EXPRESSION.as_bytes())).parse_all()) }); } @@ -35,13 +28,7 @@ fn hello_world_parser(c: &mut Criterion) { // We include the lexing in the benchmarks, since they will get together soon, anyways. c.bench_function("Hello World (Parser)", move |b| { - b.iter(|| { - let lexer = Lexer::new(black_box(HELLO_WORLD.as_bytes())); - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - - Parser::new(&black_box(tokens)).parse_all() - }) + b.iter(|| Parser::new(black_box(HELLO_WORLD.as_bytes())).parse_all()) }); } @@ -61,14 +48,7 @@ fn for_loop_parser(c: &mut Criterion) { // We include the lexing in the benchmarks, since they will get together soon, anyways. c.bench_function("For loop (Parser)", move |b| { - b.iter(|| { - let lexer = Lexer::new(black_box(FOR_LOOP.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - - Parser::new(&black_box(tokens)).parse_all() - }) + b.iter(|| Parser::new(black_box(FOR_LOOP.as_bytes())).parse_all()) }); } @@ -105,13 +85,7 @@ fn long_file_parser(c: &mut Criterion) { b.iter(|| { let file_str = fs::read_to_string(FILE_NAME) .unwrap_or_else(|_| panic!("could not read {}", FILE_NAME)); - - let lexer = Lexer::new(black_box(file_str.as_bytes())); - - // Goes through and lexes entire given string. - let tokens = lexer.collect::, _>>().expect("failed to lex"); - - Parser::new(&black_box(tokens)).parse_all() + Parser::new(black_box(file_str.as_bytes())).parse_all() }) }); diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 097c6063ee6..a799ba9329c 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -85,18 +85,18 @@ where // a=>{} TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield) - | TokenKind::Keyword(Keyword::Await) - if cursor.peek_expect_no_lineterminator(1).is_ok() => - { - if let Some(tok) = cursor.peek(1) { - if tok?.kind() == &TokenKind::Punctuator(Punctuator::Arrow) { - return ArrowFunction::new( - self.allow_in, - self.allow_yield, - self.allow_await, - ) - .parse(cursor) - .map(Node::ArrowFunctionDecl); + | TokenKind::Keyword(Keyword::Await) => { + if cursor.peek_expect_no_lineterminator(1).is_ok() { + if let Some(tok) = cursor.peek(1) { + if tok?.kind() == &TokenKind::Punctuator(Punctuator::Arrow) { + return ArrowFunction::new( + self.allow_in, + self.allow_yield, + self.allow_await, + ) + .parse(cursor) + .map(Node::ArrowFunctionDecl); + } } } } diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 235f7d757a9..35f1a0197d3 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -83,14 +83,15 @@ where }; while let Some(tok) = cursor.peek(0) { - let token = tok?; + let token = tok?.clone(); match token.kind() { TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; lhs = Node::from(Call::new(lhs, args)); } TokenKind::Punctuator(Punctuator::Dot) => { - let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. + cursor.next().ok_or(ParseError::AbruptEnd)??; // We move the parser forward. + match &cursor.next().ok_or(ParseError::AbruptEnd)??.kind() { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into(); @@ -101,7 +102,7 @@ where _ => { return Err(ParseError::expected( vec![TokenKind::identifier("identifier")], - token.clone(), + token, "call expression", )); } From 3dda03304095bb576ba60fa549057fe721fa7b47 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 17:03:50 +0100 Subject: [PATCH 071/289] Updated cli/wasm usage of parser --- boa_cli/src/main.rs | 6 +++--- boa_wasm/src/lib.rs | 9 +-------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/boa_cli/src/main.rs b/boa_cli/src/main.rs index f2445372a5a..f92775e2d02 100644 --- a/boa_cli/src/main.rs +++ b/boa_cli/src/main.rs @@ -124,10 +124,10 @@ fn lex_source(src: &str) -> Result, String> { /// /// Returns a error of type String with a message, /// if the token stream has a parsing error. -fn parse_tokens(tokens: Vec) -> Result { +fn parse_tokens(src: &str) -> Result { use boa::syntax::parser::Parser; - Parser::new(&tokens) + Parser::new(src.as_bytes()) .parse_all() .map_err(|e| format!("ParsingError: {}", e)) } @@ -152,7 +152,7 @@ fn dump(src: &str, args: &Opt) -> Result<(), String> { None => println!("{:#?}", tokens), } } else if let Some(ref arg) = args.dump_ast { - let ast = parse_tokens(tokens)?; + let ast = parse_tokens(src)?; match arg { Some(format) => match format { diff --git a/boa_wasm/src/lib.rs b/boa_wasm/src/lib.rs index fa5b5a09d5c..9e1abe2e955 100644 --- a/boa_wasm/src/lib.rs +++ b/boa_wasm/src/lib.rs @@ -3,14 +3,7 @@ use wasm_bindgen::prelude::*; #[wasm_bindgen] pub fn evaluate(src: &str) -> Result { - let lexer = Lexer::new(src.as_bytes()); - - // Goes through and lexes entire given string. - let tokens = lexer - .collect::, _>>() - .map_err(|e| format!("Lexing Error: {}", e))?; - - let expr = Parser::new(&tokens) + let expr = Parser::new(src.as_bytes()) .parse_all() .map_err(|e| JsValue::from(format!("Parsing Error: {}", e)))?; From 9715baef823ba9916ee245a23dc49509097d1bc5 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 17:09:56 +0100 Subject: [PATCH 072/289] Back to a compiling state --- boa/benches/exec.rs | 2 +- boa/benches/parser.rs | 2 +- boa/src/syntax/lexer/cursor.rs | 14 -------------- boa/src/syntax/lexer/number.rs | 1 - boa/src/syntax/lexer/regex.rs | 2 +- boa/src/syntax/parser/cursor.rs | 15 --------------- .../parser/expression/left_hand_side/mod.rs | 2 +- .../expression/primary/object_initializer/mod.rs | 2 +- boa/src/syntax/parser/expression/update.rs | 8 ++++---- boa/src/syntax/parser/mod.rs | 4 ---- boa/src/syntax/parser/statement/variable.rs | 2 +- boa/src/syntax/parser/tests.rs | 15 ++++++--------- boa_wasm/src/lib.rs | 2 +- 13 files changed, 17 insertions(+), 54 deletions(-) diff --git a/boa/benches/exec.rs b/boa/benches/exec.rs index 040e1ebdac7..de4bafd26a0 100644 --- a/boa/benches/exec.rs +++ b/boa/benches/exec.rs @@ -1,6 +1,6 @@ //! Benchmarks of the whole execution engine in Boa. -use boa::{exec::Interpreter, realm::Realm, Executable, Lexer, Parser}; +use boa::{exec::Interpreter, realm::Realm, Executable, Parser}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; #[cfg(all(target_arch = "x86_64", target_os = "linux", target_env = "gnu"))] diff --git a/boa/benches/parser.rs b/boa/benches/parser.rs index 49111673b06..d4d9c535029 100644 --- a/boa/benches/parser.rs +++ b/boa/benches/parser.rs @@ -1,6 +1,6 @@ //! Benchmarks of the parsing process in Boa. -use boa::syntax::{lexer::Lexer, parser::Parser}; +use boa::syntax::parser::Parser; use criterion::{black_box, criterion_group, criterion_main, Criterion}; #[cfg(all(target_arch = "x86_64", target_os = "linux", target_env = "gnu"))] diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 439dad72eb8..c2ed005324b 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -143,22 +143,10 @@ where } } - /// Retrieves the given number of characters and adds them to the buffer. - pub(super) fn take(&mut self, count: usize, buf: &mut String) -> io::Result<()> { - unimplemented!() - } - /// It will fill the buffer with checked ASCII bytes. pub(super) fn fill_bytes(&mut self, buf: &mut [u8]) -> io::Result<()> { unimplemented!() } - - /// Retrieves the next character as an ASCII character. - /// - /// It will make sure that the next character is an ASCII byte, or return an error otherwise. - pub(super) fn next_as_byte(&mut self) -> Option> { - unimplemented!() - } } impl Iterator for Cursor @@ -205,8 +193,6 @@ where type Item = io::Result; fn next(&mut self) -> Option { - use std::convert::TryFrom; - let first_byte = match self.iter.next()? { Ok(b) => b, Err(e) => return Some(Err(e)), diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index f773d294a1d..fd87683b84b 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -2,7 +2,6 @@ use super::{Cursor, Error, TokenKind, Tokenizer}; use crate::builtins::BigInt; use crate::syntax::ast::{Position, Span}; use crate::syntax::lexer::{token::Numeric, Token}; -use std::convert::TryFrom; use std::io::Read; use std::str::FromStr; diff --git a/boa/src/syntax/lexer/regex.rs b/boa/src/syntax/lexer/regex.rs index be503ee481d..0bf0e2f52f1 100644 --- a/boa/src/syntax/lexer/regex.rs +++ b/boa/src/syntax/lexer/regex.rs @@ -78,7 +78,7 @@ impl Tokenizer for RegexLiteral { // body was parsed, now look for flags let mut flags = String::new(); - cursor.take_until_pred(&mut flags, &char::is_alphabetic); + cursor.take_until_pred(&mut flags, &char::is_alphabetic)?; Ok(Token::new( TokenKind::regular_expression_literal(body, flags.parse()?), diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 51daae6ac55..e2b4be183dd 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -1,7 +1,6 @@ //! Cursor implementation for the parser. use super::ParseError; -use crate::syntax::ast::Punctuator; use crate::syntax::lexer::Lexer; use crate::syntax::lexer::{Token, TokenKind}; @@ -30,20 +29,6 @@ where } } - /// Retrieves the current position of the cursor in the token stream. - pub(super) fn pos(&self) -> usize { - unimplemented!(); - // self.pos - } - - /// Moves the cursor to the given position. - /// - /// This is intended to be used *always* with `Cursor::pos()`. - pub(super) fn seek(&mut self, pos: usize) { - unimplemented!(); - // self.pos = pos - } - /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { unimplemented!(); diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 40c89772421..e8c1992bc7f 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -17,7 +17,7 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{Node, Punctuator}, - parser::{AllowAwait, AllowYield, Cursor, ParseResult, TokenParser}, + parser::{AllowAwait, AllowYield, Cursor, TokenParser}, }, BoaProfiler, }; diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 0ccafddb0a8..a7eb9c029f1 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -9,7 +9,7 @@ #[cfg(test)] mod tests; -use crate::syntax::lexer::{Token, TokenKind}; +use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{ diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index 7b25eb9bd3e..22adb9c8c5b 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -50,7 +50,7 @@ where let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; match tok?.kind() { TokenKind::Punctuator(Punctuator::Inc) => { - cursor.next().expect("token disappeared"); + cursor.next().expect("token disappeared")?; return Ok(node::UnaryOp::new( UnaryOp::IncrementPre, LeftHandSideExpression::new(self.allow_yield, self.allow_await) @@ -59,7 +59,7 @@ where .into()); } TokenKind::Punctuator(Punctuator::Dec) => { - cursor.next().expect("token disappeared"); + cursor.next().expect("token disappeared")?; return Ok(node::UnaryOp::new( UnaryOp::DecrementPre, LeftHandSideExpression::new(self.allow_yield, self.allow_await) @@ -74,11 +74,11 @@ where if let Some(tok) = cursor.peek(0) { match tok?.kind() { TokenKind::Punctuator(Punctuator::Inc) => { - cursor.next().expect("token disappeared"); + cursor.next().expect("token disappeared")?; return Ok(node::UnaryOp::new(UnaryOp::IncrementPost, lhs).into()); } TokenKind::Punctuator(Punctuator::Dec) => { - cursor.next().expect("token disappeared"); + cursor.next().expect("token disappeared")?; return Ok(node::UnaryOp::new(UnaryOp::DecrementPost, lhs).into()); } _ => {} diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 36501428518..6aac550d18d 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -11,12 +11,8 @@ mod tests; use self::error::{ParseError, ParseResult}; use crate::syntax::ast::node::StatementList; use crate::syntax::ast::Node; -use crate::syntax::lexer::InputElement; -use crate::syntax::lexer::Lexer; use crate::syntax::lexer::Token; -use ParseError as Error; - use cursor::Cursor; use std::io::Read; diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 5ecb1f56642..9f333b851f8 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -9,7 +9,7 @@ use crate::{ }, parser::{ expression::Initializer, statement::BindingIdentifier, AllowAwait, AllowIn, AllowYield, - ParseError, Parser, TokenParser, + ParseError, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 5ae51aa7eb9..8de593d9458 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -1,16 +1,13 @@ //! Tests for the parser. use super::Parser; -use crate::syntax::{ - ast::{ - node::{ - field::GetConstField, Assign, BinOp, Call, FunctionDecl, Identifier, New, Node, Return, - StatementList, UnaryOp, VarDecl, VarDeclList, - }, - op::{self, NumOp}, - Const, +use crate::syntax::ast::{ + node::{ + field::GetConstField, Assign, BinOp, Call, FunctionDecl, Identifier, New, Node, Return, + StatementList, UnaryOp, VarDecl, VarDeclList, }, - lexer::Lexer, + op::{self, NumOp}, + Const, }; /// Checks that the given JavaScript string gives the expected expression. diff --git a/boa_wasm/src/lib.rs b/boa_wasm/src/lib.rs index 9e1abe2e955..5bb0e5cfab7 100644 --- a/boa_wasm/src/lib.rs +++ b/boa_wasm/src/lib.rs @@ -1,4 +1,4 @@ -use boa::{Executable, Interpreter, Lexer, Parser, Realm}; +use boa::{Executable, Interpreter, Parser, Realm}; use wasm_bindgen::prelude::*; #[wasm_bindgen] From 68f41fac416e8c2373d5d1eaa6e4407fc34d5403 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 22 Jun 2020 23:01:20 +0100 Subject: [PATCH 073/289] Seperated cursor peek and peek(skip > 0) --- boa/src/syntax/lexer/error.rs | 1 - boa/src/syntax/lexer/number.rs | 2 +- boa/src/syntax/parser/cursor.rs | 115 +++++++++++++----- boa/src/syntax/parser/error.rs | 14 ++- .../expression/assignment/arrow_function.rs | 4 +- .../expression/assignment/exponentiation.rs | 2 +- .../parser/expression/assignment/mod.rs | 4 +- .../parser/expression/left_hand_side/call.rs | 4 +- .../expression/left_hand_side/member.rs | 4 +- .../parser/expression/left_hand_side/mod.rs | 2 +- boa/src/syntax/parser/expression/mod.rs | 2 +- .../primary/array_initializer/mod.rs | 2 +- .../primary/object_initializer/mod.rs | 4 +- boa/src/syntax/parser/expression/update.rs | 4 +- boa/src/syntax/parser/function/mod.rs | 6 +- boa/src/syntax/parser/mod.rs | 20 ++- boa/src/syntax/parser/statement/block/mod.rs | 2 +- .../parser/statement/declaration/mod.rs | 2 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 2 +- .../statement/iteration/do_while_statement.rs | 2 +- .../statement/iteration/for_statement.rs | 2 +- boa/src/syntax/parser/statement/mod.rs | 6 +- boa/src/syntax/parser/statement/throw/mod.rs | 2 +- .../syntax/parser/statement/try_stm/mod.rs | 4 +- 24 files changed, 137 insertions(+), 75 deletions(-) diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index a9688eebde4..3c1b65e2a3b 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -5,7 +5,6 @@ pub enum Error { IO(io::Error), Syntax(Box), StrictMode(Box), // Not 100% decided on this name. - // Reverted(String), } impl From for Error { diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index fd87683b84b..dad60a6e722 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -148,7 +148,7 @@ impl Tokenizer for NumberLiteral { )); } Some(Err(e)) => { - // todo!(); + todo!(); } Some(Ok('x')) | Some(Ok('X')) => { // Remove the initial '0' from buffer. diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index e2b4be183dd..b5772199d44 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -5,6 +5,7 @@ use crate::syntax::lexer::Lexer; use crate::syntax::lexer::{Token, TokenKind}; use std::io::Read; +use std::collections::VecDeque; /// Token cursor. /// @@ -16,6 +17,10 @@ pub(super) struct Cursor { lexer: Lexer, // The current position within the tokens. // pos: usize, + + peeked: Option>, + + // peeked: Option>, } impl Cursor @@ -26,49 +31,103 @@ where pub(super) fn new(reader: R) -> Self { Self { lexer: Lexer::new(reader), + peeked: None, } } /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { - unimplemented!(); - // loop { - // let token = self.tokens.get(self.pos); - // if let Some(tk) = token { - // self.pos += 1; + let peeked = self.peeked.as_ref(); + + match peeked { + Some(val) => { + match val { + Some(token) => return Some(Ok(token.clone())), + None => return None + } + } + None => {} // No value has been peeked ahead already so need to go get the next value. + } - // if tk.kind != TokenKind::LineTerminator { - // break Some(tk); - // } - // } else { - // break None; - // } - // } + self.peeked = None; // Consuming peeked value. + + loop { + match self.lexer.next() { + Some(Ok(tk)) => { + if tk.kind != TokenKind::LineTerminator { + return Some(Ok(tk)); + } + } + Some (Err(e)) => { + return Some(Err(ParseError::lex(e))); + } + None => { + return None; + } + } + } } /// Peeks the next token without moving the cursor. - pub(super) fn peek(&self, skip: usize) -> Option> { - unimplemented!(); - // let mut count = 0; - // let mut skipped = 0; - // loop { - // let token = self.tokens.get(self.pos + count); - // count += 1; + pub(super) fn peek(&mut self) -> Option> { - // if let Some(tk) = token { - // if tk.kind != TokenKind::LineTerminator { - // if skipped == skip { - // break Some(tk); - // } + // if skip > 0 { + // unimplemented!(); + // } - // skipped += 1; - // } - // } else { - // break None; + match self.peeked.as_ref() { + Some(Some(token)) => { + return Some(Ok(token.clone())); + } + Some(None) => { + return None; + } + None => { + // self.next(); + + } + } + + self.peeked = Some(match self.next() { + Some(Ok(token)) => { + Some(token) + } + Some(Err(e)) => { + return Some(Err(e)); + } + None => { + None + } + }); + + match self.peeked.as_ref() { + Some(Some(token)) => { + return Some(Ok(token.clone())); + } + Some(None) => { + return None; + } + None => { + // self.next(); + unimplemented!(); + } + } + + // match self.peeked.as_ref() { + // Some(Some(x)) => { + // Some(Ok(x.clone())) + // }, + // Some(None) => { + // None // } + // None => unreachable!("Value self.peeked assigned above but now gone") // } } + pub(super) fn peek_more(&mut self, skip: i32) -> Option> { + unimplemented!(); + } + /// Moves the cursor to the previous token and returns the token. pub(super) fn back(&mut self) -> Option> { unimplemented!(); diff --git a/boa/src/syntax/parser/error.rs b/boa/src/syntax/parser/error.rs index 27065b6600a..36922b8d83e 100644 --- a/boa/src/syntax/parser/error.rs +++ b/boa/src/syntax/parser/error.rs @@ -1,6 +1,6 @@ //! Error and result implementation for the parser. use crate::syntax::ast::{position::Position, Node}; -use crate::syntax::lexer::{Token, TokenKind}; +use crate::syntax::lexer::{Token, TokenKind, Error}; use std::fmt; /// Result of a parsing operation. @@ -17,7 +17,7 @@ impl ErrorContext for Result { } /// `ParseError` is an enum which represents errors encounted during parsing an expression -#[derive(Debug, Clone)] +#[derive(Debug)] pub enum ParseError { /// When it expected a certain kind of token, but got another as part of something Expected { @@ -32,6 +32,9 @@ pub enum ParseError { }, /// When there is an abrupt end to the parsing AbruptEnd, + Lex { + err: Error + }, /// Catch all General Error General { message: &'static str, @@ -76,6 +79,12 @@ impl ParseError { pub(super) fn general(message: &'static str, position: Position) -> Self { Self::General { message, position } } + + pub(super) fn lex(e: Error) -> Self { + Self::Lex { + err: e + } + } } impl fmt::Display for ParseError { @@ -140,6 +149,7 @@ impl fmt::Display for ParseError { position.line_number(), position.column_number() ), + Self::Lex {err} => write!(f, "Syntax Error: {}", err), } } } diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index bb7c81ec59b..947ce2e88ac 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -70,7 +70,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ArrowFunction", "Parsing"); - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)?; let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token?.kind() { // CoverParenthesizedExpressionAndArrowParameterList cursor.expect(Punctuator::OpenParen, "arrow function")?; @@ -119,7 +119,7 @@ where type Output = StatementList; fn parse(self, cursor: &mut Cursor) -> Result { - match cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() { + match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { TokenKind::Punctuator(Punctuator::OpenBlock) => { let _ = cursor.next(); let body = FunctionBody::new(false, false).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 864c6fcaa8f..8247c558a00 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -59,7 +59,7 @@ fn is_unary_expression(cursor: &mut Cursor) -> Result where R: Read, { - Ok(if let Some(tok) = cursor.peek(0) { + Ok(if let Some(tok) = cursor.peek() { match tok?.kind() { TokenKind::Keyword(Keyword::Delete) | TokenKind::Keyword(Keyword::Void) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index a799ba9329c..fa963de209a 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -81,13 +81,13 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); // Arrow function - match cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() { + match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { // a=>{} TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield) | TokenKind::Keyword(Keyword::Await) => { if cursor.peek_expect_no_lineterminator(1).is_ok() { - if let Some(tok) = cursor.peek(1) { + if let Some(tok) = cursor.peek_more(1) { if tok?.kind() == &TokenKind::Punctuator(Punctuator::Arrow) { return ArrowFunction::new( self.allow_in, diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 35f1a0197d3..58934ab9930 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -65,7 +65,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("CallExpression", "Parsing"); - let tk = cursor.peek(0); + let tk = cursor.peek(); let mut lhs = match tk { Some(_) if tk.unwrap()?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { @@ -82,7 +82,7 @@ where } }; - while let Some(tok) = cursor.peek(0) { + while let Some(tok) = cursor.peek() { let token = tok?.clone(); match token.kind() { TokenKind::Punctuator(Punctuator::OpenParen) => { diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index b05cac83149..23720f21851 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -60,7 +60,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("MemberExpression", "Parsing"); - let mut lhs = if cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() + let mut lhs = if cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() == &TokenKind::Keyword(Keyword::New) { let _ = cursor.next().expect("keyword disappeared"); @@ -72,7 +72,7 @@ where } else { PrimaryExpression::new(self.allow_yield, self.allow_await).parse(cursor)? }; - while let Some(tok) = cursor.peek(0) { + while let Some(tok) = cursor.peek() { let token = tok?.clone(); match token.kind() { &TokenKind::Punctuator(Punctuator::Dot) => { diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index e8c1992bc7f..7c4ef7be0bf 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -62,7 +62,7 @@ where let _timer = BoaProfiler::global().start_event("LeftHandSIdeExpression", "Parsing"); // TODO: Implement NewExpression: new MemberExpression let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - match cursor.peek(0) { + match cursor.peek() { Some(tok) => { if tok?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) { CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor) diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index f814e88cffe..2eb3003b070 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -62,7 +62,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; - while let Some(tok) = cursor.peek(0) { + while let Some(tok) = cursor.peek() { match tok?.kind() { &TokenKind::Punctuator(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index 13c6fd7e86b..6d70423b4a6 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -74,7 +74,7 @@ where break; } - let _ = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; // Check that there are more tokens to read. + let _ = cursor.peek().ok_or(ParseError::AbruptEnd)?; // Check that there are more tokens to read. if cursor.next_if(Punctuator::Spread).is_some() { let node = AssignmentExpression::new(true, self.allow_yield, self.allow_await) diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index a7eb9c029f1..855bb7c016e 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -148,7 +148,7 @@ where .parse(cursor); } - let pos = cursor.peek(0).ok_or(ParseError::AbruptEnd)??.span().start(); + let pos = cursor.peek().ok_or(ParseError::AbruptEnd)??.span().start(); Err(ParseError::general("expected property definition", pos)) } } @@ -196,7 +196,7 @@ where TokenKind::Punctuator(Punctuator::OpenParen), "property method definition", )?; - let first_param = cursor.peek(0).expect("current token disappeared")?.clone(); + let first_param = cursor.peek().expect("current token disappeared")?.clone(); let params = FormalParameters::new(false, false).parse(cursor)?; cursor.expect(Punctuator::CloseParen, "method definition")?; if idn == "get" { diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index 22adb9c8c5b..90bfd07629d 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -47,7 +47,7 @@ where type Output = Node; fn parse(self, cursor: &mut Cursor) -> ParseResult { - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek().ok_or(ParseError::AbruptEnd)?; match tok?.kind() { TokenKind::Punctuator(Punctuator::Inc) => { cursor.next().expect("token disappeared")?; @@ -71,7 +71,7 @@ where } let lhs = LeftHandSideExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.peek(0) { + if let Some(tok) = cursor.peek() { match tok?.kind() { TokenKind::Punctuator(Punctuator::Inc) => { cursor.next().expect("token disappeared")?; diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index ff303ed72f0..817f3fc2e05 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -62,7 +62,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let mut params = Vec::new(); - if cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() + if cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() == &TokenKind::Punctuator(Punctuator::CloseParen) { return Ok(params.into_boxed_slice()); @@ -78,7 +78,7 @@ where FormalParameter::new(self.allow_yield, self.allow_await).parse(cursor)? }); - if cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() + if cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() == &TokenKind::Punctuator(Punctuator::CloseParen) { break; @@ -242,7 +242,7 @@ where type Output = node::StatementList; fn parse(self, cursor: &mut Cursor) -> Result { - if let Some(tk) = cursor.peek(0) { + if let Some(tk) = cursor.peek() { if tk?.kind() == &Punctuator::CloseBlock.into() { return Ok(Vec::new().into()); } diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 6aac550d18d..f520a225d83 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -32,20 +32,14 @@ where /// This method needs to be provided by the implementor type. fn parse(self, cursor: &mut Cursor) -> Result; + /// Tries to parse the following tokens with this parser. fn try_parse(self, cursor: &mut Cursor) -> Option { - unimplemented!(); + if let Ok(node) = self.parse(cursor) { + Some(node) + } else { + None + } } - - // /// Tries to parse the following tokens with this parser. - // fn try_parse(self, parser: Parser) -> Option { - // let initial_pos = cursor.pos(); - // if let Ok(node) = self.parse(cursor) { - // Some(node) - // } else { - // cursor.seek(initial_pos); - // None - // } - // } } /// Boolean representing if the parser should allow a `yield` keyword. @@ -152,7 +146,7 @@ where type Output = StatementList; fn parse(self, cursor: &mut Cursor) -> Result { - if cursor.peek(0).is_some() { + if cursor.peek().is_some() { ScriptBody.parse(cursor) } else { Ok(StatementList::from(Vec::new())) diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 3da3313e40a..b33e68d98c6 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -71,7 +71,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Block", "Parsing"); cursor.expect(Punctuator::OpenBlock, "block")?; - if let Some(tk) = cursor.peek(0) { + if let Some(tk) = cursor.peek() { if tk?.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) { cursor.next(); return Ok(node::Block::from(vec![])); diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index 89d68decc6c..5e5a5465e5b 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -58,7 +58,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Declaration", "Parsing"); - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek().ok_or(ParseError::AbruptEnd)?; match tok?.kind() { TokenKind::Keyword(Keyword::Function) => { diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index 8288003ec7a..e00851fc918 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -68,7 +68,7 @@ where let then_stm = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let else_tok = cursor.peek(0); + let else_tok = cursor.peek(); let else_stm = match else_tok { Some(_) if else_tok.unwrap()?.kind() == &TokenKind::Keyword(Keyword::Else) => { diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 1aab2091a1b..2b3d63d05c7 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -68,7 +68,7 @@ where let body = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; if next_token.kind() != &TokenKind::Keyword(Keyword::While) { return Err(ParseError::expected( diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index 063a03499c0..75aee805256 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -72,7 +72,7 @@ where cursor.expect(Keyword::For, "for statement")?; cursor.expect(Punctuator::OpenParen, "for statement")?; - let init = match cursor.peek(0).ok_or(ParseError::AbruptEnd)??.kind() { + let init = match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { TokenKind::Keyword(Keyword::Var) => Some( VariableDeclarationList::new(false, self.allow_yield, self.allow_await) .parse(cursor) diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 976a4d227af..f4fcca04f9f 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -102,7 +102,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Statement", "Parsing"); // TODO: add BreakableStatement and divide Whiles, fors and so on to another place. - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)??; + let tok = cursor.peek().ok_or(ParseError::AbruptEnd)??; match tok.kind() { TokenKind::Keyword(Keyword::If) => { @@ -227,7 +227,7 @@ where let mut items = Vec::new(); loop { - match cursor.peek(0) { + match cursor.peek() { Some(Ok(token)) if token.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => { @@ -306,7 +306,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("StatementListItem", "Parsing"); - let tok = cursor.peek(0).ok_or(ParseError::AbruptEnd)??; + let tok = cursor.peek().ok_or(ParseError::AbruptEnd)??; match tok.kind { TokenKind::Keyword(Keyword::Function) diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index 2cd03464bbb..dd57ef0efb1 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -53,7 +53,7 @@ where cursor.peek_expect_no_lineterminator(0)?; let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.peek(0) { + if let Some(tok) = cursor.peek() { if tok?.kind() == &TokenKind::Punctuator(Punctuator::Semicolon) { let _ = cursor.next(); } diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index 31b6d1c7570..37ed66d9a88 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -63,7 +63,7 @@ where let try_clause = Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let next_token = cursor.peek(0).ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; if next_token.kind() != &TokenKind::Keyword(Keyword::Catch) && next_token.kind() != &TokenKind::Keyword(Keyword::Finally) @@ -84,7 +84,7 @@ where None }; - let next_token = cursor.peek(0); + let next_token = cursor.peek(); let finally_block = match next_token { Some(token) => match token?.kind() { TokenKind::Keyword(Keyword::Finally) => Some( From 067e3106d50372142caa806fb0b86517f37b8732 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 24 Jun 2020 10:23:44 +0100 Subject: [PATCH 074/289] VecDeque used to allow peeking 1 ahead --- boa/src/syntax/parser/cursor.rs | 116 +++++++++++++++----------------- 1 file changed, 53 insertions(+), 63 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index b5772199d44..a797245894f 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -18,7 +18,9 @@ pub(super) struct Cursor { // The current position within the tokens. // pos: usize, - peeked: Option>, + // peeked: Option>, + + peeked: VecDeque>, // peeked: Option>, } @@ -31,26 +33,22 @@ where pub(super) fn new(reader: R) -> Self { Self { lexer: Lexer::new(reader), - peeked: None, + peeked: VecDeque::new(), } } /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { - let peeked = self.peeked.as_ref(); - - match peeked { - Some(val) => { - match val { - Some(token) => return Some(Ok(token.clone())), - None => return None - } + match self.peeked.pop_front() { + Some(None) => { + return None; + } + Some(Some(token)) => { + return Some(Ok(token)); } None => {} // No value has been peeked ahead already so need to go get the next value. } - self.peeked = None; // Consuming peeked value. - loop { match self.lexer.next() { Some(Ok(tk)) => { @@ -70,62 +68,55 @@ where /// Peeks the next token without moving the cursor. pub(super) fn peek(&mut self) -> Option> { - - // if skip > 0 { - // unimplemented!(); - // } - - match self.peeked.as_ref() { - Some(Some(token)) => { - return Some(Ok(token.clone())); - } + match self.peeked.pop_front() { Some(None) => { + self.peeked.push_front(None); // Push the value back onto the peeked stack. return None; } - None => { - // self.next(); - + Some(Some(token)) => { + self.peeked.push_front(Some(token.clone())); // Push the value back onto the peeked stack. + return Some(Ok(token)); } + None => {} // No value has been peeked ahead already so need to go get the next value. } - self.peeked = Some(match self.next() { + match self.next() { Some(Ok(token)) => { - Some(token) + self.peeked.push_back(Some(token.clone())); + Some(Ok(token)) } Some(Err(e)) => { - return Some(Err(e)); + Some(Err(e)) } None => { + self.peeked.push_back(None); None } - }); + } + } - match self.peeked.as_ref() { - Some(Some(token)) => { - return Some(Ok(token.clone())); - } - Some(None) => { - return None; - } - None => { - // self.next(); - unimplemented!(); + pub(super) fn peek_more(&mut self, skip: usize) -> Option> { + if skip != 1 { + // I don't believe we ever need to skip more than a single token? + unimplemented!("Attempting to peek ahead more than a single token"); + } + + // Add elements to the peeked buffer upto the amount required to skip the given amount ahead. + while self.peeked.len() < skip + 1 { + match self.next() { + Some(Ok(token)) => self.peeked.push_back(Some(token.clone())), + Some(Err(e)) => return Some(Err(e)), + None => self.peeked.push_back(None), } } + + let temp = self.peeked.pop_front().unwrap(); + let ret = self.peeked.pop_front().unwrap(); - // match self.peeked.as_ref() { - // Some(Some(x)) => { - // Some(Ok(x.clone())) - // }, - // Some(None) => { - // None - // } - // None => unreachable!("Value self.peeked assigned above but now gone") - // } - } + self.peeked.push_front(ret.clone()); + self.peeked.push_front(temp); - pub(super) fn peek_more(&mut self, skip: i32) -> Option> { - unimplemented!(); + ret.map(|token| Ok(token)) } /// Moves the cursor to the previous token and returns the token. @@ -178,19 +169,18 @@ where where K: Into, { - unimplemented!(); - // let next_token = self.next().ok_or(ParseError::AbruptEnd)?; - // let kind = kind.into(); - - // if next_token.kind == kind { - // Ok(()) - // } else { - // Err(ParseError::expected( - // vec![kind], - // next_token.clone(), - // context, - // )) - // } + let next_token = self.next().ok_or(ParseError::AbruptEnd)?; + let kind = kind.into(); + + if next_token.kind == kind { + Ok(()) + } else { + Err(ParseError::expected( + vec![kind], + next_token.clone(), + context, + )) + } } /// It will peek for the next token, to see if it's a semicolon. From efb0ab3bb0ea1e5e4cf06ae71cc748a36da1e68f Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 24 Jun 2020 15:52:02 +0100 Subject: [PATCH 075/289] Removing back/prev from parser cursor --- boa/src/syntax/parser/cursor.rs | 291 ++++++++++-------- boa/src/syntax/parser/error.rs | 10 +- .../expression/assignment/conditional.rs | 5 +- .../expression/assignment/exponentiation.rs | 5 +- .../parser/expression/assignment/mod.rs | 9 +- .../expression/left_hand_side/arguments.rs | 12 +- boa/src/syntax/parser/expression/unary.rs | 14 +- boa/src/syntax/parser/function/mod.rs | 5 +- .../syntax/parser/statement/break_stm/mod.rs | 2 +- .../parser/statement/continue_stm/mod.rs | 2 +- .../parser/statement/declaration/lexical.rs | 2 +- .../syntax/parser/statement/return_stm/mod.rs | 2 +- boa/src/syntax/parser/statement/variable.rs | 2 +- 13 files changed, 202 insertions(+), 159 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index a797245894f..fb1d5312d8b 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -1,11 +1,15 @@ //! Cursor implementation for the parser. use super::ParseError; +use crate::syntax::ast::Punctuator; use crate::syntax::lexer::Lexer; use crate::syntax::lexer::{Token, TokenKind}; -use std::io::Read; use std::collections::VecDeque; +use std::io::Read; + +/// The maximum number of values stored by the cursor to allow back(). +const BACK_QUEUE_MAX_LEN: usize = 3; /// Token cursor. /// @@ -19,8 +23,9 @@ pub(super) struct Cursor { // pos: usize, // peeked: Option>, - peeked: VecDeque>, + // Values are added to this queue when they are retrieved (next) to allow moving backwards. + // back_queue: VecDeque>, // peeked: Option>, } @@ -34,6 +39,7 @@ where Self { lexer: Lexer::new(reader), peeked: VecDeque::new(), + // back_queue: VecDeque::new(), } } @@ -41,9 +47,21 @@ where pub(super) fn next(&mut self) -> Option> { match self.peeked.pop_front() { Some(None) => { + // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { + // self.back_queue.pop_front(); // Remove the value from the front of the queue. + // } + + // self.back_queue.push_back(None); + return None; } Some(Some(token)) => { + // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { + // self.back_queue.pop_front(); // Remove the value from the front of the queue. + // } + + // self.back_queue.push_back(Some(token.clone())); + return Some(Ok(token)); } None => {} // No value has been peeked ahead already so need to go get the next value. @@ -53,13 +71,25 @@ where match self.lexer.next() { Some(Ok(tk)) => { if tk.kind != TokenKind::LineTerminator { + // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { + // self.back_queue.pop_front(); // Remove the value from the front of the queue. + // } + + // self.back_queue.push_back(Some(tk.clone())); + return Some(Ok(tk)); } } - Some (Err(e)) => { + Some(Err(e)) => { return Some(Err(ParseError::lex(e))); } None => { + // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { + // self.back_queue.pop_front(); // Remove the value from the front of the queue. + // } + + // self.back_queue.push_back(None); + return None; } } @@ -85,9 +115,7 @@ where self.peeked.push_back(Some(token.clone())); Some(Ok(token)) } - Some(Err(e)) => { - Some(Err(e)) - } + Some(Err(e)) => Some(Err(e)), None => { self.peeked.push_back(None); None @@ -109,7 +137,7 @@ where None => self.peeked.push_back(None), } } - + let temp = self.peeked.pop_front().unwrap(); let ret = self.peeked.pop_front().unwrap(); @@ -119,48 +147,61 @@ where ret.map(|token| Ok(token)) } - /// Moves the cursor to the previous token and returns the token. - pub(super) fn back(&mut self) -> Option> { - unimplemented!(); - - // debug_assert!( - // self.pos > 0, - // "cannot go back in a cursor that is at the beginning of the list of tokens" - // ); - - // self.pos -= 1; - // while self - // .tokens - // .get(self.pos - 1) - // .expect("token disappeared") - // .kind - // == TokenKind::LineTerminator - // && self.pos > 0 - // { - // self.pos -= 1; - // } - } + // /// Moves the cursor to the previous token and returns the token. + // pub(super) fn back(&mut self) -> Option> { + // unimplemented!(); - /// Peeks the previous token without moving the cursor. - pub(super) fn peek_prev(&self) -> Option> { - unimplemented!(); - // if self.pos == 0 { - // None - // } else { - // let mut back = 1; - // let mut tok = self.tokens.get(self.pos - back).expect("token disappeared"); - // while self.pos >= back && tok.kind == TokenKind::LineTerminator { - // back += 1; - // tok = self.tokens.get(self.pos - back).expect("token disappeared"); - // } - - // if back == self.pos { - // None - // } else { - // Some(tok) - // } - // } - } + // // debug_assert!( + // // self.back_queue.len() > 0, + // // "cannot go back in a cursor that is at the beginning of the list of tokens" + // // ); + + // // let token = self.back_queue.pop_back().unwrap(); + + // // self.peeked.push_front(token.clone()); + + // // token.map(|t| Ok(t)) + + // // unimplemented!(); + + // // debug_assert!( + // // self.pos > 0, + // // "cannot go back in a cursor that is at the beginning of the list of tokens" + // // ); + + // // self.pos -= 1; + // // while self + // // .tokens + // // .get(self.pos - 1) + // // .expect("token disappeared") + // // .kind + // // == TokenKind::LineTerminator + // // && self.pos > 0 + // // { + // // self.pos -= 1; + // // } + // } + + // /// Peeks the previous token without moving the cursor. + // pub(super) fn peek_prev(&self) -> Option> { + // unimplemented!(); + // // if self.pos == 0 { + // // None + // // } else { + // // let mut back = 1; + // // let mut tok = self.tokens.get(self.pos - back).expect("token disappeared"); + // // while self.pos >= back && tok.kind == TokenKind::LineTerminator { + // // back += 1; + // // tok = self.tokens.get(self.pos - back).expect("token disappeared"); + // // } + + // // if back == self.pos { + // // None + // // } else { + // // Some(tok) + // // } + // // } + // } /// Returns an error if the next token is not of kind `kind`. /// @@ -169,11 +210,11 @@ where where K: Into, { - let next_token = self.next().ok_or(ParseError::AbruptEnd)?; + let next_token = self.next().ok_or(ParseError::AbruptEnd)??; let kind = kind.into(); - if next_token.kind == kind { - Ok(()) + if next_token.kind() == &kind { + Ok(next_token) } else { Err(ParseError::expected( vec![kind], @@ -188,35 +229,35 @@ where /// It will automatically insert a semicolon if needed, as specified in the [spec][spec]. /// /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion - pub(super) fn peek_semicolon(&self, do_while: bool) -> (bool, Option<&Token>) { - unimplemented!(); - // match self.tokens.get(self.pos) { - // Some(tk) => match tk.kind { - // TokenKind::Punctuator(Punctuator::Semicolon) => (true, Some(tk)), - // TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { - // (true, Some(tk)) - // } - // _ => { - // if do_while { - // debug_assert!( - // self.pos != 0, - // "cannot be finishing a do-while if we are at the beginning" - // ); - - // let tok = self - // .tokens - // .get(self.pos - 1) - // .expect("could not find previous token"); - // if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { - // return (true, Some(tk)); - // } - // } - - // (false, Some(tk)) - // } - // }, - // None => (true, None), - // } + pub(super) fn peek_semicolon( + &mut self, + do_while: bool, + ) -> Result<(bool, Option), ParseError> { + match self.peek() { + Some(Ok(tk)) => match tk.kind { + TokenKind::Punctuator(Punctuator::Semicolon) => Ok((true, Some(tk))), + TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { + Ok((true, Some(tk))) + } + _ => { + if do_while { + todo!(); + + // let tok = self + // .tokens + // .get(self.pos - 1) + // .expect("could not find previous token"); + // if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { + // return Ok((true, Some(tk))); + // } + } + + Ok((false, Some(tk))) + } + }, + Some(Err(e)) => Err(e), + None => Ok((true, None)), + } } /// It will check if the next token is a semicolon. @@ -228,49 +269,46 @@ where &mut self, do_while: bool, context: &'static str, - ) -> Result { - unimplemented!(); - - // match self.peek_semicolon(do_while) { - // (true, Some(tk)) => match tk.kind { - // TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { - // self.pos += 1; - // Ok(()) - // } - // _ => Ok(()), - // }, - // (true, None) => Ok(()), - // (false, Some(tk)) => Err(ParseError::expected( - // vec![TokenKind::Punctuator(Punctuator::Semicolon)], - // tk.clone(), - // context, - // )), - // (false, None) => unreachable!(), - // } + ) -> Result, ParseError> { + match self.peek_semicolon(do_while)? { + (true, Some(tk)) => match tk.kind() { + TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { + self.next(); // Consume the token. + Ok(Some(tk)) + } + _ => Ok(Some(tk)), + }, + (true, None) => Ok(None), + (false, Some(tk)) => Err(ParseError::expected( + vec![TokenKind::Punctuator(Punctuator::Semicolon)], + tk.clone(), + context, + )), + (false, None) => unreachable!(), + } } /// It will make sure that the next token is not a line terminator. /// /// It expects that the token stream does not end here. pub(super) fn peek_expect_no_lineterminator(&mut self, skip: usize) -> Result<(), ParseError> { - unimplemented!(); - // let mut count = 0; - // let mut skipped = 0; - // loop { - // let token = self.tokens.get(self.pos + count); - // count += 1; - // if let Some(tk) = token { - // if skipped == skip && tk.kind == TokenKind::LineTerminator { - // break Err(ParseError::unexpected(tk.clone(), None)); - // } else if skipped == skip && tk.kind != TokenKind::LineTerminator { - // break Ok(()); - // } else if tk.kind != TokenKind::LineTerminator { - // skipped += 1; - // } - // } else { - // break Err(ParseError::AbruptEnd); - // } - // } + let token = if skip == 0 { + self.peek() + } else { + self.peek_more(skip) + }; + + match token { + Some(Ok(t)) => { + if t.kind() == &TokenKind::LineTerminator { + Err(ParseError::unexpected(t, None)) + } else { + Ok(()) + } + } + Some(Err(e)) => Err(e), + None => Err(ParseError::AbruptEnd), + } } /// Advance the cursor to the next token and retrieve it, only if it's of `kind` type. @@ -281,13 +319,16 @@ where where K: Into, { - unimplemented!(); - // let next_token = self.peek(0)?; - - // if next_token.kind == kind.into() { - // self.next() - // } else { - // None - // } + match self.peek() { + Some(Ok(token)) => { + if token.kind() == &kind.into() { + self.next() + } else { + None + } + } + Some(Err(e)) => Some(Err(e)), + None => None, + } } } diff --git a/boa/src/syntax/parser/error.rs b/boa/src/syntax/parser/error.rs index 36922b8d83e..6965ee458a6 100644 --- a/boa/src/syntax/parser/error.rs +++ b/boa/src/syntax/parser/error.rs @@ -1,6 +1,6 @@ //! Error and result implementation for the parser. use crate::syntax::ast::{position::Position, Node}; -use crate::syntax::lexer::{Token, TokenKind, Error}; +use crate::syntax::lexer::{Error, Token, TokenKind}; use std::fmt; /// Result of a parsing operation. @@ -33,7 +33,7 @@ pub enum ParseError { /// When there is an abrupt end to the parsing AbruptEnd, Lex { - err: Error + err: Error, }, /// Catch all General Error General { @@ -81,9 +81,7 @@ impl ParseError { } pub(super) fn lex(e: Error) -> Self { - Self::Lex { - err: e - } + Self::Lex { err: e } } } @@ -149,7 +147,7 @@ impl fmt::Display for ParseError { position.line_number(), position.column_number() ), - Self::Lex {err} => write!(f, "Syntax Error: {}", err), + Self::Lex { err } => write!(f, "Syntax Error: {}", err), } } } diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index fed241d03a4..ee2e0d2b23d 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -68,8 +68,9 @@ where let lhs = LogicalORExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - if let Some(tok) = cursor.next() { + if let Some(tok) = cursor.peek() { if tok?.kind() == &TokenKind::Punctuator(Punctuator::Question) { + cursor.next(); // Consume the token. let then_clause = AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; @@ -79,8 +80,6 @@ where AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; return Ok(ConditionalOp::new(lhs, then_clause, else_clause).into()); - } else { - cursor.back(); } } diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 8247c558a00..f910d1e4dd5 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -88,11 +88,10 @@ where } let lhs = UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.next() { + if let Some(tok) = cursor.peek() { if let TokenKind::Punctuator(Punctuator::Exp) = tok?.kind() { + cursor.next(); // Consume the token. return Ok(BinOp::new(NumOp::Exp, lhs, self.parse(cursor)?).into()); - } else { - cursor.back(); } } Ok(lhs) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index fa963de209a..3a48797769e 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -116,19 +116,20 @@ where let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - if let Some(tok) = cursor.next() { + if let Some(tok) = cursor.peek() { match tok?.kind() { TokenKind::Punctuator(Punctuator::Assign) => { + cursor.next(); // Consume the token. lhs = Assign::new(lhs, self.parse(cursor)?).into(); } TokenKind::Punctuator(p) if p.as_binop().is_some() => { + cursor.next(); // Consume the token. + let expr = self.parse(cursor)?; let binop = p.as_binop().expect("binop disappeared"); lhs = BinOp::new(binop, lhs, expr).into(); } - _ => { - cursor.back(); - } + _ => {} } } diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index 8490cbea45c..ee7cdfadc13 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -60,10 +60,15 @@ where cursor.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); loop { - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; match next_token.kind() { - TokenKind::Punctuator(Punctuator::CloseParen) => break, + TokenKind::Punctuator(Punctuator::CloseParen) => { + cursor.next(); // Consume the token. + break; + } TokenKind::Punctuator(Punctuator::Comma) => { + cursor.next(); // Consume the token. + if args.is_empty() { return Err(ParseError::unexpected(next_token.clone(), None)); } @@ -74,6 +79,7 @@ where } _ => { if !args.is_empty() { + cursor.next(); // Consume the token. return Err(ParseError::expected( vec![ TokenKind::Punctuator(Punctuator::Comma), @@ -82,8 +88,6 @@ where next_token.clone(), "argument list", )); - } else { - cursor.back(); } } } diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 055ae2927fd..e65f27c6717 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -57,33 +57,37 @@ where type Output = Node; fn parse(self, cursor: &mut Cursor) -> ParseResult { - let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek().ok_or(ParseError::AbruptEnd)?; match tok?.kind() { TokenKind::Keyword(Keyword::Delete) => { + cursor.next(); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Delete, self.parse(cursor)?).into()) } TokenKind::Keyword(Keyword::Void) => { + cursor.next(); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Void, self.parse(cursor)?).into()) } TokenKind::Keyword(Keyword::TypeOf) => { + cursor.next(); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::TypeOf, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Add) => { + cursor.next(); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Plus, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Sub) => { + cursor.next(); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Minus, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Neg) => { + cursor.next(); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Tilde, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Not) => { + cursor.next(); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Not, self.parse(cursor)?).into()) } - _ => { - cursor.back(); - UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor) - } + _ => UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor), } } } diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 817f3fc2e05..887a6b331d0 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -86,10 +86,7 @@ where if rest_param { return Err(ParseError::unexpected( - cursor - .peek_prev() - .expect("current token disappeared")? - .clone(), + cursor.peek().expect("Peek token disappeared")?, "rest parameter must be the last formal parameter", )); } diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index 1230545eb41..81ff206e294 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -61,7 +61,7 @@ where let _timer = BoaProfiler::global().start_event("BreakStatement", "Parsing"); cursor.expect(Keyword::Break, "break statement")?; - let label = if let (true, tok) = cursor.peek_semicolon(false) { + let label = if let (true, tok) = cursor.peek_semicolon(false)? { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) => { let _ = cursor.next(); diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index f1ec954db74..63f62e22ce5 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -61,7 +61,7 @@ where let _timer = BoaProfiler::global().start_event("ContinueStatement", "Parsing"); cursor.expect(Keyword::Continue, "continue statement")?; - let label = if let (true, tok) = cursor.peek_semicolon(false) { + let label = if let (true, tok) = cursor.peek_semicolon(false)? { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) => { let _ = cursor.next(); diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index f08d91d6755..9ddd1d5f879 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -142,7 +142,7 @@ where let_decls.push(LetDecl::new(ident, init)); } - match cursor.peek_semicolon(false) { + match cursor.peek_semicolon(false)? { (true, _) => break, (false, Some(tk)) if tk.kind == TokenKind::Punctuator(Punctuator::Comma) => { let _ = cursor.next(); diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index a17b17ed375..71b40164db7 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -50,7 +50,7 @@ where let _timer = BoaProfiler::global().start_event("ReturnStatement", "Parsing"); cursor.expect(Keyword::Return, "return statement")?; - if let (true, tok) = cursor.peek_semicolon(false) { + if let (true, tok) = cursor.peek_semicolon(false)? { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 9f333b851f8..b3dab4d3ed3 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -115,7 +115,7 @@ where .parse(cursor)?, ); - match cursor.peek_semicolon(false) { + match cursor.peek_semicolon(false)? { (true, _) => break, (false, Some(tk)) if tk.kind == TokenKind::Punctuator(Punctuator::Comma) => { let _ = cursor.next(); From efc6bc14924b1d705e5a48c84fc11bfa81eba3cb Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 00:27:42 +0100 Subject: [PATCH 076/289] Fixed peek_more parser cursor which caused infinite loop --- boa/src/syntax/parser/cursor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index fb1d5312d8b..7462169b903 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -131,9 +131,9 @@ where // Add elements to the peeked buffer upto the amount required to skip the given amount ahead. while self.peeked.len() < skip + 1 { - match self.next() { + match self.lexer.next() { Some(Ok(token)) => self.peeked.push_back(Some(token.clone())), - Some(Err(e)) => return Some(Err(e)), + Some(Err(e)) => return Some(Err(ParseError::lex(e))), None => self.peeked.push_back(None), } } From 8dc3159261a971c198f37f993dbdb2209e8cc1ae Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 10:25:41 +0100 Subject: [PATCH 077/289] Modified parser cursor.expect behaviour to fit usage --- boa/src/syntax/parser/cursor.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 7462169b903..0fa8cc55dda 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -205,15 +205,16 @@ where /// Returns an error if the next token is not of kind `kind`. /// - /// Note: it will consume the next token. + /// Note: it will consume the next token only if the next token is the expected type. pub(super) fn expect(&mut self, kind: K, context: &'static str) -> Result where K: Into, { - let next_token = self.next().ok_or(ParseError::AbruptEnd)??; + let next_token = self.peek().ok_or(ParseError::AbruptEnd)??; let kind = kind.into(); if next_token.kind() == &kind { + self.next(); Ok(next_token) } else { Err(ParseError::expected( From 0f9b23b417ce8cb5038ff17bbe9f6b63179742b8 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 16:35:43 +0100 Subject: [PATCH 078/289] Traced down / fixed bug related to semicolon insertion --- boa/src/syntax/lexer/tests.rs | 14 +++++ boa/src/syntax/parser/cursor.rs | 58 +++++++++++-------- .../expression/left_hand_side/arguments.rs | 4 ++ .../parser/expression/left_hand_side/call.rs | 9 +++ boa/src/syntax/parser/expression/mod.rs | 1 + boa/src/syntax/parser/statement/mod.rs | 14 +++++ boa/src/syntax/parser/tests.rs | 42 ++++++++++++++ 7 files changed, 117 insertions(+), 25 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 9562d5ce9c7..b77615d29a6 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -23,6 +23,20 @@ where ); } +#[test] +fn check_no_semicolon_statement_lex() { + let s1 = r#"arr.pop() + return arr.len < 3; + "#; + let mut lexer = Lexer::new(s1.as_bytes()); + + for l in lexer { + println!("{:?}", l); + } + + assert!(false, "This is debug test and should be removed"); +} + #[test] fn check_single_line_comment() { let s1 = "var \n//This is a comment\ntrue"; diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 0fa8cc55dda..20cb915257d 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -70,15 +70,17 @@ where loop { match self.lexer.next() { Some(Ok(tk)) => { - if tk.kind != TokenKind::LineTerminator { - // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { - // self.back_queue.pop_front(); // Remove the value from the front of the queue. - // } + return Some(Ok(tk)); - // self.back_queue.push_back(Some(tk.clone())); + // if tk.kind != TokenKind::LineTerminator { + // // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { + // // self.back_queue.pop_front(); // Remove the value from the front of the queue. + // // } - return Some(Ok(tk)); - } + // // self.back_queue.push_back(Some(tk.clone())); + + // return Some(Ok(tk)); + // } } Some(Err(e)) => { return Some(Err(ParseError::lex(e))); @@ -235,25 +237,28 @@ where do_while: bool, ) -> Result<(bool, Option), ParseError> { match self.peek() { - Some(Ok(tk)) => match tk.kind { - TokenKind::Punctuator(Punctuator::Semicolon) => Ok((true, Some(tk))), - TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { - Ok((true, Some(tk))) - } - _ => { - if do_while { - todo!(); - - // let tok = self - // .tokens - // .get(self.pos - 1) - // .expect("could not find previous token"); - // if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { - // return Ok((true, Some(tk))); - // } + Some(Ok(tk)) => { + println!("Token: {:?}", tk); + match tk.kind() { + TokenKind::Punctuator(Punctuator::Semicolon) => Ok((true, Some(tk))), + TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { + Ok((true, Some(tk))) + } + _ => { + if do_while { + todo!(); + + // let tok = self + // .tokens + // .get(self.pos - 1) + // .expect("could not find previous token"); + // if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { + // return Ok((true, Some(tk))); + // } + } + + Ok((false, Some(tk))) } - - Ok((false, Some(tk))) } }, Some(Err(e)) => Err(e), @@ -271,6 +276,9 @@ where do_while: bool, context: &'static str, ) -> Result, ParseError> { + println!("Context: {}", context); + println!("Peek: {:?}", self.peek()); + match self.peek_semicolon(do_while)? { (true, Some(tk)) => match tk.kind() { TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index ee7cdfadc13..e44f504e81d 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -57,10 +57,14 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Arguments", "Parsing"); + println!("Parsing args"); + cursor.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); loop { let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; + println!("Next token: {:?}", next_token); + match next_token.kind() { TokenKind::Punctuator(Punctuator::CloseParen) => { cursor.next(); // Consume the token. diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 58934ab9930..31968a30ac9 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -67,12 +67,18 @@ where let tk = cursor.peek(); + println!("Parse Call Expression Token: {:?}", tk); + let mut lhs = match tk { Some(_) if tk.unwrap()?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { + println!("Open paran"); let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; + println!("After args: {:?}", cursor.peek()); + Node::from(Call::new(self.first_member_expr, args)) } _ => { + println!("Not open paran"); let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::OpenParen)], @@ -82,8 +88,11 @@ where } }; + println!("Parse Call Expression Finish Token: {:?}", cursor.peek()); + while let Some(tok) = cursor.peek() { let token = tok?.clone(); + println!("Call expression parsing... token: {:?}", token); match token.kind() { TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 2eb3003b070..0768e9276a4 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -62,6 +62,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; + println!("expression lhs: {:?}", lhs); while let Some(tok) = cursor.peek() { match tok?.kind() { &TokenKind::Punctuator(op) if $( op == $op )||* => { diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index f4fcca04f9f..71c451e2bc5 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -228,6 +228,13 @@ where loop { match cursor.peek() { + Some(Ok(token)) + if token.kind() == &TokenKind::LineTerminator => + { + // Skip line terminators. + cursor.next(); + continue; + } Some(Ok(token)) if token.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => { @@ -355,9 +362,16 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExpressionStatement", "Parsing"); + + println!("Express statement before node peek: {:?}", cursor.peek()); + // TODO: lookahead let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + println!("Expression: {:?}", expr); + + println!("Cursor peek value after node peek: {:?}", cursor.peek()); + cursor.expect_semicolon(false, "expression statement")?; Ok(expr) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 8de593d9458..ba1a88c9dfb 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -31,6 +31,48 @@ pub(super) fn check_invalid(js: &str) { assert!(Parser::new(js.as_bytes()).parse_all().is_err()); } +#[test] +fn check_single_statement() { + let init = r#" + arr.pop(); + "#; + let res = Parser::new(init.as_bytes()) + .parse_all() + .expect("failed to parse"); + + println!("Result {:?}", res); +} + +#[test] +fn check_no_semicolon_statement() { + let init = r#" + arr.pop(); + return arr.len(); + "#; + let res = Parser::new(init.as_bytes()) + .parse_all() + .expect("failed to parse"); + + println!("Result {:?}", res); +} + +#[test] +fn check_function_no_semicolon_statement() { + let init = r#" + var arr = [11, 23, 45]; + + function foo() { + arr.pop() + return arr.len < 3; + } + "#; + let res = Parser::new(init.as_bytes()) + .parse_all() + .expect("failed to parse"); + + println!("Result {:?}", res); +} + /// Should be parsed as `new Class().method()` instead of `new (Class().method())` #[test] fn check_construct_call_precedence() { From da68b137d635d3c8a29d0af331534824113abd36 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 18:08:43 +0100 Subject: [PATCH 079/289] Added push-back() which seems to be required for parsing the different between (expr) and an arrow function --- boa/src/builtins/array/tests.rs | 16 +++++++++---- boa/src/syntax/parser/cursor.rs | 4 ++++ .../expression/assignment/arrow_function.rs | 24 ++++++++++++++----- .../parser/expression/assignment/mod.rs | 6 +++++ boa/src/syntax/parser/expression/mod.rs | 1 + boa/src/syntax/parser/statement/mod.rs | 2 ++ boa/src/syntax/parser/tests.rs | 15 ++++++++++++ 7 files changed, 57 insertions(+), 11 deletions(-) diff --git a/boa/src/builtins/array/tests.rs b/boa/src/builtins/array/tests.rs index 3794361817f..08c053acb63 100644 --- a/boa/src/builtins/array/tests.rs +++ b/boa/src/builtins/array/tests.rs @@ -409,6 +409,17 @@ fn last_index_of() { assert_eq!(second_in_many, String::from("1")); } +#[test] +fn fill_obj_ref() { + let realm = Realm::create(); + let mut engine = Interpreter::new(realm); + + // test object reference + forward(&mut engine, "a = (new Array(3)).fill({});"); + forward(&mut engine, "a[0].hi = 'hi';"); + assert_eq!(forward(&mut engine, "a[0].hi"), String::from("hi")); +} + #[test] fn fill() { let realm = Realm::create(); @@ -499,11 +510,6 @@ fn fill() { forward(&mut engine, "a.fill().join()"), String::from("undefined,undefined,undefined") ); - - // test object reference - forward(&mut engine, "a = (new Array(3)).fill({});"); - forward(&mut engine, "a[0].hi = 'hi';"); - assert_eq!(forward(&mut engine, "a[0].hi"), String::from("hi")); } #[test] diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 20cb915257d..32bfa6daa29 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -149,6 +149,10 @@ where ret.map(|token| Ok(token)) } + pub(super) fn push_back(&mut self, token: Token) { + self.peeked.push_front(Some(token)); + } + // /// Moves the cursor to the previous token and returns the token. // pub(super) fn back(&mut self) -> Option> { // unimplemented!(); diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index 947ce2e88ac..d24efc98b95 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -19,7 +19,7 @@ use crate::{ error::{ErrorContext, ParseError, ParseResult}, function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - AllowAwait, AllowIn, AllowYield, Cursor, TokenParser, + AllowAwait, AllowIn, AllowYield, Cursor, TokenParser, Token }, }, BoaProfiler, @@ -70,13 +70,25 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ArrowFunction", "Parsing"); - let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)?; - let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token?.kind() { + let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; + let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token.kind() { // CoverParenthesizedExpressionAndArrowParameterList + + // Problem code - This doesn't work if the statement is of the form (expr) because the first '(' is consumed + cursor.expect(Punctuator::OpenParen, "arrow function")?; - let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect(Punctuator::CloseParen, "arrow function")?; - params + + match FormalParameters::new(self.allow_yield, self.allow_await).parse(cursor) { + Ok(params) => { + cursor.expect(Punctuator::CloseParen, "arrow function")?; + params + } + Err(e) => { + cursor.push_back(next_token); + return Err(e); + } + } + // let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(cursor)?; } else { let param = BindingIdentifier::new(self.allow_yield, self.allow_await) .parse(cursor) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 3a48797769e..6cb8919e561 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -100,8 +100,11 @@ where } } } + + // (a,b)=>{} TokenKind::Punctuator(Punctuator::OpenParen) => { + println!("Before arrow function attempt cursor.peek = {:?}", cursor.peek()); if let Some(node) = ArrowFunction::new(self.allow_in, self.allow_yield, self.allow_await) .try_parse(cursor) @@ -109,7 +112,10 @@ where { return Ok(node); } + println!("After arrow function attempt cursor.peek = {:?}", cursor.peek()); } + + _ => {} } diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 0768e9276a4..e72e35327d4 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -64,6 +64,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; println!("expression lhs: {:?}", lhs); while let Some(tok) = cursor.peek() { + println!("Token peeked = {:?}", tok); match tok?.kind() { &TokenKind::Punctuator(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 71c451e2bc5..957eef9b323 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -315,6 +315,8 @@ where let _timer = BoaProfiler::global().start_event("StatementListItem", "Parsing"); let tok = cursor.peek().ok_or(ParseError::AbruptEnd)??; + println!("StatementListItem start token: {:?}", tok); + match tok.kind { TokenKind::Keyword(Keyword::Function) | TokenKind::Keyword(Keyword::Const) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index ba1a88c9dfb..26ea70c7620 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -31,6 +31,21 @@ pub(super) fn check_invalid(js: &str) { assert!(Parser::new(js.as_bytes()).parse_all().is_err()); } +// +// a[0].hi = 'hi'; +// a[0].hi +// + +#[test] +fn array_fill_obj_ref() { + let init = r#"a = (new Array(3)).fill({});"#; + let res = Parser::new(init.as_bytes()) + .parse_all() + .expect("failed to parse"); + + println!("Result {:?}", res); +} + #[test] fn check_single_statement() { let init = r#" From 554ae907b5996ffd4195837c9aff12c36f311c68 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 20:22:09 +0100 Subject: [PATCH 080/289] Fixed bug where line terminators were not ignored during object literal parsing --- boa/src/syntax/parser/cursor.rs | 4 +- .../primary/object_initializer/mod.rs | 7 ++ .../parser/statement/declaration/lexical.rs | 3 + boa/src/syntax/parser/tests.rs | 82 +++++++++---------- 4 files changed, 53 insertions(+), 43 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 32bfa6daa29..9ebc967f795 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -326,8 +326,8 @@ where /// Advance the cursor to the next token and retrieve it, only if it's of `kind` type. /// - /// When the next token is a `kind` token, get the token, otherwise return `None`. This - /// function skips line terminators. + /// When the next token is a `kind` token, get the token, otherwise return `None`. + pub(super) fn next_if(&mut self, kind: K) -> Option> where K: Into, diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 855bb7c016e..096aa933bda 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -68,6 +68,12 @@ where if cursor.next_if(Punctuator::CloseBlock).is_some() { break; } + + if cursor.next_if(TokenKind::LineTerminator).is_some() { + // Skip line terminators. + continue; + } + elements .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(cursor)?); @@ -133,6 +139,7 @@ where } let prop_name = cursor.next().ok_or(ParseError::AbruptEnd)??.to_string(); + println!("Prop_name: {:?}", prop_name); if cursor.next_if(Punctuator::Colon).is_some() { let val = AssignmentExpression::new(true, self.allow_yield, self.allow_await) .parse(cursor)?; diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 9ddd1d5f879..b112c58d6d7 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -128,6 +128,9 @@ where LexicalBinding::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; + println!("LexicalBinding: ident: {:?}", ident); + println!("LexicalBinding: init: {:?}", init); + if self.is_const { if let Some(init) = init { const_decls.push(ConstDecl::new(ident, init)); diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 26ea70c7620..ac09d577e12 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -46,47 +46,47 @@ fn array_fill_obj_ref() { println!("Result {:?}", res); } -#[test] -fn check_single_statement() { - let init = r#" - arr.pop(); - "#; - let res = Parser::new(init.as_bytes()) - .parse_all() - .expect("failed to parse"); - - println!("Result {:?}", res); -} - -#[test] -fn check_no_semicolon_statement() { - let init = r#" - arr.pop(); - return arr.len(); - "#; - let res = Parser::new(init.as_bytes()) - .parse_all() - .expect("failed to parse"); - - println!("Result {:?}", res); -} - -#[test] -fn check_function_no_semicolon_statement() { - let init = r#" - var arr = [11, 23, 45]; - - function foo() { - arr.pop() - return arr.len < 3; - } - "#; - let res = Parser::new(init.as_bytes()) - .parse_all() - .expect("failed to parse"); - - println!("Result {:?}", res); -} +// #[test] +// fn check_single_statement() { +// let init = r#" +// arr.pop(); +// "#; +// let res = Parser::new(init.as_bytes()) +// .parse_all() +// .expect("failed to parse"); + +// println!("Result {:?}", res); +// } + +// // #[test] +// // fn check_no_semicolon_statement() { +// // let init = r#" +// // arr.pop(); +// // return arr.len(); +// // "#; +// // let res = Parser::new(init.as_bytes()) +// // .parse_all() +// // .expect("failed to parse"); +// // +// // println!("Result {:?}", res); +// // } + +// #[test] +// fn check_function_no_semicolon_statement() { +// let init = r#" +// var arr = [11, 23, 45]; + +// function foo() { +// arr.pop() +// return arr.len < 3; +// } +// "#; +// let res = Parser::new(init.as_bytes()) +// .parse_all() +// .expect("failed to parse"); + +// println!("Result {:?}", res); +// } /// Should be parsed as `new Class().method()` instead of `new (Class().method())` #[test] From e48de3ddf903392d4bd994db89ce274dd09ba5c1 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 20:35:17 +0100 Subject: [PATCH 081/289] Fixed bug where line terminators were rejected within object literal definition --- .../parser/expression/primary/object_initializer/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 096aa933bda..72876549612 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -78,10 +78,17 @@ where elements .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(cursor)?); + println!("Object literal elements: {:?}", elements); + if cursor.next_if(Punctuator::CloseBlock).is_some() { break; } + if cursor.next_if(TokenKind::LineTerminator).is_some() { + // Skip line terminators. + continue; + } + if cursor.next_if(Punctuator::Comma).is_none() { let next_token = cursor.next().ok_or(ParseError::AbruptEnd)??; return Err(ParseError::expected( From 16480d9f4f85697164aabafd8fb97f5879bcc892 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 21:35:34 +0100 Subject: [PATCH 082/289] fixed bug where anonymous functions as function arguments could not be parsed --- .../expression/primary/function_expression.rs | 5 +++++ boa/src/syntax/parser/expression/primary/mod.rs | 1 + boa/src/syntax/parser/statement/mod.rs | 17 +++++++++++++---- boa/src/syntax/parser/tests.rs | 10 ++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/parser/expression/primary/function_expression.rs b/boa/src/syntax/parser/expression/primary/function_expression.rs index b239e8c295f..781a32d0471 100644 --- a/boa/src/syntax/parser/expression/primary/function_expression.rs +++ b/boa/src/syntax/parser/expression/primary/function_expression.rs @@ -40,8 +40,13 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("FunctionExpression", "Parsing"); + + println!("Before binding identifier, cursor peek: {:?}", cursor.peek()); + let name = BindingIdentifier::new(false, false).try_parse(cursor); + println!("Cursor peek, func expression: {:?}", cursor.peek()); + cursor.expect(Punctuator::OpenParen, "function expression")?; let params = FormalParameters::new(false, false).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 7f2128f8373..44978e9ad66 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -71,6 +71,7 @@ where TokenKind::Keyword(Keyword::This) => Ok(Node::This), // TokenKind::Keyword(Keyword::Arguments) => Ok(Node::new(NodeBase::Arguments, tok.pos)), TokenKind::Keyword(Keyword::Function) => { + println!("Cursor peek: {:?}", cursor.peek()); FunctionExpression.parse(cursor).map(Node::from) } TokenKind::Punctuator(Punctuator::OpenParen) => { diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 957eef9b323..c5a975bf2f9 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -426,12 +426,21 @@ where let _timer = BoaProfiler::global().start_event("BindingIdentifier", "Parsing"); // TODO: strict mode. - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; match next_token.kind() { - TokenKind::Identifier(ref s) => Ok(s.clone()), - TokenKind::Keyword(k @ Keyword::Yield) if !self.allow_yield.0 => Ok(k.as_str().into()), - TokenKind::Keyword(k @ Keyword::Await) if !self.allow_await.0 => Ok(k.as_str().into()), + TokenKind::Identifier(ref s) => { + cursor.next(); // Consume the token. + Ok(s.clone()) + }, + TokenKind::Keyword(k @ Keyword::Yield) if !self.allow_yield.0 => { + cursor.next(); + Ok(k.as_str().into()) + }, + TokenKind::Keyword(k @ Keyword::Await) if !self.allow_await.0 => { + cursor.next(); + Ok(k.as_str().into()) + }, _ => Err(ParseError::expected( vec![TokenKind::identifier("identifier")], next_token.clone(), diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index ac09d577e12..100f35bd0be 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -31,6 +31,16 @@ pub(super) fn check_invalid(js: &str) { assert!(Parser::new(js.as_bytes()).parse_all().is_err()); } +#[test] +fn function_dec() { + let init = r#"function(k, v){return k;}"#; + let res = Parser::new(init.as_bytes()) + .parse_all() + .expect("failed to parse"); + + println!("Result {:?}", res); +} + // // a[0].hi = 'hi'; // a[0].hi From bd0822465928e2e04aba0711c265a46ecb7bfe51 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 21:53:31 +0100 Subject: [PATCH 083/289] Fixed bug with line terminators being supported in WhileLoop --- boa/src/syntax/lexer/tests.rs | 14 ----------- .../parser/statement/iteration/tests.rs | 23 ++++++++++++++++++- .../statement/iteration/while_statement.rs | 11 +++++++++ boa/src/syntax/parser/tests.rs | 10 -------- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index b77615d29a6..9562d5ce9c7 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -23,20 +23,6 @@ where ); } -#[test] -fn check_no_semicolon_statement_lex() { - let s1 = r#"arr.pop() - return arr.len < 3; - "#; - let mut lexer = Lexer::new(s1.as_bytes()); - - for l in lexer { - println!("{:?}", l); - } - - assert!(false, "This is debug test and should be removed"); -} - #[test] fn check_single_line_comment() { let s1 = "var \n//This is a comment\ntrue"; diff --git a/boa/src/syntax/parser/statement/iteration/tests.rs b/boa/src/syntax/parser/statement/iteration/tests.rs index 06c00772aab..4e0860880fb 100644 --- a/boa/src/syntax/parser/statement/iteration/tests.rs +++ b/boa/src/syntax/parser/statement/iteration/tests.rs @@ -1,7 +1,7 @@ use crate::syntax::{ ast::{ node::{ - field::GetConstField, BinOp, Block, Call, DoWhileLoop, Identifier, UnaryOp, VarDecl, + field::GetConstField, BinOp, Block, Break, Call, DoWhileLoop, WhileLoop, Identifier, UnaryOp, VarDecl, VarDeclList, }, op::{self, AssignOp, CompOp}, @@ -59,3 +59,24 @@ fn check_do_while_semicolon_insertion() { ], ); } + +/// Checks parsing of a while statement which is seperated out with line terminators. +#[test] +fn while_spaces() { + check_parser( + r#" + + while + + ( + + true + + ) + + break; + + "#, + vec![WhileLoop::new(Const::from(true), Break::new::<_, Box>(None)).into()], + ); +} \ No newline at end of file diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index 71239c0cfc1..e47396b40f9 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -5,6 +5,7 @@ use crate::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, }, + lexer::TokenKind::LineTerminator, }, BoaProfiler, }; @@ -55,12 +56,22 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("WhileStatement", "Parsing"); cursor.expect(Keyword::While, "while statement")?; + + // Line terminators can exist between a While and the condition. + while cursor.next_if(LineTerminator).is_some() {} + cursor.expect(Punctuator::OpenParen, "while statement")?; + while cursor.next_if(LineTerminator).is_some() {} + let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + while cursor.next_if(LineTerminator).is_some() {} + cursor.expect(Punctuator::CloseParen, "while statement")?; + while cursor.next_if(LineTerminator).is_some() {} + let body = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 100f35bd0be..ac09d577e12 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -31,16 +31,6 @@ pub(super) fn check_invalid(js: &str) { assert!(Parser::new(js.as_bytes()).parse_all().is_err()); } -#[test] -fn function_dec() { - let init = r#"function(k, v){return k;}"#; - let res = Parser::new(init.as_bytes()) - .parse_all() - .expect("failed to parse"); - - println!("Result {:?}", res); -} - // // a[0].hi = 'hi'; // a[0].hi From 8b57683714da9c053f86e6c2d2d6607e1dcf76e6 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 22:12:21 +0100 Subject: [PATCH 084/289] Removed need for special peek_semicolon with do_while flag --- boa/src/syntax/parser/cursor.rs | 18 ++---------------- .../syntax/parser/statement/break_stm/mod.rs | 4 ++-- .../parser/statement/continue_stm/mod.rs | 4 ++-- .../parser/statement/declaration/lexical.rs | 2 +- .../statement/iteration/do_while_statement.rs | 7 ++++++- boa/src/syntax/parser/statement/mod.rs | 2 +- .../syntax/parser/statement/return_stm/mod.rs | 4 ++-- boa/src/syntax/parser/statement/variable.rs | 4 ++-- 8 files changed, 18 insertions(+), 27 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 9ebc967f795..edc99e6122d 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -237,8 +237,7 @@ where /// /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion pub(super) fn peek_semicolon( - &mut self, - do_while: bool, + &mut self ) -> Result<(bool, Option), ParseError> { match self.peek() { Some(Ok(tk)) => { @@ -249,18 +248,6 @@ where Ok((true, Some(tk))) } _ => { - if do_while { - todo!(); - - // let tok = self - // .tokens - // .get(self.pos - 1) - // .expect("could not find previous token"); - // if tok.kind == TokenKind::Punctuator(Punctuator::CloseParen) { - // return Ok((true, Some(tk))); - // } - } - Ok((false, Some(tk))) } } @@ -277,13 +264,12 @@ where /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion pub(super) fn expect_semicolon( &mut self, - do_while: bool, context: &'static str, ) -> Result, ParseError> { println!("Context: {}", context); println!("Peek: {:?}", self.peek()); - match self.peek_semicolon(do_while)? { + match self.peek_semicolon()? { (true, Some(tk)) => match tk.kind() { TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { self.next(); // Consume the token. diff --git a/boa/src/syntax/parser/statement/break_stm/mod.rs b/boa/src/syntax/parser/statement/break_stm/mod.rs index 81ff206e294..bb353a9b4c7 100644 --- a/boa/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa/src/syntax/parser/statement/break_stm/mod.rs @@ -61,7 +61,7 @@ where let _timer = BoaProfiler::global().start_event("BreakStatement", "Parsing"); cursor.expect(Keyword::Break, "break statement")?; - let label = if let (true, tok) = cursor.peek_semicolon(false)? { + let label = if let (true, tok) = cursor.peek_semicolon()? { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) => { let _ = cursor.next(); @@ -72,7 +72,7 @@ where None } else { let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect_semicolon(false, "continue statement")?; + cursor.expect_semicolon("continue statement")?; Some(label) }; diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index 63f62e22ce5..d7e920e5b02 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -61,7 +61,7 @@ where let _timer = BoaProfiler::global().start_event("ContinueStatement", "Parsing"); cursor.expect(Keyword::Continue, "continue statement")?; - let label = if let (true, tok) = cursor.peek_semicolon(false)? { + let label = if let (true, tok) = cursor.peek_semicolon()? { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) => { let _ = cursor.next(); @@ -72,7 +72,7 @@ where None } else { let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect_semicolon(false, "continue statement")?; + cursor.expect_semicolon( "continue statement")?; Some(label) }; diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index b112c58d6d7..8e237f4db4f 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -145,7 +145,7 @@ where let_decls.push(LetDecl::new(ident, init)); } - match cursor.peek_semicolon(false)? { + match cursor.peek_semicolon()? { (true, _) => break, (false, Some(tk)) if tk.kind == TokenKind::Punctuator(Punctuator::Comma) => { let _ = cursor.next(); diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 2b3d63d05c7..c40227984e2 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -84,7 +84,12 @@ where let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; cursor.expect(Punctuator::CloseParen, "do while statement")?; - cursor.expect_semicolon(true, "do while statement")?; + + // TODO, expect_semicolon with auto insertion for do-while. + + todo!("Expect semicolon with auto-insertion for do-while"); + + // cursor.expect_semicolon(true, "do while statement")?; Ok(DoWhileLoop::new(body, cond)) } diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index c5a975bf2f9..022f0ff3ad2 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -374,7 +374,7 @@ where println!("Cursor peek value after node peek: {:?}", cursor.peek()); - cursor.expect_semicolon(false, "expression statement")?; + cursor.expect_semicolon("expression statement")?; Ok(expr) } diff --git a/boa/src/syntax/parser/statement/return_stm/mod.rs b/boa/src/syntax/parser/statement/return_stm/mod.rs index 71b40164db7..33a232bfb7e 100644 --- a/boa/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa/src/syntax/parser/statement/return_stm/mod.rs @@ -50,7 +50,7 @@ where let _timer = BoaProfiler::global().start_event("ReturnStatement", "Parsing"); cursor.expect(Keyword::Return, "return statement")?; - if let (true, tok) = cursor.peek_semicolon(false)? { + if let (true, tok) = cursor.peek_semicolon()? { match tok { Some(tok) if tok.kind == TokenKind::Punctuator(Punctuator::Semicolon) @@ -66,7 +66,7 @@ where let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect_semicolon(false, "return statement")?; + cursor.expect_semicolon("return statement")?; Ok(Return::new(expr)) } diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index b3dab4d3ed3..6502d3cc44d 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -59,7 +59,7 @@ where let decl_list = VariableDeclarationList::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect_semicolon(false, "variable statement")?; + cursor.expect_semicolon("variable statement")?; Ok(decl_list) } @@ -115,7 +115,7 @@ where .parse(cursor)?, ); - match cursor.peek_semicolon(false)? { + match cursor.peek_semicolon()? { (true, _) => break, (false, Some(tk)) if tk.kind == TokenKind::Punctuator(Punctuator::Comma) => { let _ = cursor.next(); From 4c0b7989c53349a03ec1902f88e8881b1c17993c Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 22:43:08 +0100 Subject: [PATCH 085/289] Added dowhile semicolon insertion --- .../statement/iteration/do_while_statement.rs | 38 ++++++++++++++++--- .../parser/statement/iteration/tests.rs | 31 +++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index c40227984e2..584e768d7eb 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -85,12 +85,40 @@ where cursor.expect(Punctuator::CloseParen, "do while statement")?; - // TODO, expect_semicolon with auto insertion for do-while. - - todo!("Expect semicolon with auto-insertion for do-while"); - - // cursor.expect_semicolon(true, "do while statement")?; + expect_semicolon_dowhile(cursor)?; Ok(DoWhileLoop::new(body, cond)) } } +/// Checks that the next token is a semicolon with regards to the automatic semicolon insertion rules +/// as specified in spec. +/// +/// This is used for the check at the end of a DoWhileLoop as-opposed to the regular cursor.expect() because +/// do_while represents a special condition for automatic semicolon insertion. +/// +/// [spec]: https://tc39.es/ecma262/#sec-rules-of-automatic-semicolon-insertion +fn expect_semicolon_dowhile(cursor: &mut Cursor) -> Result<(), ParseError> +where + R: Read +{ + // The previous token is already known to be a CloseParan as this is checked as part of the dowhile parsing. + // This means that + + match cursor.peek() { + None => { + // If a do while statement ends a stream then a semicolon is automatically inserted. + cursor.next(); // Consume value. + Ok(()) + } + Some(Ok(tk)) => { + if tk.kind() == &TokenKind::Punctuator(Punctuator::Semicolon) { + cursor.next(); // Consume semicolon. + } + Ok(()) + } + Some(Err(e)) => { + Err(e) + } + } + +} diff --git a/boa/src/syntax/parser/statement/iteration/tests.rs b/boa/src/syntax/parser/statement/iteration/tests.rs index 4e0860880fb..4cfebcb70b0 100644 --- a/boa/src/syntax/parser/statement/iteration/tests.rs +++ b/boa/src/syntax/parser/statement/iteration/tests.rs @@ -60,6 +60,37 @@ fn check_do_while_semicolon_insertion() { ); } +// Checks automatic semicolon insertion after do-while with no space between closing paren +// and next statement. +#[test] +fn check_do_while_semicolon_insertion_no_space() { + check_parser( + r#"var i = 0; + do {console.log("hello");} while(i++ < 10)console.log("end");"#, + vec![ + VarDeclList::from(vec![VarDecl::new("i", Some(Const::from(0).into()))]).into(), + DoWhileLoop::new( + Block::from(vec![Call::new( + GetConstField::new(Identifier::from("console"), "log"), + vec![Const::from("hello").into()], + ) + .into()]), + BinOp::new( + CompOp::LessThan, + UnaryOp::new(op::UnaryOp::IncrementPost, Identifier::from("i")), + Const::from(10), + ), + ) + .into(), + Call::new( + GetConstField::new(Identifier::from("console"), "log"), + vec![Const::from("end").into()], + ) + .into(), + ], + ); +} + /// Checks parsing of a while statement which is seperated out with line terminators. #[test] fn while_spaces() { From 273b1e4f72d82635b6b1c78a7413f2612dcf94c1 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 23:10:27 +0100 Subject: [PATCH 086/289] Fixed bug where line terminators were not allowed between parts of dowhile --- boa/src/exec/tests.rs | 3 ++ boa/src/syntax/parser/cursor.rs | 6 +++- .../statement/iteration/do_while_statement.rs | 14 +++++++++ .../parser/statement/iteration/tests.rs | 30 ++++++++++++++++++- .../statement/iteration/while_statement.rs | 8 ++--- 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/boa/src/exec/tests.rs b/boa/src/exec/tests.rs index a332c9f9756..9a2458d5ac5 100644 --- a/boa/src/exec/tests.rs +++ b/boa/src/exec/tests.rs @@ -310,7 +310,10 @@ fn do_while_loop() { b "#; assert_eq!(&exec(multiline_statement), "256"); +} +#[test] +fn do_while_loop_at_least_once() { let body_is_executed_at_least_once = r#" a = 0; do diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index edc99e6122d..e2e6e818fef 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -313,7 +313,6 @@ where /// Advance the cursor to the next token and retrieve it, only if it's of `kind` type. /// /// When the next token is a `kind` token, get the token, otherwise return `None`. - pub(super) fn next_if(&mut self, kind: K) -> Option> where K: Into, @@ -330,4 +329,9 @@ where None => None, } } + + /// Advance the cursor to skip 0, 1 or more line terminators. + pub(super) fn skip_line_terminators(&mut self) { + while self.next_if(TokenKind::LineTerminator).is_some() {} + } } diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 584e768d7eb..7bf0971d7bd 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -65,8 +65,13 @@ where let _timer = BoaProfiler::global().start_event("DoWhileStatement", "Parsing"); cursor.expect(Keyword::Do, "do while statement")?; + // There can be space between the Do and the body. + cursor.skip_line_terminators(); + let body = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; + + cursor.skip_line_terminators(); let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; @@ -78,11 +83,20 @@ where )); } + cursor.skip_line_terminators(); + cursor.expect(Keyword::While, "do while statement")?; + + cursor.skip_line_terminators(); + cursor.expect(Punctuator::OpenParen, "do while statement")?; + cursor.skip_line_terminators(); + let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + cursor.skip_line_terminators(); + cursor.expect(Punctuator::CloseParen, "do while statement")?; expect_semicolon_dowhile(cursor)?; diff --git a/boa/src/syntax/parser/statement/iteration/tests.rs b/boa/src/syntax/parser/statement/iteration/tests.rs index 4cfebcb70b0..3d959e118df 100644 --- a/boa/src/syntax/parser/statement/iteration/tests.rs +++ b/boa/src/syntax/parser/statement/iteration/tests.rs @@ -2,7 +2,7 @@ use crate::syntax::{ ast::{ node::{ field::GetConstField, BinOp, Block, Break, Call, DoWhileLoop, WhileLoop, Identifier, UnaryOp, VarDecl, - VarDeclList, + VarDeclList, Node }, op::{self, AssignOp, CompOp}, Const, @@ -110,4 +110,32 @@ fn while_spaces() { "#, vec![WhileLoop::new(Const::from(true), Break::new::<_, Box>(None)).into()], ); +} + +/// Checks parsing of a while statement which is seperated out with line terminators. +#[test] +fn do_while_spaces() { + check_parser( + r#" + + do + + { + + break; + + } + + while (true) + + "#, + vec![DoWhileLoop::new( + Block::from( + vec![ + Break::new::>, Box>(None).into() + ] + ), + Const::Bool(true) + ).into()], + ); } \ No newline at end of file diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index e47396b40f9..b27df91401f 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -58,19 +58,19 @@ where cursor.expect(Keyword::While, "while statement")?; // Line terminators can exist between a While and the condition. - while cursor.next_if(LineTerminator).is_some() {} + cursor.skip_line_terminators(); cursor.expect(Punctuator::OpenParen, "while statement")?; - while cursor.next_if(LineTerminator).is_some() {} + cursor.skip_line_terminators(); let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - while cursor.next_if(LineTerminator).is_some() {} + cursor.skip_line_terminators(); cursor.expect(Punctuator::CloseParen, "while statement")?; - while cursor.next_if(LineTerminator).is_some() {} + cursor.skip_line_terminators(); let body = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; From a5cd476257c7cae71cbf0ca037513f9f91004642 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 23:13:09 +0100 Subject: [PATCH 087/289] Disabled 2 new lexer tests which didn't pass anyway --- boa/src/syntax/lexer/tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 9562d5ce9c7..dad921cb34b 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -417,6 +417,7 @@ fn big_exp_numbers() { } #[test] +#[ignore] fn big_literal_numbers() { let mut lexer = Lexer::new(&b"10000000000000000000000000"[0..]); @@ -639,6 +640,7 @@ fn illegal_following_numeric_literal() { } #[test] +#[ignore] fn illegal_code_point_following_numeric_literal() { // Checks as per https://tc39.es/ecma262/#sec-literals-numeric-literals that a NumericLiteral cannot // be immediately followed by an IdentifierStart where the IdentifierStart From 9370e6f7967405b258c07eeec00aa4432c0f79d9 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 23:13:37 +0100 Subject: [PATCH 088/289] Cargo fmt --- boa/src/syntax/parser/cursor.rs | 12 ++++-------- .../expression/assignment/arrow_function.rs | 6 +++--- .../parser/expression/assignment/mod.rs | 12 ++++++++---- .../expression/primary/function_expression.rs | 5 ++++- .../primary/object_initializer/mod.rs | 3 +-- .../parser/statement/continue_stm/mod.rs | 2 +- .../statement/iteration/do_while_statement.rs | 13 +++++-------- .../parser/statement/iteration/tests.rs | 19 ++++++++----------- .../statement/iteration/while_statement.rs | 2 +- boa/src/syntax/parser/statement/mod.rs | 12 +++++------- boa/src/syntax/parser/tests.rs | 4 ++-- 11 files changed, 42 insertions(+), 48 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index e2e6e818fef..ca93d8bd862 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -236,9 +236,7 @@ where /// It will automatically insert a semicolon if needed, as specified in the [spec][spec]. /// /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion - pub(super) fn peek_semicolon( - &mut self - ) -> Result<(bool, Option), ParseError> { + pub(super) fn peek_semicolon(&mut self) -> Result<(bool, Option), ParseError> { match self.peek() { Some(Ok(tk)) => { println!("Token: {:?}", tk); @@ -247,11 +245,9 @@ where TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { Ok((true, Some(tk))) } - _ => { - Ok((false, Some(tk))) - } + _ => Ok((false, Some(tk))), } - }, + } Some(Err(e)) => Err(e), None => Ok((true, None)), } @@ -331,7 +327,7 @@ where } /// Advance the cursor to skip 0, 1 or more line terminators. - pub(super) fn skip_line_terminators(&mut self) { + pub(super) fn skip_line_terminators(&mut self) { while self.next_if(TokenKind::LineTerminator).is_some() {} } } diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index d24efc98b95..b7682928ef2 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -19,7 +19,7 @@ use crate::{ error::{ErrorContext, ParseError, ParseResult}, function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - AllowAwait, AllowIn, AllowYield, Cursor, TokenParser, Token + AllowAwait, AllowIn, AllowYield, Cursor, Token, TokenParser, }, }, BoaProfiler, @@ -74,7 +74,7 @@ where let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token.kind() { // CoverParenthesizedExpressionAndArrowParameterList - // Problem code - This doesn't work if the statement is of the form (expr) because the first '(' is consumed + // Problem code - This doesn't work if the statement is of the form (expr) because the first '(' is consumed cursor.expect(Punctuator::OpenParen, "arrow function")?; @@ -88,7 +88,7 @@ where return Err(e); } } - // let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(cursor)?; + // let params = FormalParameters::new(self.allow_yield, self.allow_await).parse(cursor)?; } else { let param = BindingIdentifier::new(self.allow_yield, self.allow_await) .parse(cursor) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 6cb8919e561..44e870128b8 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -101,10 +101,12 @@ where } } - // (a,b)=>{} TokenKind::Punctuator(Punctuator::OpenParen) => { - println!("Before arrow function attempt cursor.peek = {:?}", cursor.peek()); + println!( + "Before arrow function attempt cursor.peek = {:?}", + cursor.peek() + ); if let Some(node) = ArrowFunction::new(self.allow_in, self.allow_yield, self.allow_await) .try_parse(cursor) @@ -112,10 +114,12 @@ where { return Ok(node); } - println!("After arrow function attempt cursor.peek = {:?}", cursor.peek()); + println!( + "After arrow function attempt cursor.peek = {:?}", + cursor.peek() + ); } - _ => {} } diff --git a/boa/src/syntax/parser/expression/primary/function_expression.rs b/boa/src/syntax/parser/expression/primary/function_expression.rs index 781a32d0471..1997b944cc0 100644 --- a/boa/src/syntax/parser/expression/primary/function_expression.rs +++ b/boa/src/syntax/parser/expression/primary/function_expression.rs @@ -41,7 +41,10 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("FunctionExpression", "Parsing"); - println!("Before binding identifier, cursor peek: {:?}", cursor.peek()); + println!( + "Before binding identifier, cursor peek: {:?}", + cursor.peek() + ); let name = BindingIdentifier::new(false, false).try_parse(cursor); diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 72876549612..62b0b926307 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -68,13 +68,12 @@ where if cursor.next_if(Punctuator::CloseBlock).is_some() { break; } - + if cursor.next_if(TokenKind::LineTerminator).is_some() { // Skip line terminators. continue; } - elements .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(cursor)?); diff --git a/boa/src/syntax/parser/statement/continue_stm/mod.rs b/boa/src/syntax/parser/statement/continue_stm/mod.rs index d7e920e5b02..36011eb4e9c 100644 --- a/boa/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa/src/syntax/parser/statement/continue_stm/mod.rs @@ -72,7 +72,7 @@ where None } else { let label = LabelIdentifier::new(self.allow_yield, self.allow_await).parse(cursor)?; - cursor.expect_semicolon( "continue statement")?; + cursor.expect_semicolon("continue statement")?; Some(label) }; diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 7bf0971d7bd..3de500dd472 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -70,7 +70,7 @@ where let body = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - + cursor.skip_line_terminators(); let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; @@ -104,7 +104,7 @@ where Ok(DoWhileLoop::new(body, cond)) } } -/// Checks that the next token is a semicolon with regards to the automatic semicolon insertion rules +/// Checks that the next token is a semicolon with regards to the automatic semicolon insertion rules /// as specified in spec. /// /// This is used for the check at the end of a DoWhileLoop as-opposed to the regular cursor.expect() because @@ -113,10 +113,10 @@ where /// [spec]: https://tc39.es/ecma262/#sec-rules-of-automatic-semicolon-insertion fn expect_semicolon_dowhile(cursor: &mut Cursor) -> Result<(), ParseError> where - R: Read + R: Read, { // The previous token is already known to be a CloseParan as this is checked as part of the dowhile parsing. - // This means that + // This means that match cursor.peek() { None => { @@ -130,9 +130,6 @@ where } Ok(()) } - Some(Err(e)) => { - Err(e) - } + Some(Err(e)) => Err(e), } - } diff --git a/boa/src/syntax/parser/statement/iteration/tests.rs b/boa/src/syntax/parser/statement/iteration/tests.rs index 3d959e118df..364ae147d6e 100644 --- a/boa/src/syntax/parser/statement/iteration/tests.rs +++ b/boa/src/syntax/parser/statement/iteration/tests.rs @@ -1,8 +1,8 @@ use crate::syntax::{ ast::{ node::{ - field::GetConstField, BinOp, Block, Break, Call, DoWhileLoop, WhileLoop, Identifier, UnaryOp, VarDecl, - VarDeclList, Node + field::GetConstField, BinOp, Block, Break, Call, DoWhileLoop, Identifier, Node, + UnaryOp, VarDecl, VarDeclList, WhileLoop, }, op::{self, AssignOp, CompOp}, Const, @@ -60,7 +60,7 @@ fn check_do_while_semicolon_insertion() { ); } -// Checks automatic semicolon insertion after do-while with no space between closing paren +// Checks automatic semicolon insertion after do-while with no space between closing paren // and next statement. #[test] fn check_do_while_semicolon_insertion_no_space() { @@ -130,12 +130,9 @@ fn do_while_spaces() { "#, vec![DoWhileLoop::new( - Block::from( - vec![ - Break::new::>, Box>(None).into() - ] - ), - Const::Bool(true) - ).into()], + Block::from(vec![Break::new::>, Box>(None).into()]), + Const::Bool(true), + ) + .into()], ); -} \ No newline at end of file +} diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index b27df91401f..102ee5ba410 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -1,11 +1,11 @@ use crate::{ syntax::{ ast::{node::WhileLoop, Keyword, Punctuator}, + lexer::TokenKind::LineTerminator, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, }, - lexer::TokenKind::LineTerminator, }, BoaProfiler, }; diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 022f0ff3ad2..8c2e679df63 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -228,9 +228,7 @@ where loop { match cursor.peek() { - Some(Ok(token)) - if token.kind() == &TokenKind::LineTerminator => - { + Some(Ok(token)) if token.kind() == &TokenKind::LineTerminator => { // Skip line terminators. cursor.next(); continue; @@ -432,15 +430,15 @@ where TokenKind::Identifier(ref s) => { cursor.next(); // Consume the token. Ok(s.clone()) - }, + } TokenKind::Keyword(k @ Keyword::Yield) if !self.allow_yield.0 => { - cursor.next(); + cursor.next(); Ok(k.as_str().into()) - }, + } TokenKind::Keyword(k @ Keyword::Await) if !self.allow_await.0 => { cursor.next(); Ok(k.as_str().into()) - }, + } _ => Err(ParseError::expected( vec![TokenKind::identifier("identifier")], next_token.clone(), diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index ac09d577e12..aee076af019 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -40,8 +40,8 @@ pub(super) fn check_invalid(js: &str) { fn array_fill_obj_ref() { let init = r#"a = (new Array(3)).fill({});"#; let res = Parser::new(init.as_bytes()) - .parse_all() - .expect("failed to parse"); + .parse_all() + .expect("failed to parse"); println!("Result {:?}", res); } From 548ae5e3f801b4fbd20893e367f1e6e5dbcd2d8e Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 23:16:18 +0100 Subject: [PATCH 089/289] Cleanup --- boa/src/syntax/parser/cursor.rs | 3 - .../parser/expression/assignment/mod.rs | 8 - .../expression/left_hand_side/arguments.rs | 2 - .../parser/expression/left_hand_side/call.rs | 7 - boa/src/syntax/parser/expression/mod.rs | 2 - .../expression/primary/function_expression.rs | 7 - .../syntax/parser/expression/primary/mod.rs | 1 - .../primary/object_initializer/mod.rs | 3 - .../parser/statement/declaration/lexical.rs | 5 +- boa/src/syntax/parser/statement/mod.rs | 8 - boa/src/syntax/parser/tests.rs | 57 --- ignore_temp/number_grammar_based.rs | 261 ----------- ignore_temp/number_old.rs | 424 ------------------ 13 files changed, 1 insertion(+), 787 deletions(-) delete mode 100644 ignore_temp/number_grammar_based.rs delete mode 100644 ignore_temp/number_old.rs diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index ca93d8bd862..f01691ee852 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -239,7 +239,6 @@ where pub(super) fn peek_semicolon(&mut self) -> Result<(bool, Option), ParseError> { match self.peek() { Some(Ok(tk)) => { - println!("Token: {:?}", tk); match tk.kind() { TokenKind::Punctuator(Punctuator::Semicolon) => Ok((true, Some(tk))), TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { @@ -262,8 +261,6 @@ where &mut self, context: &'static str, ) -> Result, ParseError> { - println!("Context: {}", context); - println!("Peek: {:?}", self.peek()); match self.peek_semicolon()? { (true, Some(tk)) => match tk.kind() { diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 44e870128b8..5f01555c96d 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -103,10 +103,6 @@ where // (a,b)=>{} TokenKind::Punctuator(Punctuator::OpenParen) => { - println!( - "Before arrow function attempt cursor.peek = {:?}", - cursor.peek() - ); if let Some(node) = ArrowFunction::new(self.allow_in, self.allow_yield, self.allow_await) .try_parse(cursor) @@ -114,10 +110,6 @@ where { return Ok(node); } - println!( - "After arrow function attempt cursor.peek = {:?}", - cursor.peek() - ); } _ => {} diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index e44f504e81d..84dae149ccf 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -57,13 +57,11 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Arguments", "Parsing"); - println!("Parsing args"); cursor.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); loop { let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; - println!("Next token: {:?}", next_token); match next_token.kind() { TokenKind::Punctuator(Punctuator::CloseParen) => { diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 31968a30ac9..1d513712212 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -67,18 +67,14 @@ where let tk = cursor.peek(); - println!("Parse Call Expression Token: {:?}", tk); let mut lhs = match tk { Some(_) if tk.unwrap()?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { - println!("Open paran"); let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; - println!("After args: {:?}", cursor.peek()); Node::from(Call::new(self.first_member_expr, args)) } _ => { - println!("Not open paran"); let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::OpenParen)], @@ -88,11 +84,8 @@ where } }; - println!("Parse Call Expression Finish Token: {:?}", cursor.peek()); - while let Some(tok) = cursor.peek() { let token = tok?.clone(); - println!("Call expression parsing... token: {:?}", token); match token.kind() { TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index e72e35327d4..2eb3003b070 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -62,9 +62,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; - println!("expression lhs: {:?}", lhs); while let Some(tok) = cursor.peek() { - println!("Token peeked = {:?}", tok); match tok?.kind() { &TokenKind::Punctuator(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); diff --git a/boa/src/syntax/parser/expression/primary/function_expression.rs b/boa/src/syntax/parser/expression/primary/function_expression.rs index 1997b944cc0..2b3aba97593 100644 --- a/boa/src/syntax/parser/expression/primary/function_expression.rs +++ b/boa/src/syntax/parser/expression/primary/function_expression.rs @@ -41,15 +41,8 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("FunctionExpression", "Parsing"); - println!( - "Before binding identifier, cursor peek: {:?}", - cursor.peek() - ); - let name = BindingIdentifier::new(false, false).try_parse(cursor); - println!("Cursor peek, func expression: {:?}", cursor.peek()); - cursor.expect(Punctuator::OpenParen, "function expression")?; let params = FormalParameters::new(false, false).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 44978e9ad66..7f2128f8373 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -71,7 +71,6 @@ where TokenKind::Keyword(Keyword::This) => Ok(Node::This), // TokenKind::Keyword(Keyword::Arguments) => Ok(Node::new(NodeBase::Arguments, tok.pos)), TokenKind::Keyword(Keyword::Function) => { - println!("Cursor peek: {:?}", cursor.peek()); FunctionExpression.parse(cursor).map(Node::from) } TokenKind::Punctuator(Punctuator::OpenParen) => { diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 62b0b926307..c64ed14bb93 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -77,8 +77,6 @@ where elements .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(cursor)?); - println!("Object literal elements: {:?}", elements); - if cursor.next_if(Punctuator::CloseBlock).is_some() { break; } @@ -145,7 +143,6 @@ where } let prop_name = cursor.next().ok_or(ParseError::AbruptEnd)??.to_string(); - println!("Prop_name: {:?}", prop_name); if cursor.next_if(Punctuator::Colon).is_some() { let val = AssignmentExpression::new(true, self.allow_yield, self.allow_await) .parse(cursor)?; diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 8e237f4db4f..47263099b9c 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -127,10 +127,7 @@ where let (ident, init) = LexicalBinding::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - - println!("LexicalBinding: ident: {:?}", ident); - println!("LexicalBinding: init: {:?}", init); - + if self.is_const { if let Some(init) = init { const_decls.push(ConstDecl::new(ident, init)); diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 8c2e679df63..27c62470986 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -313,8 +313,6 @@ where let _timer = BoaProfiler::global().start_event("StatementListItem", "Parsing"); let tok = cursor.peek().ok_or(ParseError::AbruptEnd)??; - println!("StatementListItem start token: {:?}", tok); - match tok.kind { TokenKind::Keyword(Keyword::Function) | TokenKind::Keyword(Keyword::Const) @@ -363,15 +361,9 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("ExpressionStatement", "Parsing"); - println!("Express statement before node peek: {:?}", cursor.peek()); - // TODO: lookahead let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - println!("Expression: {:?}", expr); - - println!("Cursor peek value after node peek: {:?}", cursor.peek()); - cursor.expect_semicolon("expression statement")?; Ok(expr) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index aee076af019..8de593d9458 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -31,63 +31,6 @@ pub(super) fn check_invalid(js: &str) { assert!(Parser::new(js.as_bytes()).parse_all().is_err()); } -// -// a[0].hi = 'hi'; -// a[0].hi -// - -#[test] -fn array_fill_obj_ref() { - let init = r#"a = (new Array(3)).fill({});"#; - let res = Parser::new(init.as_bytes()) - .parse_all() - .expect("failed to parse"); - - println!("Result {:?}", res); -} - -// #[test] -// fn check_single_statement() { -// let init = r#" -// arr.pop(); -// "#; -// let res = Parser::new(init.as_bytes()) -// .parse_all() -// .expect("failed to parse"); - -// println!("Result {:?}", res); -// } - -// // #[test] -// // fn check_no_semicolon_statement() { -// // let init = r#" -// // arr.pop(); -// // return arr.len(); -// // "#; -// // let res = Parser::new(init.as_bytes()) -// // .parse_all() -// // .expect("failed to parse"); -// // -// // println!("Result {:?}", res); -// // } - -// #[test] -// fn check_function_no_semicolon_statement() { -// let init = r#" -// var arr = [11, 23, 45]; - -// function foo() { -// arr.pop() -// return arr.len < 3; -// } -// "#; -// let res = Parser::new(init.as_bytes()) -// .parse_all() -// .expect("failed to parse"); - -// println!("Result {:?}", res); -// } - /// Should be parsed as `new Class().method()` instead of `new (Class().method())` #[test] fn check_construct_call_precedence() { diff --git a/ignore_temp/number_grammar_based.rs b/ignore_temp/number_grammar_based.rs deleted file mode 100644 index 963ad4238b2..00000000000 --- a/ignore_temp/number_grammar_based.rs +++ /dev/null @@ -1,261 +0,0 @@ -use super::{Cursor, Error, TokenKind, Tokenizer}; -use crate::builtins::BigInt; -use crate::syntax::ast::{Position, Span}; -use crate::syntax::lexer::{token::Numeric, Token}; -use std::io::Read; -use std::str::FromStr; - -/// Number literal lexing. -/// -/// Assumes the initial digit is consumed by the cursor (stored in init). -/// -/// More information: -/// - [ECMAScript reference][spec] -/// - [MDN documentation][mdn] -/// -/// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals -/// [mdn]: -#[derive(Debug, Clone, Copy)] -pub(super) struct NumberLiteral { - init: char, - strict_mode: bool, -} - -impl NumberLiteral { - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode } - } -} - -/// This is a helper structure -/// -/// This structure helps with identifying what numerical type it is and what base is it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum NumericKind { - Rational, - Integer(u8), - BigInt(u8), -} - -impl NumericKind { - /// Get the base of the number kind. - fn base(self) -> u32 { - match self { - Self::Rational => 10, - Self::Integer(base) => base as u32, - Self::BigInt(base) => base as u32, - } - } - - /// Converts `self` to BigInt kind. - fn to_bigint(self) -> Self { - match self { - Self::Rational => unreachable!("can not convert rational number to BigInt"), - Self::Integer(base) => Self::BigInt(base), - Self::BigInt(base) => Self::BigInt(base), - } - } -} - -impl Tokenizer for NumberLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - let buf = self.init.to_string(); - - if let Ok(token) = DecimalLiteral::new(self.init, self.strict_mode).lex(cursor, start_pos) { - return Ok(token); // Parsed successfully. - } - if let Ok(token) = DecimalBigIntegerLiteral::new(self.init).lex(cursor, start_pos) { - return Ok(token); // Parsed successfully. - } - if let Ok(token) = NonDecimalIntegerLiteral::new(self.init).lex(cursor, start_pos) { - return Ok(token); // Parsed successfully. - } - - Err(Error::Reverted()) - - - // Ok(Token::new( - // TokenKind::NumericLiteral(num), - // Span::new(start_pos, cursor.pos()), - // )) - } -} - -#[derive(Debug, Clone, Copy)] -pub(super) struct DecimalLiteral { - init: char, - strict_mode: bool, -} - -impl DecimalLiteral { - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode} - } -} - -impl Tokenizer for DecimalLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - - - let dil = DecimalIntegerLiteral::new(self.init, self.strict_mode).lex(cursor, start_pos); - match dil { - Ok(dil_token) => { - // DecimalIntegerLiteral - - if cursor.next_is('.')? { - // DecimalIntegerLiteral. - - // Consume the '.' - cursor.next(); - - - // May be followed by DecimalDigits - let dd = DecimalDigits::new(self.strict_mode).lex(cursor, start_pos); - match dd { - Ok(dd_token) => { - // DecimalIntegerLiteral.DecimalDigits - let ep = ExponentPart::new(self.strict_mode).lex(cursor, start_pos); - match ep { - Ok(ep_token) => { - // DecimalIntegerLiteral.DecimalDigits ExponentPart - // Terminal pattern. - dil + dd + ep - } - Err(Error::Reverted()) => { - // DecimalIntegerLiteral.DecimalDigits - // Terminal pattern. - dil + dd - } - Err (e) => { - // Some other error preventing lexing. - Err(e) - } - } - } - Err(Error::Reverted()) => { - // DecimalIntegerLiteral. - // Terminal pattern. - dd - } - Err(e) => { - // Some other error preventing lexing. - Err(e) - } - } - } else { - // DecimalIntegerLiteral - - // May be followed by ExponentPart - let ep = ExponentPart::new(self.strict_mode).lex(cursor, start_pos); - match ep { - Ok(ep_token) => { - // DecimalIntegerLiteral ExponentPart - // Terminal pattern. - dil + ep - } - Err(Error::Reverted()) => { - // DecimalIntegerLiteral - dil - } - Err (e) => { - // Some other error preventing lexing. - Err(e) - } - } - } - } - Err(Error::Reverted(buf)) => { - // If a decimal literal doesn't start with a DecimalIntegerLiteral it must start with a '.' followed by DecimalDigits. - if cursor.next_is('.')? { - // . - let dd = DecimalDigits::new(self.strict_mode).lex(cursor, start_pos); - match dd { - Ok(dd_token) => { - // . DecimalDigits - - // May be followed by ExponentPart - let ep = ExponentPart::new(self.strict_mode).lex(cursor, start_pos); - match ep { - Ok(ep_token) => { - // . DecimalDigits ExponentPart - dd + ep - } - Err(Error::Reverted()) => { - // . DecimalDigits - dd - } - Err (e) => { - // Some other error preventing lexing. - Err(e) - } - } - } - Err(e) => { - // A DecimalDigits couldn't be lexed or some other error prevents lexing. - Err(e) - } - } - } else { - Err(Error::Reverted()) - } - } - Err(e) => { - // Some other error. - Err(e) - } - } - } -} - -#[derive(Debug, Clone, Copy)] -pub(super) struct DecimalBigIntegerLiteral { - init: char, - strict_mode: bool, -} - -impl DecimalBigIntegerLiteral{ - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode } - } -} - -impl Tokenizer for DecimalBigIntegerLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - Err(Error::Reverted()) - } -} - - -#[derive(Debug, Clone, Copy)] -pub(super) struct NonDecimalIntegerLiteral { - init: char, - strict_mode: bool, -} - -impl NonDecimalIntegerLiteral { - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode } - } -} - -impl Tokenizer for NonDecimalIntegerLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - Err(Error::Reverted()) - } -} \ No newline at end of file diff --git a/ignore_temp/number_old.rs b/ignore_temp/number_old.rs deleted file mode 100644 index 796ef68abdc..00000000000 --- a/ignore_temp/number_old.rs +++ /dev/null @@ -1,424 +0,0 @@ -use super::{Cursor, Error, TokenKind, Tokenizer}; -use crate::builtins::BigInt; -use crate::syntax::ast::{Position, Span}; -use crate::syntax::lexer::{token::Numeric, Token}; -use std::io::Read; -use std::str::FromStr; - -/// Number literal lexing. -/// -/// Assumes the digit is consumed by the cursor (stored in init). -/// -/// More information: -/// - [ECMAScript reference][spec] -/// - [MDN documentation][mdn] -/// -/// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals -/// [mdn]: -#[derive(Debug, Clone, Copy)] -pub(super) struct NumberLiteral { - init: char, - strict_mode: bool, -} - -impl NumberLiteral { - /// Creates a new string literal lexer. - pub(super) fn new(init: char, strict_mode: bool) -> Self { - Self { init, strict_mode } - } -} - -/// This is a helper structure -/// -/// This structure helps with identifying what numerical type it is and what base is it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum NumericKind { - Rational, - Integer(u8), - BigInt(u8), -} - -impl NumericKind { - /// Get the base of the number kind. - fn base(self) -> u32 { - match self { - Self::Rational => 10, - Self::Integer(base) => base as u32, - Self::BigInt(base) => base as u32, - } - } - - /// Converts `self` to BigInt kind. - fn to_bigint(self) -> Self { - match self { - Self::Rational => unreachable!("can not convert rational number to BigInt"), - Self::Integer(base) => Self::BigInt(base), - Self::BigInt(base) => Self::BigInt(base), - } - } -} - -impl Tokenizer for NumberLiteral { - fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result - where - R: Read, - { - let mut buf = self.init.to_string(); - - // Default assume the number is a base 10 integer. - let mut kind = NumericKind::Integer(10); - - if self.init == '0' { - if cursor.next_is('x') | cursor.next_is('X') { - // HexIntegerLiteral - - // Consume the 0x. - cursor.next(); - buf.pop(); - - kind = NumericKind::Integer(16); - } else if cursor.next_is('o') | cursor.next_is('O') { - // OctalIntegerLiteral - - // Consume the 0o. - cursor.next(); - buf.pop(); - - kind = NumericKind::Integer(8); - } else if cursor.next_is('b') | cursor.next_is('B') { - // BinaryIntegerLiteral - - // Consume the 0b. - cursor.next(); - buf.pop(); - - kind = NumericKind::Integer(2); - } else if cursor.next_is('n') { - // DecimalBigIntegerLiteral '0n' - - // Consume the 'n' - cursor.next(); - - return Ok(Token::new( - TokenKind::NumericLiteral(Numeric::BigInt(0.into())), - Span::new(start_pos, cursor.pos()), - )); - } else { - let ch = cursor.peek(); - if ch.is_some() { - if ch.is_digit(8) { - // LegacyOctalIntegerLiteral - if self.strict_mode { - // LegacyOctalIntegerLiteral is forbidden with strict mode true. - return Err(Error::strict( - "Implicit octal literals are not allowed in strict mode.", - )); - } else { - cursor.next(); - buf.push(*ch); - - kind = NumericKind::Integer(8); - } - } else if ch.is_digit(10) { - // Indicates a numerical digit comes after then 0 but it isn't an octal digit - // so therefore this must be a number with an unneeded leading 0. This is - // forbidden in strict mode. - if self.strict_mode { - // LegacyOctalIntegerLiteral is forbidden with strict mode true. - return Err(Error::strict( - "Leading 0's are not allowed in strict mode.", - )); - } else { - cursor.next(); - buf.push(*ch); - } - } else { - // Indicates that the symbol is a non-number, this is valid if it is a dot or similar. - } - } - - - - } - - if ch.is_digit(8) { - // // LegacyOctalIntegerLiteral - // if self.strict_mode { - // // LegacyOctalIntegerLiteral is forbidden with strict mode true. - // return Err(Error::strict( - // "Implicit octal literals are not allowed in strict mode.", - // )); - // } else { - // cursor.next(); - // buf.push(*ch); - - // kind = NumericKind::Integer(8); - // } - } - } - - - - - // match peek_ch { - // None => { - // // DecimalLiteral lexing. - // // Indicates that the number is just a single 0. - // return Ok(Token::new( - // TokenKind::NumericLiteral(Numeric::Integer(0)), - // Span::new(start_pos, cursor.pos()), - // )); - // } - // Some(Err(e)) => { - // todo!(); - // // TODO - // } - // Some(Ok('x')) | Some(Ok('X')) => { - // // HexIntegerLiteral - - // cursor.next(); // Consume the 0x. - // buf.pop(); - - // kind = NumericKind::Integer(16); - // } - // Some(Ok('o')) | Some(Ok('O')) => { - // // OctalIntegerLiteral - - // cursor.next(); // Consume the 0o. - // buf.pop(); - - // kind = NumericKind::Integer(8); - // } - // Some(Ok('b')) | Some(Ok('B')) => { - // // BinaryIntegerLiteral - - // cursor.next(); // Consume the 0b. - // buf.pop(); - - // kind = NumericKind::Integer(2); - // } - // Some(Ok('n')) => { - // // DecimalBigIntegerLiteral '0n' - // return Ok(Token::new( - // TokenKind::NumericLiteral(Numeric::BigInt(0.into())), - // Span::new(start_pos, cursor.pos()), - // )); - // } - // Some(Ok(ch)) => { - // if ch.is_digit(8) { - // // LegacyOctalIntegerLiteral - // if self.strict_mode { - // // LegacyOctalIntegerLiteral is forbidden with strict mode true. - // return Err(Error::strict( - // "Implicit octal literals are not allowed in strict mode.", - // )); - // } else { - // cursor.next(); - // buf.push(*ch); - - // kind = NumericKind::Integer(8); - // } - // } else if ch.is_digit(10) { - // // Indicates a numerical digit comes after then 0 but it isn't an octal digit - // // so therefore this must be a number with an unneeded leading 0. This is - // // forbidden in strict mode. - // if self.strict_mode { - // // LegacyOctalIntegerLiteral is forbidden with strict mode true. - // return Err(Error::strict( - // "Leading 0's are not allowed in strict mode.", - // )); - // } else { - // cursor.next(); - // buf.push(*ch); - // } - // } else { - // // Indicates that the symbol is a non-number, this is valid if it is a dot or similar. - // } - // } - } - } - - println!("{:?}", cursor.peek()); - - // if let Some(ch) = c { - // buf.push(ch?); - // } - - loop { - if let Some(ch) = cursor.peek() { - match ch { - Err(_e) => { - // TODO, handle. - } - Ok(c) if c.is_digit(kind.base()) => { - let s = cursor.next().unwrap().unwrap(); - buf.push(s); - } - _ => { - // A non-number symbol detected, this might be a dot or similar. - break; - } - } - } - } - - // while let Some(ch) = cursor.peek() { - // match ch { - // Err(_e) => { - // // TODO, handle. - // } - // Ok(c) if c.is_digit(kind.base()) => { - // let s = cursor.next().unwrap().unwrap(); - // buf.push(s); - // } - // _ => { - // // A non-number symbol detected, this might be a dot or similar. - // break; - // } - // } - // } - - if cursor.next_is('n')? { - // DecimalBigIntegerLiteral - kind = kind.to_bigint(); - } - - if let NumericKind::Integer(10) = kind { - 'digitloop: while let Some(ch) = cursor.peek() { - match ch { - Err(_e) => { - // TODO - } - Ok('.') => loop { - kind = NumericKind::Rational; - match cursor.next() { - None => { - // Finished - break; - } - - Some(Err(e)) => { - return Err(Error::from(e)); - } - - Some(Ok(c)) => { - buf.push(c); - } - } - - match cursor.peek() { - None => { - break; - } - Some(Err(_e)) => { - // TODO - } - Some(Ok('e')) | Some(Ok('E')) => { - // TODO scientific notation. - - unimplemented!(); - - // match self - // .preview_multiple_next(2) - // .unwrap_or_default() - // .to_digit(10) - // { - // Some(0..=9) | None => { - // buf.push(self.next()); - // } - // _ => { - // break 'digitloop; - // } - // } - } - Some(Ok(cx)) if !cx.is_digit(10) => { - break 'digitloop; - } - _ => {} - } - }, - Ok('e') | Ok('E') => { - // TODO scientific notation. - unimplemented!(); - - // kind = NumericKind::Rational; - // match self - // .preview_multiple_next(2) - // .unwrap_or_default() - // .to_digit(10) - // { - // Some(0..=9) | None => { - // buf.push(self.next()); - // } - // _ => { - // break; - // } - // } - // buf.push(self.next()); - } - Ok('+') | Ok('-') => { - break; - } - Ok(cx) if cx.is_digit(10) => { - // cursor.next(); - match cursor.next() { - None => { - // Finished - break; - } - - Some(Err(e)) => { - return Err(Error::from(e)); - } - - Some(Ok(c)) => { - buf.push(c); - } - } - // buf.push(*cx); - } - Ok(_) => break, - } - } - } - - // TODO - //self.check_after_numeric_literal()?; - - let num = match kind { - NumericKind::BigInt(base) => { - Numeric::BigInt( - BigInt::from_string_radix(&buf, base as u32).expect("Could not convert to BigInt") - ) - } - NumericKind::Rational /* base: 10 */ => { - Numeric::Rational( - f64::from_str(&buf) - .map_err(|_| Error::syntax("Could not convert value to f64"))?, - ) - } - NumericKind::Integer(base) => { - if let Ok(num) = i32::from_str_radix(&buf, base as u32) { - Numeric::Integer( - num - ) - } else { - let b = f64::from(base); - let mut result = 0.0_f64; - for c in buf.chars() { - let digit = f64::from(c.to_digit(base as u32).unwrap()); - result = result * b + digit; - } - - Numeric::Rational(result) - } - - } - }; - - Ok(Token::new( - TokenKind::NumericLiteral(num), - Span::new(start_pos, cursor.pos()), - )) - } -} From f22f171f6c522acd38ea57dc677718ac3a4b8493 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 28 Jun 2020 23:16:40 +0100 Subject: [PATCH 090/289] Fmt --- boa/src/syntax/parser/cursor.rs | 15 ++++++--------- .../parser/expression/left_hand_side/call.rs | 1 - .../parser/statement/declaration/lexical.rs | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index f01691ee852..a16aa8f658a 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -238,15 +238,13 @@ where /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion pub(super) fn peek_semicolon(&mut self) -> Result<(bool, Option), ParseError> { match self.peek() { - Some(Ok(tk)) => { - match tk.kind() { - TokenKind::Punctuator(Punctuator::Semicolon) => Ok((true, Some(tk))), - TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { - Ok((true, Some(tk))) - } - _ => Ok((false, Some(tk))), + Some(Ok(tk)) => match tk.kind() { + TokenKind::Punctuator(Punctuator::Semicolon) => Ok((true, Some(tk))), + TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { + Ok((true, Some(tk))) } - } + _ => Ok((false, Some(tk))), + }, Some(Err(e)) => Err(e), None => Ok((true, None)), } @@ -261,7 +259,6 @@ where &mut self, context: &'static str, ) -> Result, ParseError> { - match self.peek_semicolon()? { (true, Some(tk)) => match tk.kind() { TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 1d513712212..24902916ca7 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -67,7 +67,6 @@ where let tk = cursor.peek(); - let mut lhs = match tk { Some(_) if tk.unwrap()?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 47263099b9c..9047f0b5b70 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -127,7 +127,7 @@ where let (ident, init) = LexicalBinding::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - + if self.is_const { if let Some(init) = init { const_decls.push(ConstDecl::new(ident, init)); From ac1d238cd8089cc5552f7a3cd50dbdf9730b1328 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 29 Jun 2020 00:28:52 +0100 Subject: [PATCH 091/289] Tidying --- boa/src/syntax/lexer/cursor.rs | 4 ++-- boa/src/syntax/parser/cursor.rs | 3 --- .../expression/assignment/arrow_function.rs | 2 +- boa/src/syntax/parser/mod.rs | 16 ---------------- .../syntax/parser/statement/iteration/tests.rs | 4 ++-- .../statement/iteration/while_statement.rs | 1 - 6 files changed, 5 insertions(+), 25 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index c2ed005324b..70620ba6525 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -129,8 +129,8 @@ where } else { match self.next() { None => { - unimplemented!(); - // unreachable!(); + // next_is_pred will return false if the next value is None so the None case should already be handled. + unreachable!(); } Some(Err(e)) => { return Err(e); diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index a16aa8f658a..77f32c70c62 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -8,9 +8,6 @@ use crate::syntax::lexer::{Token, TokenKind}; use std::collections::VecDeque; use std::io::Read; -/// The maximum number of values stored by the cursor to allow back(). -const BACK_QUEUE_MAX_LEN: usize = 3; - /// Token cursor. /// /// This internal structure gives basic testable operations to the parser. diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index b7682928ef2..73dc26839bf 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -19,7 +19,7 @@ use crate::{ error::{ErrorContext, ParseError, ParseResult}, function::{FormalParameters, FunctionBody}, statement::BindingIdentifier, - AllowAwait, AllowIn, AllowYield, Cursor, Token, TokenParser, + AllowAwait, AllowIn, AllowYield, Cursor, TokenParser, }, }, BoaProfiler, diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index f520a225d83..1c5ec933b71 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -10,8 +10,6 @@ mod tests; use self::error::{ParseError, ParseResult}; use crate::syntax::ast::node::StatementList; -use crate::syntax::ast::Node; -use crate::syntax::lexer::Token; use cursor::Cursor; @@ -114,20 +112,6 @@ impl Parser { { Script.parse(&mut self.cursor) } - - // Note these unimplemented methods may be removed before this parser refractor is finished. - - pub fn next(&mut self) -> Result { - unimplemented!(); - } - - pub fn peek(&mut self, amount: i32) -> Result { - unimplemented!(); - } - - pub fn expect(&mut self, token: Token, context_msg: &str) { - unimplemented!(); - } } /// Parses a full script. diff --git a/boa/src/syntax/parser/statement/iteration/tests.rs b/boa/src/syntax/parser/statement/iteration/tests.rs index 364ae147d6e..dae63693184 100644 --- a/boa/src/syntax/parser/statement/iteration/tests.rs +++ b/boa/src/syntax/parser/statement/iteration/tests.rs @@ -1,8 +1,8 @@ use crate::syntax::{ ast::{ node::{ - field::GetConstField, BinOp, Block, Break, Call, DoWhileLoop, Identifier, Node, - UnaryOp, VarDecl, VarDeclList, WhileLoop, + field::GetConstField, BinOp, Block, Break, Call, DoWhileLoop, Identifier, UnaryOp, + VarDecl, VarDeclList, WhileLoop, }, op::{self, AssignOp, CompOp}, Const, diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index 102ee5ba410..211afd6bc03 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -1,7 +1,6 @@ use crate::{ syntax::{ ast::{node::WhileLoop, Keyword, Punctuator}, - lexer::TokenKind::LineTerminator, parser::{ expression::Expression, statement::Statement, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser, From dde889312aa1442e27f6d2450b03efdda7e0e64b Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 29 Jun 2020 00:45:57 +0100 Subject: [PATCH 092/289] Starting to add goal symbol set statements --- boa/src/syntax/parser/cursor.rs | 12 +++++++++--- boa/src/syntax/parser/expression/primary/mod.rs | 9 ++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 77f32c70c62..2f758542d3b 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -1,9 +1,10 @@ //! Cursor implementation for the parser. use super::ParseError; -use crate::syntax::ast::Punctuator; -use crate::syntax::lexer::Lexer; -use crate::syntax::lexer::{Token, TokenKind}; +use crate::syntax::{ + ast::Punctuator, + lexer::{Lexer, Token, TokenKind, InputElement}, +}; use std::collections::VecDeque; use std::io::Read; @@ -40,6 +41,11 @@ where } } + /// Sets the goal symbol for the lexer. + pub(crate) fn set_goal(&mut self, elm: InputElement) { + self.lexer.set_goal(elm) + } + /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { match self.peeked.pop_front() { diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 7f2128f8373..751aa3bf660 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -18,7 +18,7 @@ use self::{ object_initializer::ObjectLiteral, }; use super::Expression; -use crate::syntax::lexer::{token::Numeric, TokenKind}; +use crate::syntax::lexer::{token::Numeric, TokenKind, InputElement}; use crate::syntax::{ ast::{ node::{Call, Identifier, New, Node}, @@ -101,13 +101,16 @@ where TokenKind::NumericLiteral(Numeric::Rational(num)) => Ok(Const::from(*num).into()), TokenKind::NumericLiteral(Numeric::BigInt(num)) => Ok(Const::from(num.clone()).into()), TokenKind::RegularExpressionLiteral(body, flags) => { - Ok(Node::from(New::from(Call::new( + cursor.set_goal(InputElement::RegExp); + let res = Ok(Node::from(New::from(Call::new( Identifier::from("RegExp"), vec![ Const::from(body.as_ref()).into(), Const::from(flags.to_string()).into(), ], - )))) + )))); + cursor.set_goal(InputElement::default()); + res } _ => Err(ParseError::unexpected(tok.clone(), "primary expression")), } From 7af447abd6b8cf2bef44bf6a2e4699ed082e1827 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 29 Jun 2020 18:19:07 +0100 Subject: [PATCH 093/289] Added goal symbol set to expression parser macro --- boa/src/lib.rs | 19 +++++ boa/src/syntax/lexer/mod.rs | 8 +- .../parser/expression/assignment/mod.rs | 2 +- .../parser/expression/left_hand_side/mod.rs | 5 +- boa/src/syntax/parser/expression/mod.rs | 77 +++++++++++++++---- .../syntax/parser/expression/primary/mod.rs | 8 +- 6 files changed, 97 insertions(+), 22 deletions(-) diff --git a/boa/src/lib.rs b/boa/src/lib.rs index 37da335d27c..54d858f6ab5 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -97,3 +97,22 @@ pub fn exec(src: &str) -> String { let mut engine = Interpreter::new(realm); forward(&mut engine, src) } + +#[test] +fn regex_func_arg() { + let realm = Realm::create(); + let mut engine = Interpreter::new(realm); + let init = r#"a = a.replace(/c(o)(o)(l)/, replacer);"#; + + let res = Parser::new(init.as_bytes()) + .parse_all() + .map_err(|e| format!("Parsing Error: {}", e)).unwrap(); + + println!("Result: {:?}", res); + + // assert_eq!(forward(&mut engine, "a"), "ecmascript is awesome!"); + + // assert_eq!(forward(&mut engine, "p1"), "o"); + // assert_eq!(forward(&mut engine, "p2"), "o"); + // assert_eq!(forward(&mut engine, "p3"), "l"); +} diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 1abe7427e52..26fe2bff6e5 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -209,14 +209,14 @@ where let result = TemplateLiteral::new().lex(&mut self.cursor, start); // A regex may follow a template literal but a DivPunctuator or TemplateSubstitutionTail may not. - self.set_goal(InputElement::RegExp); + // self.set_goal(InputElement::RegExp); result } _ if next_chr.is_digit(10) => { let result = NumberLiteral::new(next_chr, strict_mode).lex(&mut self.cursor, start); // A regex may not directly follow a NumericLiteral but a DivPunctuator may. // Note that the goal cannot be set to InputElementTemplateTail at this point as a TemplateSubstitutionTail would be invalid. - self.set_goal(InputElement::Div); + // self.set_goal(InputElement::Div); result } _ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => { @@ -224,7 +224,7 @@ where // A regex may not directly follow an Identifier but a DivPunctuator may. // Note that the goal cannot be set to InputElementTemplateTail at this point as a TemplateSubstitutionTail would be invalid. - self.set_goal(InputElement::Div); + // self.set_goal(InputElement::Div); result } ';' => Ok(Token::new( @@ -272,7 +272,7 @@ where '=' | '*' | '+' | '-' | '%' | '|' | '&' | '^' | '<' | '>' | '!' | '~' => { let result = Operator::new(next_chr).lex(&mut self.cursor, start); - self.set_goal(InputElement::RegExpOrTemplateTail); + // self.set_goal(InputElement::RegExpOrTemplateTail); result } diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 5f01555c96d..837aaf8613a 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -12,7 +12,7 @@ mod conditional; mod exponentiation; use self::{arrow_function::ArrowFunction, conditional::ConditionalExpression}; -use crate::syntax::lexer::TokenKind; +use crate::syntax::lexer::{TokenKind, InputElement}; use crate::{ syntax::{ ast::{ diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 7c4ef7be0bf..02117521ef1 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -13,7 +13,7 @@ mod member; use self::{call::CallExpression, member::MemberExpression}; use super::super::ParseError; -use crate::syntax::lexer::TokenKind; +use crate::syntax::lexer::{TokenKind, InputElement}; use crate::{ syntax::{ ast::{Node, Punctuator}, @@ -60,6 +60,9 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("LeftHandSIdeExpression", "Parsing"); + + // cursor.set_goal(InputElement::TemplateTail); + // TODO: Implement NewExpression: new MemberExpression let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; match cursor.peek() { diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 2eb3003b070..6dedd23a664 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -18,7 +18,7 @@ mod update; use self::assignment::ExponentiationExpression; pub(super) use self::{assignment::AssignmentExpression, primary::Initializer}; use super::{AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser}; -use crate::syntax::lexer::TokenKind; +use crate::syntax::lexer::{TokenKind, InputElement}; use crate::{ profiler::BoaProfiler, syntax::ast::{ @@ -52,7 +52,9 @@ impl PartialEq for Keyword { /// - The `$lower` identifier will contain the parser for lower level expressions. /// /// Those exressions are divided by the punctuators passed as the third parameter. -macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $low_param:ident ),*] ) => { +/// +/// The fifth parameter is an Option which sets the goal symbol to set before parsing (or None to leave it as is). +macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $low_param:ident ),*], $goal:expr ) => { impl TokenParser for $name where R: Read @@ -61,6 +63,10 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); + if $goal.is_some() { + cursor.set_goal($goal.unwrap()); + } + let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; while let Some(tok) = cursor.peek() { match tok?.kind() { @@ -123,7 +129,8 @@ expression!( Expression, AssignmentExpression, [Punctuator::Comma], - [allow_in, allow_yield, allow_await] + [allow_in, allow_yield, allow_await], + None:: ); /// Parses a logical `OR` expression. @@ -161,7 +168,8 @@ expression!( LogicalORExpression, LogicalANDExpression, [Punctuator::BoolOr], - [allow_in, allow_yield, allow_await] + [allow_in, allow_yield, allow_await], + None:: ); /// Parses a logical `AND` expression. @@ -199,7 +207,8 @@ expression!( LogicalANDExpression, BitwiseORExpression, [Punctuator::BoolAnd], - [allow_in, allow_yield, allow_await] + [allow_in, allow_yield, allow_await], + None:: ); /// Parses a bitwise `OR` expression. @@ -237,7 +246,8 @@ expression!( BitwiseORExpression, BitwiseXORExpression, [Punctuator::Or], - [allow_in, allow_yield, allow_await] + [allow_in, allow_yield, allow_await], + None:: ); /// Parses a bitwise `XOR` expression. @@ -275,7 +285,8 @@ expression!( BitwiseXORExpression, BitwiseANDExpression, [Punctuator::Xor], - [allow_in, allow_yield, allow_await] + [allow_in, allow_yield, allow_await], + None:: ); /// Parses a bitwise `AND` expression. @@ -313,7 +324,8 @@ expression!( BitwiseANDExpression, EqualityExpression, [Punctuator::And], - [allow_in, allow_yield, allow_await] + [allow_in, allow_yield, allow_await], + None:: ); /// Parses an equality expression. @@ -356,7 +368,8 @@ expression!( Punctuator::StrictEq, Punctuator::StrictNotEq ], - [allow_in, allow_yield, allow_await] + [allow_in, allow_yield, allow_await], + None:: ); /// Parses a relational expression. @@ -400,7 +413,8 @@ expression!( Punctuator::GreaterThanOrEq, Keyword::In ], - [allow_yield, allow_await] + [allow_yield, allow_await], + None:: ); /// Parses a bitwise shift expression. @@ -439,7 +453,8 @@ expression!( Punctuator::RightSh, Punctuator::URightSh ], - [allow_yield, allow_await] + [allow_yield, allow_await], + None:: ); /// Parses an additive expression. @@ -476,7 +491,8 @@ expression!( AdditiveExpression, MultiplicativeExpression, [Punctuator::Add, Punctuator::Sub], - [allow_yield, allow_await] + [allow_yield, allow_await], + None:: ); /// Parses a multiplicative expression. @@ -509,9 +525,44 @@ impl MultiplicativeExpression { } } +// impl TokenParser for MultiplicativeExpression +// where +// R: Read +// { +// type Output = Node; + +// fn parse(self, cursor: &mut Cursor) -> ParseResult { +// let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); +// let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; +// while let Some(tok) = cursor.peek() { +// match tok?.kind() { +// &TokenKind::Punctuator(op) if $( op == $op )||* => { +// let _ = cursor.next().expect("token disappeared"); +// lhs = BinOp::new( +// op.as_binop().expect("Could not get binary operation."), +// lhs, +// $lower::new($( self.$low_param ),*).parse(cursor)? +// ).into(); +// } +// &TokenKind::Keyword(op) if $( op == $op )||* => { +// let _ = cursor.next().expect("token disappeared"); +// lhs = BinOp::new( +// op.as_binop().expect("Could not get binary operation."), +// lhs, +// $lower::new($( self.$low_param ),*).parse(cursor)? +// ).into(); +// } +// _ => break +// } +// } +// Ok(lhs) +// } +// } + expression!( MultiplicativeExpression, ExponentiationExpression, [Punctuator::Mul, Punctuator::Div, Punctuator::Mod], - [allow_yield, allow_await] + [allow_yield, allow_await], + Some(InputElement::Div) ); diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 751aa3bf660..0f7d9c7db95 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -96,12 +96,15 @@ where } TokenKind::NullLiteral => Ok(Const::Null.into()), TokenKind::Identifier(ident) => Ok(Identifier::from(ident.as_ref()).into()), // TODO: IdentifierReference - TokenKind::StringLiteral(s) => Ok(Const::from(s.as_ref()).into()), + TokenKind::StringLiteral(s) => { + Ok(Const::from(s.as_ref()).into()) + + }, TokenKind::NumericLiteral(Numeric::Integer(num)) => Ok(Const::from(*num).into()), TokenKind::NumericLiteral(Numeric::Rational(num)) => Ok(Const::from(*num).into()), TokenKind::NumericLiteral(Numeric::BigInt(num)) => Ok(Const::from(num.clone()).into()), TokenKind::RegularExpressionLiteral(body, flags) => { - cursor.set_goal(InputElement::RegExp); + println!("Regex body: {:?}", body); let res = Ok(Node::from(New::from(Call::new( Identifier::from("RegExp"), vec![ @@ -109,7 +112,6 @@ where Const::from(flags.to_string()).into(), ], )))); - cursor.set_goal(InputElement::default()); res } _ => Err(ParseError::unexpected(tok.clone(), "primary expression")), From 03690ff03f5e41af5af6fad956f1d7e44ef06f5b Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 29 Jun 2020 19:55:22 +0100 Subject: [PATCH 094/289] Trying to workout where to put goal symbols --- boa/src/syntax/parser/cursor.rs | 6 +++++ .../parser/expression/assignment/mod.rs | 2 ++ boa/src/syntax/parser/expression/mod.rs | 24 +++++++++++-------- .../syntax/parser/expression/primary/mod.rs | 3 +++ boa/src/syntax/parser/function/mod.rs | 4 +++- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 2f758542d3b..b8b3429a36a 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -43,9 +43,15 @@ where /// Sets the goal symbol for the lexer. pub(crate) fn set_goal(&mut self, elm: InputElement) { + println!("Set goal: {:?}", elm); self.lexer.set_goal(elm) } + /// Gets the goal symbol for the lexer. + pub(crate) fn get_goal(&self) -> InputElement { + self.lexer.get_goal() + } + /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { match self.peeked.pop_front() { diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 837aaf8613a..b64d5f11467 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -80,6 +80,8 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); + cursor.set_goal(InputElement::RegExpOrTemplateTail); + // Arrow function match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { // a=>{} diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 6dedd23a664..538cab8b68e 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -63,6 +63,8 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); + // let old_goal = cursor.get_goal(); + if $goal.is_some() { cursor.set_goal($goal.unwrap()); } @@ -89,6 +91,8 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo _ => break } } + + // cursor.set_goal(old_goal); Ok(lhs) } } @@ -130,7 +134,7 @@ expression!( AssignmentExpression, [Punctuator::Comma], [allow_in, allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses a logical `OR` expression. @@ -169,7 +173,7 @@ expression!( LogicalANDExpression, [Punctuator::BoolOr], [allow_in, allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses a logical `AND` expression. @@ -208,7 +212,7 @@ expression!( BitwiseORExpression, [Punctuator::BoolAnd], [allow_in, allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses a bitwise `OR` expression. @@ -247,7 +251,7 @@ expression!( BitwiseXORExpression, [Punctuator::Or], [allow_in, allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses a bitwise `XOR` expression. @@ -286,7 +290,7 @@ expression!( BitwiseANDExpression, [Punctuator::Xor], [allow_in, allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses a bitwise `AND` expression. @@ -325,7 +329,7 @@ expression!( EqualityExpression, [Punctuator::And], [allow_in, allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses an equality expression. @@ -369,7 +373,7 @@ expression!( Punctuator::StrictNotEq ], [allow_in, allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses a relational expression. @@ -414,7 +418,7 @@ expression!( Keyword::In ], [allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses a bitwise shift expression. @@ -454,7 +458,7 @@ expression!( Punctuator::URightSh ], [allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses an additive expression. @@ -492,7 +496,7 @@ expression!( MultiplicativeExpression, [Punctuator::Add, Punctuator::Sub], [allow_yield, allow_await], - None:: + Some(InputElement::Div) ); /// Parses a multiplicative expression. diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 0f7d9c7db95..bff6d4b6ff3 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -74,17 +74,20 @@ where FunctionExpression.parse(cursor).map(Node::from) } TokenKind::Punctuator(Punctuator::OpenParen) => { + cursor.set_goal(InputElement::RegExp); let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; cursor.expect(Punctuator::CloseParen, "primary expression")?; Ok(expr) } TokenKind::Punctuator(Punctuator::OpenBracket) => { + cursor.set_goal(InputElement::RegExp); ArrayLiteral::new(self.allow_yield, self.allow_await) .parse(cursor) .map(Node::ArrayDecl) } TokenKind::Punctuator(Punctuator::OpenBlock) => { + cursor.set_goal(InputElement::RegExp); Ok(ObjectLiteral::new(self.allow_yield, self.allow_await) .parse(cursor)? .into()) diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 887a6b331d0..7c6ec2e555e 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -10,7 +10,7 @@ #[cfg(test)] mod tests; -use crate::syntax::lexer::TokenKind; +use crate::syntax::lexer::{TokenKind, InputElement}; use crate::syntax::{ ast::{ node::{self}, @@ -60,6 +60,8 @@ where type Output = Box<[node::FormalParameter]>; fn parse(self, cursor: &mut Cursor) -> Result { + cursor.set_goal(InputElement::RegExp); + let mut params = Vec::new(); if cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() From 40db5f7343816f3043b5b536a095a56899361429 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 13:32:21 +0100 Subject: [PATCH 095/289] Added a 'fallback' mechanism to try and regex if a div is found unexpectadly - all regex tests currently pass --- boa/src/lib.rs | 5 +++- boa/src/syntax/lexer/mod.rs | 5 ++-- boa/src/syntax/lexer/tests.rs | 24 ---------------- boa/src/syntax/parser/cursor.rs | 8 +++++- .../parser/expression/assignment/mod.rs | 9 +++++- .../expression/left_hand_side/arguments.rs | 2 ++ .../parser/expression/left_hand_side/mod.rs | 2 +- boa/src/syntax/parser/expression/mod.rs | 21 +++++++------- .../syntax/parser/expression/primary/mod.rs | 28 +++++++++++++++++++ boa/src/syntax/parser/expression/tests.rs | 8 ++++++ boa/src/syntax/parser/tests.rs | 23 +++++++++++++++ 11 files changed, 95 insertions(+), 40 deletions(-) diff --git a/boa/src/lib.rs b/boa/src/lib.rs index 54d858f6ab5..e7a01c8365e 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -98,17 +98,20 @@ pub fn exec(src: &str) -> String { forward(&mut engine, src) } +// Temporary test for debugging goal symbols. #[test] fn regex_func_arg() { let realm = Realm::create(); let mut engine = Interpreter::new(realm); let init = r#"a = a.replace(/c(o)(o)(l)/, replacer);"#; + // forward(&mut engine, init); + let res = Parser::new(init.as_bytes()) .parse_all() .map_err(|e| format!("Parsing Error: {}", e)).unwrap(); - println!("Result: {:?}", res); + // println!("Result: {:?}", res); // assert_eq!(forward(&mut engine, "a"), "ecmascript is awesome!"); diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 26fe2bff6e5..b43d1d61c71 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -43,7 +43,8 @@ use self::{ string::StringLiteral, template::TemplateLiteral, }; -use crate::syntax::ast::{Position, Punctuator, Span}; +pub use crate::syntax::ast::Position; +use crate::syntax::ast::{Punctuator, Span}; use std::io::Read; pub use token::{Token, TokenKind}; @@ -107,7 +108,7 @@ impl Lexer { // that means it could be multiple different tokens depending on the input token. // // As per https://tc39.es/ecma262/#sec-ecmascript-language-lexical-grammar - fn lex_slash_token(&mut self, start: Position) -> Result + pub(crate) fn lex_slash_token(&mut self, start: Position) -> Result where R: Read, { diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index dad921cb34b..f496d6986e8 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -290,30 +290,6 @@ fn check_positions_codepoint() { ); } -#[test] -fn two_divisions_in_expression() { - let s = " return a !== 0 || 1 / a === 1 / b;"; - let mut lexer = Lexer::new(s.as_bytes()); - - let expected = [ - TokenKind::Keyword(Keyword::Return), - TokenKind::Identifier("a".into()), - TokenKind::Punctuator(Punctuator::StrictNotEq), - TokenKind::NumericLiteral(Numeric::Integer(0)), - TokenKind::Punctuator(Punctuator::BoolOr), - TokenKind::NumericLiteral(Numeric::Integer(1)), - TokenKind::Punctuator(Punctuator::Div), - TokenKind::Identifier("a".into()), - TokenKind::Punctuator(Punctuator::StrictEq), - TokenKind::NumericLiteral(Numeric::Integer(1)), - TokenKind::Punctuator(Punctuator::Div), - TokenKind::Identifier("b".into()), - TokenKind::Punctuator(Punctuator::Semicolon), - ]; - - expect_tokens(&mut lexer, &expected); -} - #[test] fn check_line_numbers() { let s = "x\ny\n"; diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index b8b3429a36a..1c2dddff8f1 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -3,7 +3,7 @@ use super::ParseError; use crate::syntax::{ ast::Punctuator, - lexer::{Lexer, Token, TokenKind, InputElement}, + lexer::{Lexer, Token, TokenKind, InputElement, Error as LexerError, Position}, }; use std::collections::VecDeque; @@ -52,6 +52,12 @@ where self.lexer.get_goal() } + // Somewhat a hack. + pub(super) fn lex_regex(&mut self, start: Position) -> Result { + self.set_goal(InputElement::RegExp); + self.lexer.lex_slash_token(start).map_err(|e| ParseError::lex(e)) + } + /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { match self.peeked.pop_front() { diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index b64d5f11467..7ac1da600a7 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -80,7 +80,8 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); - cursor.set_goal(InputElement::RegExpOrTemplateTail); + // cursor.set_goal(InputElement::RegExpOrTemplateTail); + // cursor.set_goal(InputElement::Div); // Arrow function match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { @@ -117,9 +118,15 @@ where _ => {} } + println!("Cursor peek: {:?}", cursor.peek()); + cursor.set_goal(InputElement::Div); + let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; + println!("LHS: {:?}", lhs); + println!("Cursor peek: {:?}", cursor.peek()); + // Here if let Some(tok) = cursor.peek() { match tok?.kind() { TokenKind::Punctuator(Punctuator::Assign) => { diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index 84dae149ccf..6e401eaedd1 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -15,6 +15,7 @@ use crate::{ expression::AssignmentExpression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, }, + lexer::InputElement }, BoaProfiler, }; @@ -103,6 +104,7 @@ where .into(), ); } else { + cursor.set_goal(InputElement::RegExp); args.push( AssignmentExpression::new(true, self.allow_yield, self.allow_await) .parse(cursor)?, diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 02117521ef1..7160930fd02 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -61,7 +61,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("LeftHandSIdeExpression", "Parsing"); - // cursor.set_goal(InputElement::TemplateTail); + cursor.set_goal(InputElement::TemplateTail); // TODO: Implement NewExpression: new MemberExpression let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 538cab8b68e..64157a00fd8 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -70,6 +70,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo } let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; + println!("LHS in EXP: {:?}", lhs); while let Some(tok) = cursor.peek() { match tok?.kind() { &TokenKind::Punctuator(op) if $( op == $op )||* => { @@ -134,7 +135,7 @@ expression!( AssignmentExpression, [Punctuator::Comma], [allow_in, allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses a logical `OR` expression. @@ -173,7 +174,7 @@ expression!( LogicalANDExpression, [Punctuator::BoolOr], [allow_in, allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses a logical `AND` expression. @@ -212,7 +213,7 @@ expression!( BitwiseORExpression, [Punctuator::BoolAnd], [allow_in, allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses a bitwise `OR` expression. @@ -251,7 +252,7 @@ expression!( BitwiseXORExpression, [Punctuator::Or], [allow_in, allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses a bitwise `XOR` expression. @@ -290,7 +291,7 @@ expression!( BitwiseANDExpression, [Punctuator::Xor], [allow_in, allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses a bitwise `AND` expression. @@ -329,7 +330,7 @@ expression!( EqualityExpression, [Punctuator::And], [allow_in, allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses an equality expression. @@ -373,7 +374,7 @@ expression!( Punctuator::StrictNotEq ], [allow_in, allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses a relational expression. @@ -418,7 +419,7 @@ expression!( Keyword::In ], [allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses a bitwise shift expression. @@ -458,7 +459,7 @@ expression!( Punctuator::URightSh ], [allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses an additive expression. @@ -496,7 +497,7 @@ expression!( MultiplicativeExpression, [Punctuator::Add, Punctuator::Sub], [allow_yield, allow_await], - Some(InputElement::Div) + None:: ); /// Parses a multiplicative expression. diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index bff6d4b6ff3..fdc301ee0f2 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -117,6 +117,34 @@ where )))); res } + TokenKind::Punctuator(Punctuator::Div) => { + // This is where the start of a regexp is accidentally treated as a div + + // Try parsing as a regexp. + + let tok = cursor.lex_regex(tok.span().start())?; + + if let TokenKind::RegularExpressionLiteral(body, flags) = tok.kind() { + println!("Regex body: {:?}", body); + let res = Ok(Node::from(New::from(Call::new( + Identifier::from("RegExp"), + vec![ + Const::from(body.as_ref()).into(), + Const::from(flags.to_string()).into(), + ], + )))); + res + } else { + // A regex was expected and nothing else. + unimplemented!("How to handle this case?"); + } + + + // println!("{:?}", res); + + // unimplemented!("This is where the start of a regexp is accidentally treated as a div"); + + } _ => Err(ParseError::unexpected(tok.clone(), "primary expression")), } } diff --git a/boa/src/syntax/parser/expression/tests.rs b/boa/src/syntax/parser/expression/tests.rs index 69816e4dff3..f2ba5dab7e4 100644 --- a/boa/src/syntax/parser/expression/tests.rs +++ b/boa/src/syntax/parser/expression/tests.rs @@ -7,6 +7,14 @@ use crate::syntax::{ parser::tests::check_parser, }; +#[test] +fn check_divide() { + check_parser( + "a / b", + vec![BinOp::new(NumOp::Div, Identifier::from("a"), Identifier::from("b")).into()], + ); +} + /// Checks numeric operations #[test] fn check_numeric_operations() { diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 8de593d9458..4f5d87d49da 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -95,3 +95,26 @@ fn hoisting() { ], ); } + +// #[test] +// fn two_divisions_in_expression() { +// let s = " return a !== 0 || 1 / a === 1 / b;"; + +// let expected = vec![ +// TokenKind::Keyword(Keyword::Return), +// TokenKind::Identifier("a".into()), +// TokenKind::Punctuator(Punctuator::StrictNotEq), +// TokenKind::NumericLiteral(Numeric::Integer(0)), +// TokenKind::Punctuator(Punctuator::BoolOr), +// TokenKind::NumericLiteral(Numeric::Integer(1)), +// TokenKind::Punctuator(Punctuator::Div), +// TokenKind::Identifier("a".into()), +// TokenKind::Punctuator(Punctuator::StrictEq), +// TokenKind::NumericLiteral(Numeric::Integer(1)), +// TokenKind::Punctuator(Punctuator::Div), +// TokenKind::Identifier("b".into()), +// TokenKind::Punctuator(Punctuator::Semicolon), +// ]; + +// check_parser(s, expected); +// } \ No newline at end of file From 4657f187878537f6ea239ffadebb42fd2ef16421 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 13:45:10 +0100 Subject: [PATCH 096/289] All tests pass --- boa/src/syntax/parser/expression/assignment/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 7ac1da600a7..85997c24882 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -81,7 +81,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); // cursor.set_goal(InputElement::RegExpOrTemplateTail); - // cursor.set_goal(InputElement::Div); + cursor.set_goal(InputElement::Div); // Arrow function match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { From d7401643ff8c70a3176aceefe2db22fed99d9c90 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 13:50:03 +0100 Subject: [PATCH 097/289] Removed println statements --- boa/src/lib.rs | 22 ------------------- boa/src/syntax/parser/cursor.rs | 8 +------ .../parser/expression/assignment/mod.rs | 4 ---- boa/src/syntax/parser/expression/mod.rs | 1 - .../syntax/parser/expression/primary/mod.rs | 8 ------- boa/src/syntax/parser/expression/tests.rs | 8 ------- 6 files changed, 1 insertion(+), 50 deletions(-) diff --git a/boa/src/lib.rs b/boa/src/lib.rs index e7a01c8365e..37da335d27c 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -97,25 +97,3 @@ pub fn exec(src: &str) -> String { let mut engine = Interpreter::new(realm); forward(&mut engine, src) } - -// Temporary test for debugging goal symbols. -#[test] -fn regex_func_arg() { - let realm = Realm::create(); - let mut engine = Interpreter::new(realm); - let init = r#"a = a.replace(/c(o)(o)(l)/, replacer);"#; - - // forward(&mut engine, init); - - let res = Parser::new(init.as_bytes()) - .parse_all() - .map_err(|e| format!("Parsing Error: {}", e)).unwrap(); - - // println!("Result: {:?}", res); - - // assert_eq!(forward(&mut engine, "a"), "ecmascript is awesome!"); - - // assert_eq!(forward(&mut engine, "p1"), "o"); - // assert_eq!(forward(&mut engine, "p2"), "o"); - // assert_eq!(forward(&mut engine, "p3"), "l"); -} diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 1c2dddff8f1..6c5e1720693 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -43,16 +43,10 @@ where /// Sets the goal symbol for the lexer. pub(crate) fn set_goal(&mut self, elm: InputElement) { - println!("Set goal: {:?}", elm); self.lexer.set_goal(elm) } - /// Gets the goal symbol for the lexer. - pub(crate) fn get_goal(&self) -> InputElement { - self.lexer.get_goal() - } - - // Somewhat a hack. + /// Lexes the next tokens as a regex assuming that the starting '/' has already been consumed. pub(super) fn lex_regex(&mut self, start: Position) -> Result { self.set_goal(InputElement::RegExp); self.lexer.lex_slash_token(start).map_err(|e| ParseError::lex(e)) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 85997c24882..b2f01fe971d 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -118,15 +118,11 @@ where _ => {} } - println!("Cursor peek: {:?}", cursor.peek()); cursor.set_goal(InputElement::Div); let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - println!("LHS: {:?}", lhs); - println!("Cursor peek: {:?}", cursor.peek()); - // Here if let Some(tok) = cursor.peek() { match tok?.kind() { TokenKind::Punctuator(Punctuator::Assign) => { diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 64157a00fd8..b9e1d20d61a 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -70,7 +70,6 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo } let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; - println!("LHS in EXP: {:?}", lhs); while let Some(tok) = cursor.peek() { match tok?.kind() { &TokenKind::Punctuator(op) if $( op == $op )||* => { diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index fdc301ee0f2..12a33b0ad34 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -107,7 +107,6 @@ where TokenKind::NumericLiteral(Numeric::Rational(num)) => Ok(Const::from(*num).into()), TokenKind::NumericLiteral(Numeric::BigInt(num)) => Ok(Const::from(num.clone()).into()), TokenKind::RegularExpressionLiteral(body, flags) => { - println!("Regex body: {:?}", body); let res = Ok(Node::from(New::from(Call::new( Identifier::from("RegExp"), vec![ @@ -125,7 +124,6 @@ where let tok = cursor.lex_regex(tok.span().start())?; if let TokenKind::RegularExpressionLiteral(body, flags) = tok.kind() { - println!("Regex body: {:?}", body); let res = Ok(Node::from(New::from(Call::new( Identifier::from("RegExp"), vec![ @@ -138,12 +136,6 @@ where // A regex was expected and nothing else. unimplemented!("How to handle this case?"); } - - - // println!("{:?}", res); - - // unimplemented!("This is where the start of a regexp is accidentally treated as a div"); - } _ => Err(ParseError::unexpected(tok.clone(), "primary expression")), } diff --git a/boa/src/syntax/parser/expression/tests.rs b/boa/src/syntax/parser/expression/tests.rs index f2ba5dab7e4..69816e4dff3 100644 --- a/boa/src/syntax/parser/expression/tests.rs +++ b/boa/src/syntax/parser/expression/tests.rs @@ -7,14 +7,6 @@ use crate::syntax::{ parser::tests::check_parser, }; -#[test] -fn check_divide() { - check_parser( - "a / b", - vec![BinOp::new(NumOp::Div, Identifier::from("a"), Identifier::from("b")).into()], - ); -} - /// Checks numeric operations #[test] fn check_numeric_operations() { From 60004cfabdea0488ab2b3b607c9832fda3103d36 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 14:11:34 +0100 Subject: [PATCH 098/289] Added tests showing handling of ambigious regex/div case --- boa/src/syntax/parser/tests.rs | 53 ++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 4f5d87d49da..da813cf0715 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -6,7 +6,7 @@ use crate::syntax::ast::{ field::GetConstField, Assign, BinOp, Call, FunctionDecl, Identifier, New, Node, Return, StatementList, UnaryOp, VarDecl, VarDeclList, }, - op::{self, NumOp}, + op::{self, NumOp, CompOp, LogOp}, Const, }; @@ -96,25 +96,36 @@ fn hoisting() { ); } -// #[test] -// fn two_divisions_in_expression() { -// let s = " return a !== 0 || 1 / a === 1 / b;"; +#[test] +fn ambigous_regex_divide_expression() { + let s = "1 / a === 1 / b"; + + check_parser(s, vec![ + BinOp::new( + CompOp::StrictEqual, + BinOp::new(NumOp::Div, Const::Int(1), Identifier::from("a")), + BinOp::new(NumOp::Div, Const::Int(1), Identifier::from("b")), + ).into() + ]); +} -// let expected = vec![ -// TokenKind::Keyword(Keyword::Return), -// TokenKind::Identifier("a".into()), -// TokenKind::Punctuator(Punctuator::StrictNotEq), -// TokenKind::NumericLiteral(Numeric::Integer(0)), -// TokenKind::Punctuator(Punctuator::BoolOr), -// TokenKind::NumericLiteral(Numeric::Integer(1)), -// TokenKind::Punctuator(Punctuator::Div), -// TokenKind::Identifier("a".into()), -// TokenKind::Punctuator(Punctuator::StrictEq), -// TokenKind::NumericLiteral(Numeric::Integer(1)), -// TokenKind::Punctuator(Punctuator::Div), -// TokenKind::Identifier("b".into()), -// TokenKind::Punctuator(Punctuator::Semicolon), -// ]; +#[test] +fn two_divisions_in_expression() { + let s = "a !== 0 || 1 / a === 1 / b;"; -// check_parser(s, expected); -// } \ No newline at end of file + check_parser(s, vec![ + BinOp::new( + LogOp::Or, + BinOp::new( + CompOp::StrictNotEqual, + Identifier::from("a"), + Const::Int(0) + ), + BinOp::new( + CompOp::StrictEqual, + BinOp::new(NumOp::Div, Const::Int(1), Identifier::from("a")), + BinOp::new(NumOp::Div, Const::Int(1), Identifier::from("b")), + ) + ).into() + ]); +} \ No newline at end of file From 8342712fbdfe0ea459e94265496a565cc81b0e09 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 14:11:57 +0100 Subject: [PATCH 099/289] Fmt --- boa/src/syntax/parser/cursor.rs | 6 ++-- .../parser/expression/assignment/mod.rs | 2 +- .../expression/left_hand_side/arguments.rs | 2 +- .../parser/expression/left_hand_side/mod.rs | 2 +- boa/src/syntax/parser/expression/mod.rs | 2 +- .../syntax/parser/expression/primary/mod.rs | 7 ++-- boa/src/syntax/parser/function/mod.rs | 2 +- boa/src/syntax/parser/tests.rs | 32 +++++++++---------- 8 files changed, 27 insertions(+), 28 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 6c5e1720693..370b66e2eec 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -3,7 +3,7 @@ use super::ParseError; use crate::syntax::{ ast::Punctuator, - lexer::{Lexer, Token, TokenKind, InputElement, Error as LexerError, Position}, + lexer::{Error as LexerError, InputElement, Lexer, Position, Token, TokenKind}, }; use std::collections::VecDeque; @@ -49,7 +49,9 @@ where /// Lexes the next tokens as a regex assuming that the starting '/' has already been consumed. pub(super) fn lex_regex(&mut self, start: Position) -> Result { self.set_goal(InputElement::RegExp); - self.lexer.lex_slash_token(start).map_err(|e| ParseError::lex(e)) + self.lexer + .lex_slash_token(start) + .map_err(|e| ParseError::lex(e)) } /// Moves the cursor to the next token and returns the token. diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index b2f01fe971d..479bfd00819 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -12,7 +12,7 @@ mod conditional; mod exponentiation; use self::{arrow_function::ArrowFunction, conditional::ConditionalExpression}; -use crate::syntax::lexer::{TokenKind, InputElement}; +use crate::syntax::lexer::{InputElement, TokenKind}; use crate::{ syntax::{ ast::{ diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index 6e401eaedd1..e4aca6a61ba 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -11,11 +11,11 @@ use crate::syntax::lexer::TokenKind; use crate::{ syntax::{ ast::{node::Spread, Node, Punctuator}, + lexer::InputElement, parser::{ expression::AssignmentExpression, AllowAwait, AllowYield, Cursor, ParseError, TokenParser, }, - lexer::InputElement }, BoaProfiler, }; diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 7160930fd02..ea96487066d 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -13,7 +13,7 @@ mod member; use self::{call::CallExpression, member::MemberExpression}; use super::super::ParseError; -use crate::syntax::lexer::{TokenKind, InputElement}; +use crate::syntax::lexer::{InputElement, TokenKind}; use crate::{ syntax::{ ast::{Node, Punctuator}, diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index b9e1d20d61a..9db71e183ac 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -18,7 +18,7 @@ mod update; use self::assignment::ExponentiationExpression; pub(super) use self::{assignment::AssignmentExpression, primary::Initializer}; use super::{AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser}; -use crate::syntax::lexer::{TokenKind, InputElement}; +use crate::syntax::lexer::{InputElement, TokenKind}; use crate::{ profiler::BoaProfiler, syntax::ast::{ diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 12a33b0ad34..4be0ab23c38 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -18,7 +18,7 @@ use self::{ object_initializer::ObjectLiteral, }; use super::Expression; -use crate::syntax::lexer::{token::Numeric, TokenKind, InputElement}; +use crate::syntax::lexer::{token::Numeric, InputElement, TokenKind}; use crate::syntax::{ ast::{ node::{Call, Identifier, New, Node}, @@ -99,10 +99,7 @@ where } TokenKind::NullLiteral => Ok(Const::Null.into()), TokenKind::Identifier(ident) => Ok(Identifier::from(ident.as_ref()).into()), // TODO: IdentifierReference - TokenKind::StringLiteral(s) => { - Ok(Const::from(s.as_ref()).into()) - - }, + TokenKind::StringLiteral(s) => Ok(Const::from(s.as_ref()).into()), TokenKind::NumericLiteral(Numeric::Integer(num)) => Ok(Const::from(*num).into()), TokenKind::NumericLiteral(Numeric::Rational(num)) => Ok(Const::from(*num).into()), TokenKind::NumericLiteral(Numeric::BigInt(num)) => Ok(Const::from(num.clone()).into()), diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index 7c6ec2e555e..edda574a734 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -10,7 +10,7 @@ #[cfg(test)] mod tests; -use crate::syntax::lexer::{TokenKind, InputElement}; +use crate::syntax::lexer::{InputElement, TokenKind}; use crate::syntax::{ ast::{ node::{self}, diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index da813cf0715..f3d949c4a5b 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -6,7 +6,7 @@ use crate::syntax::ast::{ field::GetConstField, Assign, BinOp, Call, FunctionDecl, Identifier, New, Node, Return, StatementList, UnaryOp, VarDecl, VarDeclList, }, - op::{self, NumOp, CompOp, LogOp}, + op::{self, CompOp, LogOp, NumOp}, Const, }; @@ -100,32 +100,32 @@ fn hoisting() { fn ambigous_regex_divide_expression() { let s = "1 / a === 1 / b"; - check_parser(s, vec![ - BinOp::new( + check_parser( + s, + vec![BinOp::new( CompOp::StrictEqual, BinOp::new(NumOp::Div, Const::Int(1), Identifier::from("a")), BinOp::new(NumOp::Div, Const::Int(1), Identifier::from("b")), - ).into() - ]); + ) + .into()], + ); } #[test] fn two_divisions_in_expression() { let s = "a !== 0 || 1 / a === 1 / b;"; - check_parser(s, vec![ - BinOp::new( + check_parser( + s, + vec![BinOp::new( LogOp::Or, - BinOp::new( - CompOp::StrictNotEqual, - Identifier::from("a"), - Const::Int(0) - ), + BinOp::new(CompOp::StrictNotEqual, Identifier::from("a"), Const::Int(0)), BinOp::new( CompOp::StrictEqual, BinOp::new(NumOp::Div, Const::Int(1), Identifier::from("a")), BinOp::new(NumOp::Div, Const::Int(1), Identifier::from("b")), - ) - ).into() - ]); -} \ No newline at end of file + ), + ) + .into()], + ); +} From 98a6faa2486805c539d32af01c036d940816e53e Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 14:46:08 +0100 Subject: [PATCH 100/289] Disabled the expression lexer benchmark --- boa/benches/lexer.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/boa/benches/lexer.rs b/boa/benches/lexer.rs index 905de9ed198..df0e38f845c 100644 --- a/boa/benches/lexer.rs +++ b/boa/benches/lexer.rs @@ -62,5 +62,7 @@ fn for_loop_lexer(c: &mut Criterion) { }); } -criterion_group!(lexer, expression_lexer, hello_world_lexer, for_loop_lexer); +criterion_group!(lexer, + // expression_lexer, + hello_world_lexer, for_loop_lexer); criterion_main!(lexer); From d501b09b3af1999f5652664c612d87e0d1382f8a Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 15:23:15 +0100 Subject: [PATCH 101/289] Downgraded some cursor pub fn to fn --- boa/src/syntax/lexer/comment.rs | 4 - boa/src/syntax/lexer/cursor.rs | 4 +- boa/src/syntax/lexer/lexer_old.rs | 872 ------------------------------ boa/src/syntax/lexer/mod.rs | 2 +- boa/src/syntax/lexer/template.rs | 7 - 5 files changed, 3 insertions(+), 886 deletions(-) delete mode 100644 boa/src/syntax/lexer/lexer_old.rs diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index a9aa208326d..d0578348064 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -32,7 +32,6 @@ impl Tokenizer for SingleLineComment { _ => {} } } - cursor.next_line(); Ok(Token::new( TokenKind::Comment, Span::new(start_pos, cursor.pos()), @@ -62,9 +61,6 @@ impl Tokenizer for BlockComment { Err(e) => { return Err(Error::IO(e)); } - Ok('\n') => { - cursor.next_line(); - } Ok('*') => { if cursor.next_is('/')? { break; diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 70620ba6525..f310d0a8b68 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -25,14 +25,14 @@ impl Cursor { /// Advances the position to the next line. #[inline] - pub(super) fn next_line(&mut self) { + fn next_line(&mut self) { let next_line = self.pos.line_number() + 1; self.pos = Position::new(next_line, 1); } /// Performs a carriage return to modify the position in the source. #[inline] - pub(super) fn carriage_return(&mut self) { + fn carriage_return(&mut self) { let current_line = self.pos.line_number(); self.pos = Position::new(current_line, 1); } diff --git a/boa/src/syntax/lexer/lexer_old.rs b/boa/src/syntax/lexer/lexer_old.rs deleted file mode 100644 index 9e426e1db3c..00000000000 --- a/boa/src/syntax/lexer/lexer_old.rs +++ /dev/null @@ -1,872 +0,0 @@ -//! A lexical analyzer for JavaScript source code. -//! -//! The Lexer splits its input source code into a sequence of input elements called tokens, represented by the [Token](../ast/token/struct.Token.html) structure. -//! It also removes whitespace and comments and attaches them to the next token. - -#[cfg(test)] -mod tests; - -use crate::builtins::BigInt; -use crate::{ - syntax::ast::{ - token::{NumericLiteral, Token, TokenKind}, - Position, Punctuator, Span, - }, - BoaProfiler, -}; -use std::{ - char::{decode_utf16, from_u32}, - error, fmt, - iter::Peekable, - str::{Chars, FromStr}, -}; - -/// `vop` tests the next token to see if we're on an assign operation of just a plain binary operation. -/// -/// If the next value is not an assignment operation it will pattern match the provided values and return the corresponding token. -macro_rules! vop { - ($this:ident, $assign_op:expr, $op:expr) => ({ - let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; - match preview { - '=' => { - $this.next(); - $this.next_column(); - $assign_op - } - _ => $op, - } - }); - ($this:ident, $assign_op:expr, $op:expr, {$($case:pat => $block:expr), +}) => ({ - let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; - match preview { - '=' => { - $this.next(); - $this.next_column(); - $assign_op - }, - $($case => { - $this.next(); - $this.next_column(); - $block - })+, - _ => $op - } - }); - ($this:ident, $op:expr, {$($case:pat => $block:expr),+}) => { - let preview = $this.preview_next().ok_or_else(|| LexerError::new("could not preview next value"))?; - match preview { - $($case => { - $this.next()?; - $this.next_column(); - $block - })+, - _ => $op - } - } -} - -/// The `op` macro handles binary operations or assignment operations and converts them into tokens. -macro_rules! op { - ($this:ident, $start_pos:expr, $assign_op:expr, $op:expr) => ({ - let punc = vop!($this, $assign_op, $op); - $this.push_punc(punc, $start_pos); - }); - ($this:ident, $start_pos:expr, $assign_op:expr, $op:expr, {$($case:pat => $block:expr),+}) => ({ - let punc = vop!($this, $assign_op, $op, {$($case => $block),+}); - $this.push_punc(punc, $start_pos); - }); -} - -/// An error that occurred during lexing or compiling of the source input. -/// -/// [LexerError] implements [fmt::Display] so you just display this value as an error -#[derive(Debug, Clone)] -pub struct LexerError { - /// details will be displayed when a LexerError occurs. - details: String, -} - -impl LexerError { - /// Create a new LexerError struct - /// - /// * `msg` - The message to show when LexerError is displayed - pub(crate) fn new(msg: M) -> Self - where - M: Into, - { - Self { - details: msg.into(), - } - } -} - -impl fmt::Display for LexerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.details) - } -} - -impl error::Error for LexerError { - fn description(&self) -> &str { - &self.details - } - - fn cause(&self) -> Option<&dyn error::Error> { - // Generic error, underlying cause isn't tracked. - None - } -} - -/// A lexical analyzer for JavaScript source code. -#[derive(Debug)] -pub struct Lexer<'a> { - /// The list of tokens generated so far. - /// - /// This field is public so you can use them once lexing has finished. - pub tokens: Vec, - /// The current position in the source code. - position: Position, - /// The full Peekable buffer, an array of [Char]s - buffer: Peekable>, -} - -impl<'a> Lexer<'a> { - /// Returns a Lexer with a buffer inside - /// - /// The buffer needs to have a lifetime as long as the Lexer instance itself - pub fn new(buffer: &'a str) -> Lexer<'a> { - Lexer { - tokens: Vec::new(), - position: Position::new(1, 1), - buffer: buffer.chars().peekable(), - } - } - - /// Push a token onto the token queue. - fn push_token(&mut self, tk: TokenKind, start: Position) { - let end = if let TokenKind::LineTerminator = tk { - self.position - } else { - Position::new( - self.position.line_number(), - self.position.column_number() - 1, - ) - }; - self.tokens.push(Token::new(tk, Span::new(start, end))) - } - - /// Push a punctuation token - fn push_punc(&mut self, punc: Punctuator, start: Position) { - self.push_token(TokenKind::Punctuator(punc), start); - } - - /// Changes the current position by advancing to the next column. - fn next_column(&mut self) { - let pos = Position::new( - self.position.line_number(), - self.position.column_number() + 1, - ); - self.position = pos; - } - - /// Changes the current position by advancing the given number of columns. - fn move_columns(&mut self, columns: u32) { - let pos = Position::new( - self.position.line_number(), - self.position.column_number() + columns, - ); - self.position = pos; - } - - fn carriage_return(&mut self) { - let pos = Position::new(self.position.line_number(), 1); - self.position = pos; - } - - /// Changes the current position by advancing to the next line. - fn next_line(&mut self) { - let pos = Position::new(self.position.line_number() + 1, 1); - self.position = pos; - } - - /// Changes the current position by advancing the given number of lines. - fn move_lines(&mut self, lines: u32) { - let pos = Position::new(self.position.line_number() + lines, 1); - self.position = pos; - } - - /// next fetches the next token and return it, or a LexerError if there are no more. - fn next(&mut self) -> char { - self.buffer.next().expect( - "No more more characters to consume from input stream, \ - use preview_next() first to check before calling next()", - ) - } - - /// Preview the next character but don't actually increment - fn preview_next(&mut self) -> Option { - self.buffer.peek().copied() - } - - /// Preview a char x indexes further in buf, without incrementing - fn preview_multiple_next(&mut self, nb_next: usize) -> Option { - let mut next_peek = None; - - for (i, x) in self.buffer.clone().enumerate() { - if i >= nb_next { - break; - } - - next_peek = Some(x); - } - - next_peek - } - - /// Utility Function, while ``f(char)`` is true, read chars and move curser. - /// All chars are returned as a string - fn take_char_while(&mut self, mut f: F) -> Result - where - F: FnMut(char) -> bool, - { - let mut s = String::new(); - while self.buffer.peek().is_some() - && f(self.preview_next().expect("Could not preview next value")) - { - s.push(self.next()); - } - - Ok(s) - } - - /// Compares the character passed in to the next character, if they match true is returned and the buffer is incremented - fn next_is(&mut self, peek: char) -> bool { - let result = self.preview_next() == Some(peek); - if result { - self.next_column(); - self.buffer.next(); - } - result - } - - /// Utility function for checkint the NumericLiteral is not followed by an `IdentifierStart` or `DecimalDigit` character. - /// - /// More information: - /// - [ECMAScript Specification][spec] - /// - /// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals - fn check_after_numeric_literal(&mut self) -> Result<(), LexerError> { - match self.preview_next() { - Some(ch) - if ch.is_ascii_alphabetic() || ch == '$' || ch == '_' || ch.is_ascii_digit() => - { - Err(LexerError::new("NumericLiteral token must not be followed by IdentifierStart nor DecimalDigit characters")) - } - Some(_) => Ok(()), - None => Ok(()) - } - } - - /// Lexes a numerical literal. - /// - /// More information: - /// - [ECMAScript Specification][spec] - /// - /// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals - fn reed_numerical_literal(&mut self, ch: char) -> Result<(), LexerError> { - /// This is a helper structure - /// - /// This structure helps with identifying what numerical type it is and what base is it. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - enum NumericKind { - Rational, - Integer(u8), - BigInt(u8), - } - - impl NumericKind { - /// Get the base of the number kind. - fn base(self) -> u32 { - match self { - Self::Rational => 10, - Self::Integer(base) => base as u32, - Self::BigInt(base) => base as u32, - } - } - - /// Converts `self` to BigInt kind. - fn to_bigint(self) -> Self { - match self { - Self::Rational => unreachable!("can not convert rational number to BigInt"), - Self::Integer(base) => Self::BigInt(base), - Self::BigInt(base) => Self::BigInt(base), - } - } - } - - // TODO: Setup strict mode. - let strict_mode = false; - - let mut buf = ch.to_string(); - let mut kind = NumericKind::Integer(10); - let start_pos = self.position; - if ch == '0' { - match self.preview_next() { - None => { - self.next_column(); - self.push_token( - TokenKind::NumericLiteral(NumericLiteral::Integer(0)), - start_pos, - ); - return Ok(()); - } - Some('x') | Some('X') => { - self.next(); - self.next_column(); - kind = NumericKind::Integer(16); - } - Some('o') | Some('O') => { - self.next(); - self.next_column(); - kind = NumericKind::Integer(8); - } - Some('b') | Some('B') => { - self.next(); - self.next_column(); - kind = NumericKind::Integer(2); - } - Some(ch) if ch.is_ascii_digit() => { - let mut is_implicit_octal = true; - while let Some(ch) = self.preview_next() { - if !ch.is_ascii_digit() { - break; - } else if !ch.is_digit(8) { - is_implicit_octal = false; - } - buf.push(self.next()); - } - if !strict_mode { - if is_implicit_octal { - kind = NumericKind::Integer(8); - } - } else { - return Err(if is_implicit_octal { - LexerError::new( - "Implicit octal literals are not allowed in strict mode.", - ) - } else { - LexerError::new( - "Decimals with leading zeros are not allowed in strict mode.", - ) - }); - } - } - Some(_) => {} - } - } - - while let Some(ch) = self.preview_next() { - if !ch.is_digit(kind.base()) { - break; - } - buf.push(self.next()); - } - - if self.next_is('n') { - kind = kind.to_bigint(); - } - - if let NumericKind::Integer(10) = kind { - 'digitloop: while let Some(ch) = self.preview_next() { - match ch { - '.' => loop { - kind = NumericKind::Rational; - buf.push(self.next()); - - let c = match self.preview_next() { - Some(ch) => ch, - None => break, - }; - - match c { - 'e' | 'E' => { - match self - .preview_multiple_next(2) - .unwrap_or_default() - .to_digit(10) - { - Some(0..=9) | None => { - buf.push(self.next()); - } - _ => { - break 'digitloop; - } - } - } - _ => { - if !c.is_digit(10) { - break 'digitloop; - } - } - } - }, - 'e' | 'E' => { - kind = NumericKind::Rational; - match self - .preview_multiple_next(2) - .unwrap_or_default() - .to_digit(10) - { - Some(0..=9) | None => { - buf.push(self.next()); - } - _ => { - break; - } - } - buf.push(self.next()); - } - '+' | '-' => { - break; - } - _ if ch.is_digit(10) => { - buf.push(self.next()); - } - _ => break, - } - } - } - - self.check_after_numeric_literal()?; - - let num = match kind { - NumericKind::BigInt(base) => { - NumericLiteral::BigInt( - BigInt::from_string_radix(&buf, base as u32).expect("Could not conver to BigInt") - ) - } - NumericKind::Rational /* base: 10 */ => { - NumericLiteral::Rational( - f64::from_str(&buf) - .map_err(|_| LexerError::new("Could not convert value to f64"))?, - ) - } - NumericKind::Integer(base) => { - if let Ok(num) = i32::from_str_radix(&buf, base as u32) { - NumericLiteral::Integer( - num - ) - } else { - let b = f64::from(base); - let mut result = 0.0_f64; - for c in buf.chars() { - let digit = f64::from(c.to_digit(base as u32).unwrap()); - result = result * b + digit; - } - - NumericLiteral::Rational(result) - } - - } - }; - - self.move_columns(buf.len() as u32); - self.push_token(TokenKind::NumericLiteral(num), start_pos); - - Ok(()) - } - - /// Runs the lexer until completion, returning a [LexerError] if there's a syntax issue, or an empty unit result - // - /// # Example - /// - /// ``` - /// # use boa::syntax::lexer::{LexerError, Lexer}; - /// fn main() -> Result<(), LexerError> { - /// let buffer = String::from("Hello World"); - /// let mut lexer = Lexer::new(&buffer); - /// lexer.lex() - /// } - /// ``` - pub fn lex(&mut self) -> Result<(), LexerError> { - let _timer = BoaProfiler::global().start_event("lex", "lexing"); - loop { - // Check if we've reached the end - if self.preview_next().is_none() { - return Ok(()); - } - let start_pos = self.position; - self.next_column(); - let ch = self.next(); - match ch { - // StringLiteral - '"' | '\'' => { - let mut buf = String::new(); - loop { - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated String")); - } - match self.next() { - '\'' if ch == '\'' => { - break; - } - '"' if ch == '"' => { - break; - } - '\\' => { - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated String")); - } - let escape_pos = self.position; - let escape = self.next(); - if escape != '\n' { - let escaped_ch = match escape { - 'n' => '\n', - 'r' => '\r', - 't' => '\t', - 'b' => '\x08', - 'f' => '\x0c', - '0' => '\0', - 'x' => { - let mut nums = String::with_capacity(2); - for _ in 0_u8..2 { - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated String")); - } - nums.push(self.next()); - } - self.move_columns(2); - let as_num = match u64::from_str_radix(&nums, 16) { - Ok(v) => v, - Err(_) => 0, - }; - match from_u32(as_num as u32) { - Some(v) => v, - None => panic!( - "{}: {} is not a valid unicode scalar value", - self.position, as_num - ), - } - } - 'u' => { - // There are 2 types of codepoints. Surragate codepoints and unicode codepoints. - // UTF-16 could be surrogate codepoints, "\uXXXX\uXXXX" which make up a single unicode codepoint. - // We will need to loop to make sure we catch all UTF-16 codepoints - // Example Test: https://github.com/tc39/test262/blob/ee3715ee56744ccc8aeb22a921f442e98090b3c1/implementation-contributed/v8/mjsunit/es6/unicode-escapes.js#L39-L44 - - // Support \u{X..X} (Unicode Codepoint) - if self.next_is('{') { - let s = self - .take_char_while(char::is_alphanumeric) - .expect("Could not read chars"); - - // We know this is a single unicode codepoint, convert to u32 - let as_num = match u32::from_str_radix(&s, 16) { - Ok(v) => v, - Err(_) => 0, - }; - let c = from_u32(as_num).ok_or_else(|| LexerError::new("Invalid Unicode escape sequence"))?; - - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated String")); - } - self.next(); // '}' - self.move_columns(s.len() as u32); - c - } else { - let mut codepoints: Vec = vec![]; - loop { - // Collect each character after \u e.g \uD83D will give "D83D" - let s = self - .take_char_while(char::is_alphanumeric) - .expect("Could not read chars"); - - // Convert to u16 - let as_num = match u16::from_str_radix(&s, 16) { - Ok(v) => v, - Err(_) => 0, - }; - - codepoints.push(as_num); - self.move_columns(s.len() as u32); - - // Check for another UTF-16 codepoint - if self.next_is('\\') && self.next_is('u') { - continue; - } - break; - } - - // codepoints length should either be 1 (unicode codepoint) or 2 (surrogate codepoint). - // Rust's decode_utf16 will deal with it regardless - decode_utf16(codepoints.iter().cloned()) - .next() - .expect("Could not get next codepoint") - .expect("Could not get next codepoint") - } - } - '\'' | '"' | '\\' => escape, - ch => { - let details = format!("invalid escape sequence `{}` at line {}, column {}", escape_pos.line_number(), escape_pos.column_number(), ch); - return Err(LexerError { details }); - } - }; - buf.push(escaped_ch); - } - } - next_ch => buf.push(next_ch), - } - } - let str_length = buf.len() as u32; - // Why +1? Quotation marks are not included, - // So technically it would be +2, (for both " ") but we want to be 1 less - // to compensate for the incrementing at the top - self.move_columns( str_length.wrapping_add(1)); - self.push_token(TokenKind::string_literal(buf), start_pos); - } - // TemplateLiteral - '`' => { - let mut buf = String::new(); - loop { - if self.preview_next().is_none() { - return Err(LexerError::new("Unterminated template literal")); - } - match self.next() { - '`' => { - break; - } - next_ch => buf.push(next_ch), - // TODO when there is an expression inside the literal - } - } - let str_length = buf.len() as u32; - // Why +1? Quotation marks are not included, - // So technically it would be +2, (for both " ") but we want to be 1 less - // to compensate for the incrementing at the top - self.move_columns( str_length.wrapping_add(1)); - self.push_token(TokenKind::template_literal(buf), start_pos); - } - _ if ch.is_digit(10) => self.reed_numerical_literal(ch)?, - _ if ch.is_alphabetic() || ch == '$' || ch == '_' => { - let mut buf = ch.to_string(); - while let Some(ch) = self.preview_next() { - if ch.is_alphabetic() || ch.is_digit(10) || ch == '_' { - buf.push(self.next()); - } else { - break; - } - } - let tk = match buf.as_str() { - "true" => TokenKind::BooleanLiteral(true), - "false" => TokenKind::BooleanLiteral(false), - "null" => TokenKind::NullLiteral, - "NaN" => TokenKind::NumericLiteral(NumericLiteral::Rational(f64::NAN)), - slice => { - if let Ok(keyword) = FromStr::from_str(slice) { - TokenKind::Keyword(keyword) - } else { - TokenKind::identifier(slice) - } - } - }; - - // Move position forward the length of the token - self.move_columns( (buf.len().wrapping_sub(1)) as u32); - - self.push_token(tk, start_pos); - } - ';' => self.push_punc(Punctuator::Semicolon, start_pos), - ':' => self.push_punc(Punctuator::Colon, start_pos), - '.' => { - // . or ... - if self.next_is('.') { - if self.next_is('.') { - self.push_punc(Punctuator::Spread, start_pos); - } else { - return Err(LexerError::new("Expecting Token .")); - } - } else { - self.push_punc(Punctuator::Dot, start_pos); - }; - } - '(' => self.push_punc(Punctuator::OpenParen, start_pos), - ')' => self.push_punc(Punctuator::CloseParen, start_pos), - ',' => self.push_punc(Punctuator::Comma, start_pos), - '{' => self.push_punc(Punctuator::OpenBlock, start_pos), - '}' => self.push_punc(Punctuator::CloseBlock, start_pos), - '[' => self.push_punc(Punctuator::OpenBracket, start_pos), - ']' => self.push_punc(Punctuator::CloseBracket, start_pos), - '?' => self.push_punc(Punctuator::Question, start_pos), - // Comments - '/' => { - if let Some(ch) = self.preview_next() { - match ch { - // line comment - '/' => { - while self.preview_next().is_some() { - if self.next() == '\n' { - break; - } - } - self.next_line() - } - // block comment - '*' => { - let mut lines = 0; - loop { - if self.preview_next().is_none() { - return Err(LexerError::new("unterminated multiline comment")); - } - match self.next() { - '*' => { - if self.next_is('/') { - break; - } - } - next_ch => { - if next_ch == '\n' { - lines += 1; - } - }, - } - } - self.move_lines(lines); - } - // division, assigndiv or regex literal - _ => { - // if we fail to parse a regex literal, store a copy of the current - // buffer to restore later on - let original_buffer = self.buffer.clone(); - let original_pos = self.position; - // first, try to parse a regex literal - let mut body = String::new(); - let mut regex = false; - loop { - self.next_column(); - match self.buffer.next() { - // end of body - Some('/') => { - regex = true; - break; - } - // newline/eof not allowed in regex literal - n @ Some('\n') | n @ Some('\r') | n @ Some('\u{2028}') - | n @ Some('\u{2029}') => { - self.carriage_return(); - if n != Some('\r') { - self.next_line(); - } - break - }, - None => { - self.position = Position::new(self.position.line_number(), self.position.column_number()-1); - break - } - // escape sequence - Some('\\') => { - body.push('\\'); - if self.preview_next().is_none() { - break; - } - match self.next() { - // newline not allowed in regex literal - '\n' | '\r' | '\u{2028}' | '\u{2029}' => break, - ch => body.push(ch), - } - } - Some(ch) => body.push(ch), - } - } - if regex { - // body was parsed, now look for flags - let flags = self.take_char_while(char::is_alphabetic)?; - self.move_columns(body.len() as u32 + 1 + flags.len() as u32); - self.push_token(TokenKind::regular_expression_literal( - body, flags.parse()?, - ), start_pos); - } else { - // failed to parse regex, restore original buffer position and - // parse either div or assigndiv - self.buffer = original_buffer; - self.position = original_pos; - if self.next_is('=') { - self.push_token(TokenKind::Punctuator( - Punctuator::AssignDiv, - ), start_pos); - } else { - self.push_token(TokenKind::Punctuator(Punctuator::Div), start_pos); - } - } - } - } - } else { - return Err(LexerError::new("Expecting Token /,*,= or regex")); - } - } - '*' => op!(self, start_pos, Punctuator::AssignMul, Punctuator::Mul, { - '*' => vop!(self, Punctuator::AssignPow, Punctuator::Exp) - }), - '+' => op!(self, start_pos, Punctuator::AssignAdd, Punctuator::Add, { - '+' => Punctuator::Inc - }), - '-' => op!(self, start_pos, Punctuator::AssignSub, Punctuator::Sub, { - '-' => { - Punctuator::Dec - } - }), - '%' => op!(self, start_pos, Punctuator::AssignMod, Punctuator::Mod), - '|' => op!(self, start_pos, Punctuator::AssignOr, Punctuator::Or, { - '|' => Punctuator::BoolOr - }), - '&' => op!(self, start_pos, Punctuator::AssignAnd, Punctuator::And, { - '&' => Punctuator::BoolAnd - }), - '^' => op!(self, start_pos, Punctuator::AssignXor, Punctuator::Xor), - '=' => op!(self, start_pos, if self.next_is('=') { - Punctuator::StrictEq - } else { - Punctuator::Eq - }, Punctuator::Assign, { - '>' => { - Punctuator::Arrow - } - }), - '<' => op!(self, start_pos, Punctuator::LessThanOrEq, Punctuator::LessThan, { - '<' => vop!(self, Punctuator::AssignLeftSh, Punctuator::LeftSh) - }), - '>' => op!(self, start_pos, Punctuator::GreaterThanOrEq, Punctuator::GreaterThan, { - '>' => vop!(self, Punctuator::AssignRightSh, Punctuator::RightSh, { - '>' => vop!(self, Punctuator::AssignURightSh, Punctuator::URightSh) - }) - }), - '!' => op!( - self, - start_pos, - vop!(self, Punctuator::StrictNotEq, Punctuator::NotEq), - Punctuator::Not - ), - '~' => self.push_punc(Punctuator::Neg, start_pos), - '\n' | '\u{2028}' | '\u{2029}' => { - self.next_line(); - self.push_token(TokenKind::LineTerminator, start_pos); - } - '\r' => { - self.carriage_return(); - } - // The rust char::is_whitespace function and the ecma standard use different sets - // of characters as whitespaces: - // * Rust uses \p{White_Space}, - // * ecma standard uses \{Space_Separator} + \u{0009}, \u{000B}, \u{000C}, \u{FEFF} - // - // Explicit whitespace: see https://tc39.es/ecma262/#table-32 - '\u{0020}' | '\u{0009}' | '\u{000B}' | '\u{000C}' | '\u{00A0}' | '\u{FEFF}' | - // Unicode Space_Seperator category (minus \u{0020} and \u{00A0} which are allready stated above) - '\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{202F}' | '\u{205F}' | '\u{3000}' => (), - _ => { - let details = format!("Unexpected '{}' at line {}, column {}", start_pos.line_number(), start_pos.column_number(), ch); - return Err(LexerError { details }); - }, - } - } - } -} diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index b43d1d61c71..98ef768fac9 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -207,7 +207,7 @@ where )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), template_match!() => { - let result = TemplateLiteral::new().lex(&mut self.cursor, start); + let result = TemplateLiteral.lex(&mut self.cursor, start); // A regex may follow a template literal but a DivPunctuator or TemplateSubstitutionTail may not. // self.set_goal(InputElement::RegExp); diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs index ad36661c4d3..4ed3b6e9e4a 100644 --- a/boa/src/syntax/lexer/template.rs +++ b/boa/src/syntax/lexer/template.rs @@ -22,13 +22,6 @@ macro_rules! template_match { #[derive(Debug, Clone, Copy)] pub(super) struct TemplateLiteral; -impl TemplateLiteral { - /// Creates a new template literal lexer. - pub(super) fn new() -> Self { - Self {} - } -} - impl Tokenizer for TemplateLiteral { fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where From b96b6147892a3d7f87d8d75092d91f02fb877ce3 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 15:42:01 +0100 Subject: [PATCH 102/289] Tidy up spread literal lex() --- boa/src/syntax/lexer/mod.rs | 8 -------- boa/src/syntax/lexer/spread.rs | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 98ef768fac9..bfce9ad8ae6 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -5,25 +5,17 @@ #[macro_use] mod comment; - mod cursor; pub mod error; - #[macro_use] mod string; pub mod token; - #[macro_use] mod template; - mod number; - mod operator; - mod spread; - mod regex; - mod identifier; // Temporary disabled while lexer in progress. diff --git a/boa/src/syntax/lexer/spread.rs b/boa/src/syntax/lexer/spread.rs index 482f8480898..08392d827cd 100644 --- a/boa/src/syntax/lexer/spread.rs +++ b/boa/src/syntax/lexer/spread.rs @@ -22,20 +22,20 @@ impl Tokenizer for SpreadLiteral { R: Read, { // . or ... - match cursor.next_is('.') { - Err(e) => Err(e.into()), - Ok(true) => match cursor.next_is('.') { - Err(e) => Err(e.into()), - Ok(true) => Ok(Token::new( + if cursor.next_is('.')? { + if cursor.next_is('.')? { + Ok(Token::new( Punctuator::Spread.into(), Span::new(start_pos, cursor.pos()), - )), - Ok(false) => Err(Error::syntax("Expecting Token .")), - }, - Ok(false) => Ok(Token::new( + )) + } else { + Err(Error::syntax("Expecting Token .")) + } + } else { + Ok(Token::new( Punctuator::Dot.into(), Span::new(start_pos, cursor.pos()), - )), + )) } } } From d665540d148e6ee3975257a7526cc1cb942b847f Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 16:04:36 +0100 Subject: [PATCH 103/289] fixed a couple small warnings --- boa/src/syntax/lexer/cursor.rs | 2 +- boa/src/syntax/parser/cursor.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index f310d0a8b68..4dd6d210819 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -145,7 +145,7 @@ where /// It will fill the buffer with checked ASCII bytes. pub(super) fn fill_bytes(&mut self, buf: &mut [u8]) -> io::Result<()> { - unimplemented!() + unimplemented!("Lexer::cursor::fill_bytes {:?}", buf) } } diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 370b66e2eec..1154eea0b7c 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -3,7 +3,7 @@ use super::ParseError; use crate::syntax::{ ast::Punctuator, - lexer::{Error as LexerError, InputElement, Lexer, Position, Token, TokenKind}, + lexer::{InputElement, Lexer, Position, Token, TokenKind}, }; use std::collections::VecDeque; From 1914c6687400c2607a47124fd498d3fd4c1de993 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 21:48:11 +0100 Subject: [PATCH 104/289] Changed lexer peek() to return values rather than references --- boa/benches/lexer.rs | 9 +++-- boa/src/syntax/lexer/cursor.rs | 54 +++++++++++++++++++++++------- boa/src/syntax/lexer/identifier.rs | 2 +- boa/src/syntax/lexer/mod.rs | 7 ++-- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/boa/benches/lexer.rs b/boa/benches/lexer.rs index df0e38f845c..68def8fde3c 100644 --- a/boa/benches/lexer.rs +++ b/boa/benches/lexer.rs @@ -62,7 +62,10 @@ fn for_loop_lexer(c: &mut Criterion) { }); } -criterion_group!(lexer, - // expression_lexer, - hello_world_lexer, for_loop_lexer); +criterion_group!( + lexer, + // expression_lexer, + hello_world_lexer, + for_loop_lexer +); criterion_main!(lexer); diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 4dd6d210819..e887588c7a0 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -1,11 +1,11 @@ use crate::syntax::ast::Position; -use std::io::{self, Bytes, ErrorKind, Read}; +use std::io::{self, Bytes, Error, ErrorKind, Read}; /// Cursor over the source code. #[derive(Debug)] pub(super) struct Cursor { iter: InnerIter, - peeked: Option>>, + peeked: Option>, pos: Position, } @@ -54,9 +54,27 @@ where /// Peeks the next character. #[inline] - pub(super) fn peek(&mut self) -> Option<&io::Result> { + pub(super) fn peek(&mut self) -> Option> { let iter = &mut self.iter; - self.peeked.get_or_insert_with(|| iter.next()).as_ref() + + match self.peeked { + None => match iter.next() { + Some(Err(e)) => { + return Some(Err(e)); + } + Some(Ok(c)) => { + self.peeked = Some(Some(c)); + return Some(Ok(c)); + } + None => { + self.peeked = Some(None); + return None; + } + }, + Some(v) => { + return v.map(|v| Ok(v)); + } + } } /// Compares the character passed in to the next character, if they match true is returned and the buffer is incremented @@ -64,12 +82,11 @@ where pub(super) fn next_is(&mut self, peek: char) -> io::Result { Ok(match self.peek() { None => false, - Some(&Ok(next)) if next == peek => { + Some(Ok(next)) if next == peek => { let _ = self.peeked.take(); true } _ => false, - // Some(&Err(_)) => return self.peeked.take().unwrap().unwrap().map(|_| false), }) } @@ -84,8 +101,8 @@ where { Ok(match self.peek() { None => false, - Some(Ok(peek)) => pred(*peek), - Some(Err(e)) => todo!(), + Some(Ok(peek)) => pred(peek), + Some(Err(e)) => return Err(e), }) } @@ -159,17 +176,28 @@ where fn next(&mut self) -> Option { let chr = match self.peeked.take() { Some(v) => v, - None => self.iter.next(), + None => { + if let Some(n) = self.iter.next() { + match n { + Err(e) => { + return Some(Err(e)); + } + Ok(c) => Some(c), + } + } else { + None + } + } }; match chr { - Some(Ok('\r')) => self.carriage_return(), - Some(Ok('\n')) | Some(Ok('\u{2028}')) | Some(Ok('\u{2029}')) => self.next_line(), - Some(Ok(_)) => self.next_column(), + Some('\r') => self.carriage_return(), + Some('\n') | Some('\u{2028}') | Some('\u{2029}') => self.next_line(), + Some(_) => self.next_column(), _ => {} } - chr + chr.map(|v| Ok(v)) } } diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index bc98097f6d3..0d5a00a7d31 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -40,7 +40,7 @@ impl Tokenizer for Identifier { break; } Some(Ok(c)) => { - if c.is_alphabetic() || c.is_digit(10) || *c == '_' { + if c.is_alphabetic() || c.is_digit(10) || c == '_' { let ch = cursor.next().unwrap()?; buf.push(ch); } else { diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index bfce9ad8ae6..6d1c87b086f 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -12,11 +12,11 @@ mod string; pub mod token; #[macro_use] mod template; +mod identifier; mod number; mod operator; -mod spread; mod regex; -mod identifier; +mod spread; // Temporary disabled while lexer in progress. #[cfg(test)] @@ -117,8 +117,7 @@ impl Lexer { self.cursor.next(); BlockComment.lex(&mut self.cursor, start) } - Ok(c) => { - let ch = *c; + Ok(ch) => { match self.get_goal() { InputElement::Div | InputElement::TemplateTail => { // Only div punctuator allowed, regex not. From 440bcbb6d8f04c11adb2cce18cedffb81a62173a Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 21:53:30 +0100 Subject: [PATCH 105/289] Handled peek error returns --- boa/src/syntax/lexer/mod.rs | 11 +-- boa/src/syntax/lexer/number.rs | 144 ++++++++++++++++----------------- 2 files changed, 74 insertions(+), 81 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 6d1c87b086f..4bac9c02356 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -105,19 +105,16 @@ impl Lexer { R: Read, { if let Some(c) = self.cursor.peek() { - match c { - Err(e) => { - todo!(); - } - Ok('/') => { + match c? { + '/' => { self.cursor.next(); // Consume the SingleLineComment.lex(&mut self.cursor, start) } - Ok('*') => { + '*' => { self.cursor.next(); BlockComment.lex(&mut self.cursor, start) } - Ok(ch) => { + ch => { match self.get_goal() { InputElement::Div | InputElement::TemplateTail => { // Only div punctuator allowed, regex not. diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index dad60a6e722..b67c4920634 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -138,84 +138,80 @@ impl Tokenizer for NumberLiteral { let c = cursor.peek(); if self.init == '0' { - match c { - None => { - // DecimalLiteral lexing. - // Indicates that the number is just a single 0. - return Ok(Token::new( - TokenKind::NumericLiteral(Numeric::Integer(0)), - Span::new(start_pos, cursor.pos()), - )); - } - Some(Err(e)) => { - todo!(); - } - Some(Ok('x')) | Some(Ok('X')) => { - // Remove the initial '0' from buffer. - cursor.next(); - buf.pop(); - - // HexIntegerLiteral - kind = NumericKind::Integer(16); - } - Some(Ok('o')) | Some(Ok('O')) => { - // Remove the initial '0' from buffer. - cursor.next(); - buf.pop(); - - // OctalIntegerLiteral - kind = NumericKind::Integer(8); - } - Some(Ok('b')) | Some(Ok('B')) => { - // Remove the initial '0' from buffer. - cursor.next(); - buf.pop(); - - // BinaryIntegerLiteral - kind = NumericKind::Integer(2); - } - Some(Ok('n')) => { - cursor.next(); - - // DecimalBigIntegerLiteral '0n' - return Ok(Token::new( - TokenKind::NumericLiteral(Numeric::BigInt(0.into())), - Span::new(start_pos, cursor.pos()), - )); - } - Some(Ok(ch)) => { - if ch.is_digit(8) { - // LegacyOctalIntegerLiteral - if self.strict_mode { - // LegacyOctalIntegerLiteral is forbidden with strict mode true. - return Err(Error::strict( - "Implicit octal literals are not allowed in strict mode.", - )); - } else { - // Remove the initial '0' from buffer. - buf.pop(); + if let Some(ch) = c { + match ch? { + 'x' | 'X' => { + // Remove the initial '0' from buffer. + cursor.next(); + buf.pop(); + + // HexIntegerLiteral + kind = NumericKind::Integer(16); + } + 'o' | 'O' => { + // Remove the initial '0' from buffer. + cursor.next(); + buf.pop(); - let char = cursor.next().unwrap().unwrap(); - buf.push(char); + // OctalIntegerLiteral + kind = NumericKind::Integer(8); + } + 'b' | 'B' => { + // Remove the initial '0' from buffer. + cursor.next(); + buf.pop(); - kind = NumericKind::Integer(8); - } - } else if ch.is_digit(10) { - // Indicates a numerical digit comes after then 0 but it isn't an octal digit - // so therefore this must be a number with an unneeded leading 0. This is - // forbidden in strict mode. - if self.strict_mode { - return Err(Error::strict( - "Leading 0's are not allowed in strict mode.", - )); - } else { - let char = cursor.next().unwrap().unwrap(); - buf.push(char); - } - } else { - // Indicates that the symbol is a non-number. + // BinaryIntegerLiteral + kind = NumericKind::Integer(2); + } + 'n' => { + cursor.next(); + + // DecimalBigIntegerLiteral '0n' + return Ok(Token::new( + TokenKind::NumericLiteral(Numeric::BigInt(0.into())), + Span::new(start_pos, cursor.pos()), + )); + } + ch => { + if ch.is_digit(8) { + // LegacyOctalIntegerLiteral + if self.strict_mode { + // LegacyOctalIntegerLiteral is forbidden with strict mode true. + return Err(Error::strict( + "Implicit octal literals are not allowed in strict mode.", + )); + } else { + // Remove the initial '0' from buffer. + buf.pop(); + + let char = cursor.next().unwrap().unwrap(); + buf.push(char); + + kind = NumericKind::Integer(8); + } + } else if ch.is_digit(10) { + // Indicates a numerical digit comes after then 0 but it isn't an octal digit + // so therefore this must be a number with an unneeded leading 0. This is + // forbidden in strict mode. + if self.strict_mode { + return Err(Error::strict( + "Leading 0's are not allowed in strict mode.", + )); + } else { + let char = cursor.next().unwrap().unwrap(); + buf.push(char); + } + } // Else indicates that the symbol is a non-number. } } + } else { + // DecimalLiteral lexing. + // Indicates that the number is just a single 0. + return Ok(Token::new( + TokenKind::NumericLiteral(Numeric::Integer(0)), + Span::new(start_pos, cursor.pos()), + )); } } From b0c6f077a7eb4bd0e64126d7403227add842552e Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 22:02:59 +0100 Subject: [PATCH 106/289] Making clippy happier --- boa/src/syntax/lexer/cursor.rs | 12 ++++-------- boa/src/syntax/lexer/number.rs | 12 +++++------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index e887588c7a0..7a9a68895b5 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -59,21 +59,17 @@ where match self.peeked { None => match iter.next() { - Some(Err(e)) => { - return Some(Err(e)); - } + Some(Err(e)) => Some(Err(e)), Some(Ok(c)) => { self.peeked = Some(Some(c)); - return Some(Ok(c)); + Some(Ok(c)) } None => { self.peeked = Some(None); - return None; + None } }, - Some(v) => { - return v.map(|v| Ok(v)); - } + Some(v) => v.map(|v| Ok(v)), } } diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index b67c4920634..647ae2118b4 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -332,8 +332,7 @@ impl Tokenizer for NumberLiteral { Numeric::Rational( (num as f64) * f64::powi(10.0, n) ) - } else { - if let Some(exp) = i32::checked_pow(10, n as u32) { + } else if let Some(exp) = i32::checked_pow(10, n as u32) { if let Some(val) = i32::checked_mul(num, exp) { Numeric::Integer(val) } else { @@ -341,11 +340,10 @@ impl Tokenizer for NumberLiteral { (num as f64) * (exp as f64) ) } - } else { - Numeric::Rational( - (num as f64) * f64::powi(10.0, n) - ) - } + } else { + Numeric::Rational( + (num as f64) * f64::powi(10.0, n) + ) } } } else { From 253f30ca3a3f26659aeec1f4d2bee52290b22657 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 22:13:48 +0100 Subject: [PATCH 107/289] Making clippy happier --- boa/src/syntax/lexer/mod.rs | 27 +++---------------- .../syntax/parser/expression/primary/mod.rs | 16 +++++------ 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 4bac9c02356..dc81577593d 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -194,27 +194,12 @@ where Span::new(start, self.cursor.pos()), )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), - template_match!() => { - let result = TemplateLiteral.lex(&mut self.cursor, start); - - // A regex may follow a template literal but a DivPunctuator or TemplateSubstitutionTail may not. - // self.set_goal(InputElement::RegExp); - result - } + template_match!() => TemplateLiteral.lex(&mut self.cursor, start), _ if next_chr.is_digit(10) => { - let result = NumberLiteral::new(next_chr, strict_mode).lex(&mut self.cursor, start); - // A regex may not directly follow a NumericLiteral but a DivPunctuator may. - // Note that the goal cannot be set to InputElementTemplateTail at this point as a TemplateSubstitutionTail would be invalid. - // self.set_goal(InputElement::Div); - result + NumberLiteral::new(next_chr, strict_mode).lex(&mut self.cursor, start) } _ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => { - let result = Identifier::new(next_chr).lex(&mut self.cursor, start); - - // A regex may not directly follow an Identifier but a DivPunctuator may. - // Note that the goal cannot be set to InputElementTemplateTail at this point as a TemplateSubstitutionTail would be invalid. - // self.set_goal(InputElement::Div); - result + Identifier::new(next_chr).lex(&mut self.cursor, start) } ';' => Ok(Token::new( Punctuator::Semicolon.into(), @@ -259,11 +244,7 @@ where )), '/' => self.lex_slash_token(start), '=' | '*' | '+' | '-' | '%' | '|' | '&' | '^' | '<' | '>' | '!' | '~' => { - let result = Operator::new(next_chr).lex(&mut self.cursor, start); - - // self.set_goal(InputElement::RegExpOrTemplateTail); - - result + Operator::new(next_chr).lex(&mut self.cursor, start) } _ => { let details = format!( diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 4be0ab23c38..b4a481e116b 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -104,34 +104,34 @@ where TokenKind::NumericLiteral(Numeric::Rational(num)) => Ok(Const::from(*num).into()), TokenKind::NumericLiteral(Numeric::BigInt(num)) => Ok(Const::from(num.clone()).into()), TokenKind::RegularExpressionLiteral(body, flags) => { - let res = Ok(Node::from(New::from(Call::new( + Ok(Node::from(New::from(Call::new( Identifier::from("RegExp"), vec![ Const::from(body.as_ref()).into(), Const::from(flags.to_string()).into(), ], - )))); - res + )))) } TokenKind::Punctuator(Punctuator::Div) => { // This is where the start of a regexp is accidentally treated as a div - // Try parsing as a regexp. let tok = cursor.lex_regex(tok.span().start())?; if let TokenKind::RegularExpressionLiteral(body, flags) = tok.kind() { - let res = Ok(Node::from(New::from(Call::new( + Ok(Node::from(New::from(Call::new( Identifier::from("RegExp"), vec![ Const::from(body.as_ref()).into(), Const::from(flags.to_string()).into(), ], - )))); - res + )))) } else { // A regex was expected and nothing else. - unimplemented!("How to handle this case?"); + Err(ParseError::unexpected( + tok, + "primary expression - expected regex", + )) } } _ => Err(ParseError::unexpected(tok.clone(), "primary expression")), From 8c261af1fb301258833d5875d2c3ccab916e20cf Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 22:16:47 +0100 Subject: [PATCH 108/289] Removed macro for chosing branch in lexer --- boa/src/syntax/lexer/mod.rs | 18 ++++++++---------- boa/src/syntax/lexer/template.rs | 6 ------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index dc81577593d..c75f6771c4b 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -3,26 +3,23 @@ //! The Lexer splits its input source code into a sequence of input elements called tokens, represented by the [Token](../ast/token/struct.Token.html) structure. //! It also removes whitespace and comments and attaches them to the next token. -#[macro_use] mod comment; mod cursor; pub mod error; -#[macro_use] -mod string; -pub mod token; -#[macro_use] -mod template; mod identifier; mod number; mod operator; mod regex; mod spread; +mod string; +mod template; +pub mod token; -// Temporary disabled while lexer in progress. #[cfg(test)] mod tests; -pub use self::error::Error; +pub use error::Error; +pub use token::{Token, TokenKind}; use self::{ comment::{BlockComment, SingleLineComment}, @@ -35,10 +32,11 @@ use self::{ string::StringLiteral, template::TemplateLiteral, }; + pub use crate::syntax::ast::Position; use crate::syntax::ast::{Punctuator, Span}; + use std::io::Read; -pub use token::{Token, TokenKind}; trait Tokenizer { /// Lexes the next token. @@ -194,7 +192,7 @@ where Span::new(start, self.cursor.pos()), )), '"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start), - template_match!() => TemplateLiteral.lex(&mut self.cursor, start), + '`' => TemplateLiteral.lex(&mut self.cursor, start), _ if next_chr.is_digit(10) => { NumberLiteral::new(next_chr, strict_mode).lex(&mut self.cursor, start) } diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs index 4ed3b6e9e4a..f85d3dee659 100644 --- a/boa/src/syntax/lexer/template.rs +++ b/boa/src/syntax/lexer/template.rs @@ -3,12 +3,6 @@ use crate::syntax::ast::{Position, Span}; use crate::syntax::lexer::{Token, TokenKind}; use std::io::{self, ErrorKind, Read}; -macro_rules! template_match { - () => { - '`' - }; -} - /// Template literal lexing. /// /// Expects: Initial ` to already be consumed by cursor. From 9b90eef98e66ed812038986429325a749637e94b Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 22:30:11 +0100 Subject: [PATCH 109/289] Cleaning up, making clippy happier --- boa/src/syntax/lexer/cursor.rs | 15 +++-- boa/src/syntax/parser/cursor.rs | 61 +------------------ .../parser/expression/left_hand_side/call.rs | 2 +- .../expression/left_hand_side/member.rs | 4 +- .../primary/object_initializer/mod.rs | 4 +- 5 files changed, 15 insertions(+), 71 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 7a9a68895b5..c2b5eb5e7aa 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -254,14 +254,13 @@ where } match std::str::from_utf8(&buf) { - Ok(s) => match s.chars().next() { - Some(chr) => chr, - None => { - return Some(Err(io::Error::new( - io::ErrorKind::InvalidData, - "stream did not contain valid UTF-8", - ))); - } + Ok(s) => if let Some(chr) = s.chars().next() { + chr + } else { + return Some(Err(io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ))); }, Err(_) => { return Some(Err(io::Error::new( diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 1154eea0b7c..c7ad92eeab0 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -160,66 +160,11 @@ where ret.map(|token| Ok(token)) } + /// Takes the given token and pushes it back onto the parser token queue (at the front so the token will be returned on next .peek()). pub(super) fn push_back(&mut self, token: Token) { self.peeked.push_front(Some(token)); } - // /// Moves the cursor to the previous token and returns the token. - // pub(super) fn back(&mut self) -> Option> { - // unimplemented!(); - - // // debug_assert!( - // // self.back_queue.len() > 0, - // // "cannot go back in a cursor that is at the beginning of the list of tokens" - // // ); - - // // let token = self.back_queue.pop_back().unwrap(); - - // // self.peeked.push_front(token.clone()); - - // // token.map(|t| Ok(t)) - - // // unimplemented!(); - - // // debug_assert!( - // // self.pos > 0, - // // "cannot go back in a cursor that is at the beginning of the list of tokens" - // // ); - - // // self.pos -= 1; - // // while self - // // .tokens - // // .get(self.pos - 1) - // // .expect("token disappeared") - // // .kind - // // == TokenKind::LineTerminator - // // && self.pos > 0 - // // { - // // self.pos -= 1; - // // } - // } - - // /// Peeks the previous token without moving the cursor. - // pub(super) fn peek_prev(&self) -> Option> { - // unimplemented!(); - // // if self.pos == 0 { - // // None - // // } else { - // // let mut back = 1; - // // let mut tok = self.tokens.get(self.pos - back).expect("token disappeared"); - // // while self.pos >= back && tok.kind == TokenKind::LineTerminator { - // // back += 1; - // // tok = self.tokens.get(self.pos - back).expect("token disappeared"); - // // } - - // // if back == self.pos { - // // None - // // } else { - // // Some(tok) - // // } - // // } - // } - /// Returns an error if the next token is not of kind `kind`. /// /// Note: it will consume the next token only if the next token is the expected type. @@ -236,7 +181,7 @@ where } else { Err(ParseError::expected( vec![kind], - next_token.clone(), + next_token, context, )) } @@ -281,7 +226,7 @@ where (true, None) => Ok(None), (false, Some(tk)) => Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::Semicolon)], - tk.clone(), + tk, context, )), (false, None) => unreachable!(), diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index 24902916ca7..24bef1039a7 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -77,7 +77,7 @@ where let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::OpenParen)], - next_token?.clone(), + next_token?, "call expression", )); } diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 23720f21851..6b4245257ed 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -75,7 +75,7 @@ where while let Some(tok) = cursor.peek() { let token = tok?.clone(); match token.kind() { - &TokenKind::Punctuator(Punctuator::Dot) => { + TokenKind::Punctuator(Punctuator::Dot) => { cursor.next().ok_or(ParseError::AbruptEnd)??; // We move the parser forward. match cursor.next().ok_or(ParseError::AbruptEnd)??.kind() { @@ -94,7 +94,7 @@ where } } } - &TokenKind::Punctuator(Punctuator::OpenBracket) => { + TokenKind::Punctuator(Punctuator::OpenBracket) => { let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. let idx = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index c64ed14bb93..2e448111b1d 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -93,7 +93,7 @@ where TokenKind::Punctuator(Punctuator::Comma), TokenKind::Punctuator(Punctuator::CloseBlock), ], - next_token.clone(), + next_token, "object literal", )); } @@ -206,7 +206,7 @@ where TokenKind::Punctuator(Punctuator::OpenParen), "property method definition", )?; - let first_param = cursor.peek().expect("current token disappeared")?.clone(); + let first_param = cursor.peek().expect("current token disappeared")?; let params = FormalParameters::new(false, false).parse(cursor)?; cursor.expect(Punctuator::CloseParen, "method definition")?; if idn == "get" { From 080c2be566c0f6a4f2c22c7aea02ef061ad10ccf Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 22:32:11 +0100 Subject: [PATCH 110/289] The insatiable clippy continues --- boa/src/syntax/parser/statement/declaration/lexical.rs | 2 +- boa/src/syntax/parser/statement/iteration/do_while_statement.rs | 2 +- boa/src/syntax/parser/statement/mod.rs | 2 +- boa/src/syntax/parser/statement/try_stm/mod.rs | 2 +- boa/src/syntax/parser/statement/variable.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 9047f0b5b70..59d217f8724 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -134,7 +134,7 @@ where } else { return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::Assign)], - cursor.next().ok_or(ParseError::AbruptEnd)??.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)??, "const declaration", )); } diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index 3de500dd472..fdc58febb32 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -78,7 +78,7 @@ where if next_token.kind() != &TokenKind::Keyword(Keyword::While) { return Err(ParseError::expected( vec![TokenKind::Keyword(Keyword::While)], - next_token.clone(), + next_token, "do while statement", )); } diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 27c62470986..24966722f15 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -239,7 +239,7 @@ where if self.break_when_closingbrase { break; } else { - return Err(ParseError::unexpected(token.clone(), None)); + return Err(ParseError::unexpected(token, None)); } } Some(Err(e)) => { diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index 37ed66d9a88..d79a31f3c24 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -73,7 +73,7 @@ where TokenKind::Keyword(Keyword::Catch), TokenKind::Keyword(Keyword::Finally), ], - next_token.clone(), + next_token, "try statement", )); } diff --git a/boa/src/syntax/parser/statement/variable.rs b/boa/src/syntax/parser/statement/variable.rs index 6502d3cc44d..0d5c2324f28 100644 --- a/boa/src/syntax/parser/statement/variable.rs +++ b/boa/src/syntax/parser/statement/variable.rs @@ -126,7 +126,7 @@ where TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - cursor.next().ok_or(ParseError::AbruptEnd)??.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)??, "lexical declaration", )) } From b021cd0e8bbd40412b99b59c6fde8b26fd603d97 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 22:33:09 +0100 Subject: [PATCH 111/289] Fmt --- boa/src/syntax/lexer/cursor.rs | 18 ++++++++++-------- boa/src/syntax/lexer/number.rs | 2 -- boa/src/syntax/parser/cursor.rs | 6 +----- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index c2b5eb5e7aa..46961236b2a 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -254,14 +254,16 @@ where } match std::str::from_utf8(&buf) { - Ok(s) => if let Some(chr) = s.chars().next() { - chr - } else { - return Some(Err(io::Error::new( - io::ErrorKind::InvalidData, - "stream did not contain valid UTF-8", - ))); - }, + Ok(s) => { + if let Some(chr) = s.chars().next() { + chr + } else { + return Some(Err(io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ))); + } + } Err(_) => { return Some(Err(io::Error::new( io::ErrorKind::InvalidData, diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 647ae2118b4..3917be2541b 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -315,7 +315,6 @@ impl Tokenizer for NumberLiteral { ) } else { let n = f64::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; - Numeric::Rational( r * f64::powf(10.0, n) ) @@ -360,7 +359,6 @@ impl Tokenizer for NumberLiteral { ) } else { let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; - Numeric::Rational( result * f64::powi(10.0, n) ) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index c7ad92eeab0..64a4b593d5a 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -179,11 +179,7 @@ where self.next(); Ok(next_token) } else { - Err(ParseError::expected( - vec![kind], - next_token, - context, - )) + Err(ParseError::expected(vec![kind], next_token, context)) } } From 5495371488793266b391bb50ff5f2f972c611def Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 22:36:05 +0100 Subject: [PATCH 112/289] Fmt, clippy --- boa/src/syntax/lexer/cursor.rs | 21 ++++++++----------- .../parser/statement/declaration/lexical.rs | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 46961236b2a..85b01df265b 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -253,23 +253,20 @@ where *b = next; } - match std::str::from_utf8(&buf) { - Ok(s) => { - if let Some(chr) = s.chars().next() { - chr - } else { - return Some(Err(io::Error::new( - io::ErrorKind::InvalidData, - "stream did not contain valid UTF-8", - ))); - } - } - Err(_) => { + if let Ok(s) = std::str::from_utf8(&buf) { + if let Some(chr) = s.chars().next() { + chr + } else { return Some(Err(io::Error::new( io::ErrorKind::InvalidData, "stream did not contain valid UTF-8", ))); } + } else { + return Some(Err(io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ))); } }; diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index 59d217f8724..e8a4737868e 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -153,7 +153,7 @@ where TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - cursor.next().ok_or(ParseError::AbruptEnd)??.clone(), + cursor.next().ok_or(ParseError::AbruptEnd)??, "lexical declaration", )) } From 7a97e71f5d8f13c3d5ac71ae123e5eca1782760c Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 3 Jul 2020 22:39:25 +0100 Subject: [PATCH 113/289] Making clippy happy... continues... --- boa/src/syntax/parser/expression/mod.rs | 42 ++----------------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 9db71e183ac..c64901d1700 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -63,7 +63,6 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); - // let old_goal = cursor.get_goal(); if $goal.is_some() { cursor.set_goal($goal.unwrap()); @@ -71,8 +70,8 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; while let Some(tok) = cursor.peek() { - match tok?.kind() { - &TokenKind::Punctuator(op) if $( op == $op )||* => { + match *tok?.kind() { + TokenKind::Punctuator(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), @@ -80,7 +79,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo $lower::new($( self.$low_param ),*).parse(cursor)? ).into(); } - &TokenKind::Keyword(op) if $( op == $op )||* => { + TokenKind::Keyword(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), @@ -92,7 +91,6 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo } } - // cursor.set_goal(old_goal); Ok(lhs) } } @@ -529,40 +527,6 @@ impl MultiplicativeExpression { } } -// impl TokenParser for MultiplicativeExpression -// where -// R: Read -// { -// type Output = Node; - -// fn parse(self, cursor: &mut Cursor) -> ParseResult { -// let _timer = BoaProfiler::global().start_event("Expression", "Parsing"); -// let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; -// while let Some(tok) = cursor.peek() { -// match tok?.kind() { -// &TokenKind::Punctuator(op) if $( op == $op )||* => { -// let _ = cursor.next().expect("token disappeared"); -// lhs = BinOp::new( -// op.as_binop().expect("Could not get binary operation."), -// lhs, -// $lower::new($( self.$low_param ),*).parse(cursor)? -// ).into(); -// } -// &TokenKind::Keyword(op) if $( op == $op )||* => { -// let _ = cursor.next().expect("token disappeared"); -// lhs = BinOp::new( -// op.as_binop().expect("Could not get binary operation."), -// lhs, -// $lower::new($( self.$low_param ),*).parse(cursor)? -// ).into(); -// } -// _ => break -// } -// } -// Ok(lhs) -// } -// } - expression!( MultiplicativeExpression, ExponentiationExpression, From 36ebeb4bd1f7427697d1e756ebbb926db2b66519 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 12:50:29 +0100 Subject: [PATCH 114/289] Tidying up --- boa/src/syntax/parser/cursor.rs | 66 ++++----------------------------- 1 file changed, 7 insertions(+), 59 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 64a4b593d5a..75eef9e7b93 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -14,18 +14,8 @@ use std::io::Read; /// This internal structure gives basic testable operations to the parser. #[derive(Debug)] pub(super) struct Cursor { - /// The tokens being input. - // tokens: &'a [Token], lexer: Lexer, - // The current position within the tokens. - // pos: usize, - - // peeked: Option>, peeked: VecDeque>, - // Values are added to this queue when they are retrieved (next) to allow moving backwards. - // back_queue: VecDeque>, - - // peeked: Option>, } impl Cursor @@ -37,7 +27,6 @@ where Self { lexer: Lexer::new(reader), peeked: VecDeque::new(), - // back_queue: VecDeque::new(), } } @@ -56,56 +45,15 @@ where /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { - match self.peeked.pop_front() { - Some(None) => { - // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { - // self.back_queue.pop_front(); // Remove the value from the front of the queue. - // } - - // self.back_queue.push_back(None); - - return None; - } - Some(Some(token)) => { - // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { - // self.back_queue.pop_front(); // Remove the value from the front of the queue. - // } - - // self.back_queue.push_back(Some(token.clone())); - - return Some(Ok(token)); - } - None => {} // No value has been peeked ahead already so need to go get the next value. + if let Some(t) = self.peeked.pop_front() { + return t.map(|v| Ok(v)); } - loop { - match self.lexer.next() { - Some(Ok(tk)) => { - return Some(Ok(tk)); - - // if tk.kind != TokenKind::LineTerminator { - // // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { - // // self.back_queue.pop_front(); // Remove the value from the front of the queue. - // // } - - // // self.back_queue.push_back(Some(tk.clone())); - - // return Some(Ok(tk)); - // } - } - Some(Err(e)) => { - return Some(Err(ParseError::lex(e))); - } - None => { - // if self.back_queue.len() >= BACK_QUEUE_MAX_LEN { - // self.back_queue.pop_front(); // Remove the value from the front of the queue. - // } - - // self.back_queue.push_back(None); - - return None; - } - } + // No value has been peeked ahead already so need to go get the next value. + if let Some(t) = self.lexer.next() { + Some(t.map_err(|e| ParseError::lex(e))) + } else { + None } } From 4ee16ed7de279cc2ed35341645ad12c3c6ca2d40 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 14:53:14 +0100 Subject: [PATCH 115/289] Clippy is happier, needed to force allow certain closures which were misreported as redundant --- boa/src/syntax/lexer/cursor.rs | 6 +++++- boa/src/syntax/lexer/number.rs | 5 ++--- boa/src/syntax/parser/cursor.rs | 8 ++++---- boa/src/syntax/parser/error.rs | 12 +++++++++--- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 85b01df265b..b7f8cc6a64b 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -69,7 +69,10 @@ where None } }, - Some(v) => v.map(|v| Ok(v)), + Some(v) => { + #[allow(clippy::redundant_closure)] // This closure is miss-reported as redundant. + v.map(|v| Ok(v)) + } } } @@ -193,6 +196,7 @@ where _ => {} } + #[allow(clippy::redundant_closure)] // This closure is miss-reported as redundant. chr.map(|v| Ok(v)) } } diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 3917be2541b..72d5b2dba6d 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -116,9 +116,8 @@ fn check_after_numeric_literal(cursor: &mut Cursor) -> Result<(), Error> where R: Read, { - if cursor.next_is_pred(&|ch: char| { - ch.is_ascii_alphabetic() || ch == '$' || ch == '_' || ch.is_ascii_digit() - })? { + let pred = |ch: char| ch.is_ascii_alphabetic() || ch == '$' || ch == '_' || ch.is_ascii_digit(); + if cursor.next_is_pred(&pred)? { Err(Error::syntax("NumericLiteral token must not be followed by IdentifierStart nor DecimalDigit characters")) } else { Ok(()) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 75eef9e7b93..e15c79a14ed 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -38,20 +38,19 @@ where /// Lexes the next tokens as a regex assuming that the starting '/' has already been consumed. pub(super) fn lex_regex(&mut self, start: Position) -> Result { self.set_goal(InputElement::RegExp); - self.lexer - .lex_slash_token(start) - .map_err(|e| ParseError::lex(e)) + self.lexer.lex_slash_token(start).map_err(|e| e.into()) } /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { if let Some(t) = self.peeked.pop_front() { + #[allow(clippy::redundant_closure)] // This closure is miss-reported as redundant. return t.map(|v| Ok(v)); } // No value has been peeked ahead already so need to go get the next value. if let Some(t) = self.lexer.next() { - Some(t.map_err(|e| ParseError::lex(e))) + Some(t.map_err(|e| e.into())) } else { None } @@ -105,6 +104,7 @@ where self.peeked.push_front(ret.clone()); self.peeked.push_front(temp); + #[allow(clippy::redundant_closure)] // This closure is miss-reported as redundant. ret.map(|token| Ok(token)) } diff --git a/boa/src/syntax/parser/error.rs b/boa/src/syntax/parser/error.rs index 6965ee458a6..a7122648de2 100644 --- a/boa/src/syntax/parser/error.rs +++ b/boa/src/syntax/parser/error.rs @@ -1,6 +1,6 @@ //! Error and result implementation for the parser. use crate::syntax::ast::{position::Position, Node}; -use crate::syntax::lexer::{Error, Token, TokenKind}; +use crate::syntax::lexer::{Error as LexError, Token, TokenKind}; use std::fmt; /// Result of a parsing operation. @@ -16,6 +16,12 @@ impl ErrorContext for Result { } } +impl From for ParseError { + fn from(e: LexError) -> ParseError { + ParseError::lex(e) + } +} + /// `ParseError` is an enum which represents errors encounted during parsing an expression #[derive(Debug)] pub enum ParseError { @@ -33,7 +39,7 @@ pub enum ParseError { /// When there is an abrupt end to the parsing AbruptEnd, Lex { - err: Error, + err: LexError, }, /// Catch all General Error General { @@ -80,7 +86,7 @@ impl ParseError { Self::General { message, position } } - pub(super) fn lex(e: Error) -> Self { + pub(super) fn lex(e: LexError) -> Self { Self::Lex { err: e } } } From 7550cd3859d7893cc486bbb934cbe028d5a2f1a2 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 15:22:59 +0100 Subject: [PATCH 116/289] Tidying cursor, modify peek_more to peek_skip --- boa/src/syntax/parser/cursor.rs | 34 +++++++------------ .../expression/assignment/arrow_function.rs | 2 +- .../parser/expression/assignment/mod.rs | 4 +-- boa/src/syntax/parser/statement/throw/mod.rs | 2 +- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index e15c79a14ed..dcabcba2327 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -58,18 +58,15 @@ where /// Peeks the next token without moving the cursor. pub(super) fn peek(&mut self) -> Option> { - match self.peeked.pop_front() { - Some(None) => { - self.peeked.push_front(None); // Push the value back onto the peeked stack. + if let Some(v) = self.peeked.front() { + if let Some(t) = v { + return Some(Ok(t.clone())); + } else { return None; } - Some(Some(token)) => { - self.peeked.push_front(Some(token.clone())); // Push the value back onto the peeked stack. - return Some(Ok(token)); - } - None => {} // No value has been peeked ahead already so need to go get the next value. } + // No value has been peeked ahead already so need to go get the next value. match self.next() { Some(Ok(token)) => { self.peeked.push_back(Some(token.clone())); @@ -83,14 +80,11 @@ where } } - pub(super) fn peek_more(&mut self, skip: usize) -> Option> { - if skip != 1 { - // I don't believe we ever need to skip more than a single token? - unimplemented!("Attempting to peek ahead more than a single token"); - } - + /// Peeks the token after the next token. + /// i.e. if there are tokens A, B, C and peek() returns A then peek_skip(1) will return B. + pub(super) fn peek_skip(&mut self) -> Option> { // Add elements to the peeked buffer upto the amount required to skip the given amount ahead. - while self.peeked.len() < skip + 1 { + while self.peeked.len() < 2 { match self.lexer.next() { Some(Ok(token)) => self.peeked.push_back(Some(token.clone())), Some(Err(e)) => return Some(Err(ParseError::lex(e))), @@ -180,12 +174,10 @@ where /// It will make sure that the next token is not a line terminator. /// /// It expects that the token stream does not end here. - pub(super) fn peek_expect_no_lineterminator(&mut self, skip: usize) -> Result<(), ParseError> { - let token = if skip == 0 { - self.peek() - } else { - self.peek_more(skip) - }; + /// + /// If skip is true then the token after the peek() token is checked instead. + pub(super) fn peek_expect_no_lineterminator(&mut self, skip: bool) -> Result<(), ParseError> { + let token = if skip { self.peek_skip() } else { self.peek() }; match token { Some(Ok(t)) => { diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index 73dc26839bf..47396014aef 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -96,7 +96,7 @@ where Box::new([FormalParameter::new(param, None, false)]) }; - cursor.peek_expect_no_lineterminator(0)?; + cursor.peek_expect_no_lineterminator(false)?; cursor.expect(Punctuator::Arrow, "arrow function")?; diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 479bfd00819..6c2eb3fe422 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -89,8 +89,8 @@ where TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield) | TokenKind::Keyword(Keyword::Await) => { - if cursor.peek_expect_no_lineterminator(1).is_ok() { - if let Some(tok) = cursor.peek_more(1) { + if cursor.peek_expect_no_lineterminator(true).is_ok() { + if let Some(tok) = cursor.peek_skip() { if tok?.kind() == &TokenKind::Punctuator(Punctuator::Arrow) { return ArrowFunction::new( self.allow_in, diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index dd57ef0efb1..e4f685d0bbc 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -50,7 +50,7 @@ where let _timer = BoaProfiler::global().start_event("ThrowStatement", "Parsing"); cursor.expect(Keyword::Throw, "throw statement")?; - cursor.peek_expect_no_lineterminator(0)?; + cursor.peek_expect_no_lineterminator(false)?; let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; if let Some(tok) = cursor.peek() { From 80c6e19f3f52470c91bc7aa768be771bd92a6228 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 16:30:40 +0100 Subject: [PATCH 117/289] fixed bug where old expect behaviour used on switch --- boa/src/syntax/parser/statement/mod.rs | 3 ++ boa/src/syntax/parser/statement/switch/mod.rs | 20 ++++++++- .../syntax/parser/statement/switch/tests.rs | 44 +++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 6b76cadd907..2beea8370e9 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -241,6 +241,7 @@ impl StatementList { let mut items = Vec::new(); loop { + cursor.skip_line_terminators(); match cursor.peek() { Some(token) => { if break_nodes.contains(&token?.kind()) { @@ -253,6 +254,8 @@ impl StatementList { let item = StatementListItem::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor)?; + + println!("Item: {:?}", item); items.push(item); // move the cursor forward for any consecutive semicolon. diff --git a/boa/src/syntax/parser/statement/switch/mod.rs b/boa/src/syntax/parser/statement/switch/mod.rs index b32b0534845..a6366c263a1 100644 --- a/boa/src/syntax/parser/statement/switch/mod.rs +++ b/boa/src/syntax/parser/statement/switch/mod.rs @@ -62,12 +62,18 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("SwitchStatement", "Parsing"); cursor.expect(Keyword::Switch, "switch statement")?; + cursor.skip_line_terminators(); cursor.expect(Punctuator::OpenParen, "switch statement")?; + cursor.skip_line_terminators(); let condition = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + cursor.skip_line_terminators(); + cursor.expect(Punctuator::CloseParen, "switch statement")?; + cursor.skip_line_terminators(); + let (cases, default) = CaseBlock::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; @@ -113,18 +119,26 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let mut cases = Vec::::new(); let mut default: Option = None; - + + cursor.skip_line_terminators(); cursor.expect(Punctuator::OpenBlock, "switch start case block")?; loop { + cursor.skip_line_terminators(); match cursor.expect(Keyword::Case, "switch case: block") { Ok(_) => { + cursor.skip_line_terminators(); + // Case statement. let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; + cursor.skip_line_terminators(); + cursor.expect(Punctuator::Colon, "switch case block start")?; + cursor.skip_line_terminators(); + let statement_list = StatementList::new( self.allow_yield, self.allow_await, @@ -154,6 +168,9 @@ where } cursor.expect(Punctuator::Colon, "switch default case block start")?; + + cursor.skip_line_terminators(); + let statement_list = StatementList::new( self.allow_yield, self.allow_await, @@ -174,6 +191,7 @@ where context: _, }) => { // End of switch block. + cursor.next().expect("Switch close block symbol vanished"); // Consume the switch close block. break; } Err(e) => { diff --git a/boa/src/syntax/parser/statement/switch/tests.rs b/boa/src/syntax/parser/statement/switch/tests.rs index 568d94a7705..124ecaefce1 100644 --- a/boa/src/syntax/parser/statement/switch/tests.rs +++ b/boa/src/syntax/parser/statement/switch/tests.rs @@ -97,3 +97,47 @@ fn check_switch_seperated_defaults() { "#, ); } + +/// Example of JS code https://jsfiddle.net/zq6jx47h/4/. +#[test] +fn check_seperated_switch() { + check_invalid( + r#" + let a = 10; + + switch + + (a) + + { + + case + + 5 + + : + + console.log(5); + + break; + + case + + 10 + + : + + console.log(10); + + break; + + default + + : + + console.log("Default") + + } + "#, + ); +} From 2ee2e81356d47fe33bf2c4821dbd5414bdde3a32 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 16:36:02 +0100 Subject: [PATCH 118/289] Fmt, clippy happy, all tests pass --- boa/src/syntax/lexer/mod.rs | 9 --------- boa/src/syntax/parser/statement/mod.rs | 7 ++----- boa/src/syntax/parser/statement/switch/mod.rs | 7 +++++-- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index c75f6771c4b..1934259a68c 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -267,12 +267,3 @@ where } } } - -// impl Tokenizer for Lexer { -// fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> io::Result -// where -// R: Read, -// { - -// } -// } diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 2beea8370e9..eb33ca0ffa3 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -35,10 +35,7 @@ use self::{ variable::VariableStatement, }; -use super::{ - expression::Expression, AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, ParseResult, - TokenParser, -}; +use super::{AllowAwait, AllowReturn, AllowYield, Cursor, ParseError, TokenParser}; use crate::syntax::lexer::TokenKind; use crate::{ @@ -254,7 +251,7 @@ impl StatementList { let item = StatementListItem::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor)?; - + println!("Item: {:?}", item); items.push(item); diff --git a/boa/src/syntax/parser/statement/switch/mod.rs b/boa/src/syntax/parser/statement/switch/mod.rs index a6366c263a1..9d9fefb3fb7 100644 --- a/boa/src/syntax/parser/statement/switch/mod.rs +++ b/boa/src/syntax/parser/statement/switch/mod.rs @@ -119,7 +119,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let mut cases = Vec::::new(); let mut default: Option = None; - + cursor.skip_line_terminators(); cursor.expect(Punctuator::OpenBlock, "switch start case block")?; @@ -159,6 +159,9 @@ where context: _, }) => { // Default statement. + // Consume the default token. + cursor.next().expect("Default token vanished")?; + if default.is_some() { // If default has already been defined then it cannot be defined again and to do so is an error. return Err(ParseError::unexpected( @@ -191,7 +194,7 @@ where context: _, }) => { // End of switch block. - cursor.next().expect("Switch close block symbol vanished"); // Consume the switch close block. + cursor.next().expect("Switch close block symbol vanished")?; // Consume the switch close block. break; } Err(e) => { From 2aac403e75dcd07a23939123e7d6d8ca76f09ed1 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 18:44:19 +0100 Subject: [PATCH 119/289] Attempting to fix issues with benchmarks --- boa/src/exec/tests.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/boa/src/exec/tests.rs b/boa/src/exec/tests.rs index f76240b1016..0f179b00d24 100644 --- a/boa/src/exec/tests.rs +++ b/boa/src/exec/tests.rs @@ -1131,3 +1131,21 @@ fn check_this_binding_in_object_literal() { assert_eq!(forward(&mut engine, init), "8"); } + +#[test] +fn array_creation_benchmark() { + let realm = Realm::create(); + let mut engine = Interpreter::new(realm); + let init = r#" + (function(){ + let testArr = []; + for (let a = 0; a <= 500; a++) { + testArr[a] = ('p' + a); + } + + return testArr; + })(); + "#; + + assert_eq!(forward(&mut engine, init), "[ p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31, p32, p33, p34, p35, p36, p37, p38, p39, p40, p41, p42, p43, p44, p45, p46, p47, p48, p49, p50, p51, p52, p53, p54, p55, p56, p57, p58, p59, p60, p61, p62, p63, p64, p65, p66, p67, p68, p69, p70, p71, p72, p73, p74, p75, p76, p77, p78, p79, p80, p81, p82, p83, p84, p85, p86, p87, p88, p89, p90, p91, p92, p93, p94, p95, p96, p97, p98, p99, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111, p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127, p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143, p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159, p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175, p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191, p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207, p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223, p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239, p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255, p256, p257, p258, p259, p260, p261, p262, p263, p264, p265, p266, p267, p268, p269, p270, p271, p272, p273, p274, p275, p276, p277, p278, p279, p280, p281, p282, p283, p284, p285, p286, p287, p288, p289, p290, p291, p292, p293, p294, p295, p296, p297, p298, p299, p300, p301, p302, p303, p304, p305, p306, p307, p308, p309, p310, p311, p312, p313, p314, p315, p316, p317, p318, p319, p320, p321, p322, p323, p324, p325, p326, p327, p328, p329, p330, p331, p332, p333, p334, p335, p336, p337, p338, p339, p340, p341, p342, p343, p344, p345, p346, p347, p348, p349, p350, p351, p352, p353, p354, p355, p356, p357, p358, p359, p360, p361, p362, p363, p364, p365, p366, p367, p368, p369, p370, p371, p372, p373, p374, p375, p376, p377, p378, p379, p380, p381, p382, p383, p384, p385, p386, p387, p388, p389, p390, p391, p392, p393, p394, p395, p396, p397, p398, p399, p400, p401, p402, p403, p404, p405, p406, p407, p408, p409, p410, p411, p412, p413, p414, p415, p416, p417, p418, p419, p420, p421, p422, p423, p424, p425, p426, p427, p428, p429, p430, p431, p432, p433, p434, p435, p436, p437, p438, p439, p440, p441, p442, p443, p444, p445, p446, p447, p448, p449, p450, p451, p452, p453, p454, p455, p456, p457, p458, p459, p460, p461, p462, p463, p464, p465, p466, p467, p468, p469, p470, p471, p472, p473, p474, p475, p476, p477, p478, p479, p480, p481, p482, p483, p484, p485, p486, p487, p488, p489, p490, p491, p492, p493, p494, p495, p496, p497, p498, p499, p500 ]"); +} From d040362806fea10ce6d9443d2772f2ca9012d846 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 19:25:14 +0100 Subject: [PATCH 120/289] Fixed bug in array decl not allowing line terminators --- boa/src/exec/tests.rs | 34 +++++++++++++++++++ .../parser/expression/assignment/mod.rs | 1 - .../primary/array_initializer/mod.rs | 2 ++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/boa/src/exec/tests.rs b/boa/src/exec/tests.rs index 0f179b00d24..a4aeea7b9a8 100644 --- a/boa/src/exec/tests.rs +++ b/boa/src/exec/tests.rs @@ -1149,3 +1149,37 @@ fn array_creation_benchmark() { assert_eq!(forward(&mut engine, init), "[ p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31, p32, p33, p34, p35, p36, p37, p38, p39, p40, p41, p42, p43, p44, p45, p46, p47, p48, p49, p50, p51, p52, p53, p54, p55, p56, p57, p58, p59, p60, p61, p62, p63, p64, p65, p66, p67, p68, p69, p70, p71, p72, p73, p74, p75, p76, p77, p78, p79, p80, p81, p82, p83, p84, p85, p86, p87, p88, p89, p90, p91, p92, p93, p94, p95, p96, p97, p98, p99, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111, p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127, p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143, p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159, p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175, p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191, p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207, p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223, p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239, p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255, p256, p257, p258, p259, p260, p261, p262, p263, p264, p265, p266, p267, p268, p269, p270, p271, p272, p273, p274, p275, p276, p277, p278, p279, p280, p281, p282, p283, p284, p285, p286, p287, p288, p289, p290, p291, p292, p293, p294, p295, p296, p297, p298, p299, p300, p301, p302, p303, p304, p305, p306, p307, p308, p309, p310, p311, p312, p313, p314, p315, p316, p317, p318, p319, p320, p321, p322, p323, p324, p325, p326, p327, p328, p329, p330, p331, p332, p333, p334, p335, p336, p337, p338, p339, p340, p341, p342, p343, p344, p345, p346, p347, p348, p349, p350, p351, p352, p353, p354, p355, p356, p357, p358, p359, p360, p361, p362, p363, p364, p365, p366, p367, p368, p369, p370, p371, p372, p373, p374, p375, p376, p377, p378, p379, p380, p381, p382, p383, p384, p385, p386, p387, p388, p389, p390, p391, p392, p393, p394, p395, p396, p397, p398, p399, p400, p401, p402, p403, p404, p405, p406, p407, p408, p409, p410, p411, p412, p413, p414, p415, p416, p417, p418, p419, p420, p421, p422, p423, p424, p425, p426, p427, p428, p429, p430, p431, p432, p433, p434, p435, p436, p437, p438, p439, p440, p441, p442, p443, p444, p445, p446, p447, p448, p449, p450, p451, p452, p453, p454, p455, p456, p457, p458, p459, p460, p461, p462, p463, p464, p465, p466, p467, p468, p469, p470, p471, p472, p473, p474, p475, p476, p477, p478, p479, p480, p481, p482, p483, p484, p485, p486, p487, p488, p489, p490, p491, p492, p493, p494, p495, p496, p497, p498, p499, p500 ]"); } + +#[test] +fn array_pop_benchmark() { + let realm = Realm::create(); + let mut engine = Interpreter::new(realm); + let init = r#" + (function(){ + let testArray = [83, 93, 27, 29, 2828, 234, 23, 56, 32, 56, 67, 77, 32, + 45, 93, 17, 28, 83, 62, 99, 36, 28, 93, 27, 29, 2828, + 234, 23, 56, 32, 56, 67, 77, 32, 45, 93, 17, 28, 83, 62, + 99, 36, 28, 93, 27, 29, 2828, 234, 23, 56, 32, 56, 67, + 77, 32, 45, 93, 17, 28, 83, 62, 99, 36, 28, 93, 27, 29, + 2828, 234, 23, 56, 32, 56, 67, 77, 32, 45, 93, 17, 28, + 83, 62, 99, 36, 28, 93, 27, 29, 2828, 234, 23, 56, 32, + 56, 67, 77, 32, 45, 93, 17, 28, 83, 62, 99, 36, 28, 93, + 27, 29, 2828, 234, 23, 56, 32, 56, 67, 77, 32, 45, 93, + 17, 28, 83, 62, 99, 36, 28, 93, 27, 29, 2828, 234, 23, + 56, 32, 56, 67, 77, 32, 45, 93, 17, 28, 83, 62, 99, 36, + 28, 93, 27, 29, 2828, 234, 23, 56, 32, 56, 67, 77, 32, + 45, 93, 17, 28, 83, 62, 99, 36, 28, 93, 27, 29, 2828, 234, + 23, 56, 32, 56, 67, 77, 32, 45, 93, 17, 28, 83, 62, 99, + 36, 28, 93, 27, 29, 2828, 234, 23, 56, 32, 56, 67, 77, 32, + 45, 93, 17, 28, 83, 62, 99, 36, 28]; + + while (testArray.length > 0) { + testArray.pop(); + } + + return testArray; + })(); + "#; + + assert_eq!(forward(&mut engine, init), "[]"); +} \ No newline at end of file diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 6c2eb3fe422..51b93100421 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -80,7 +80,6 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("AssignmentExpression", "Parsing"); - // cursor.set_goal(InputElement::RegExpOrTemplateTail); cursor.set_goal(InputElement::Div); // Arrow function diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index 6d70423b4a6..dd510d7ed13 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -65,6 +65,7 @@ where let mut elements = Vec::new(); loop { + cursor.skip_line_terminators(); // TODO: Support all features. while cursor.next_if(Punctuator::Comma).is_some() { elements.push(Node::Const(Const::Undefined)); @@ -86,6 +87,7 @@ where .parse(cursor)?, ); } + cursor.skip_line_terminators(); cursor.next_if(Punctuator::Comma); } From 5d46cdd0d367131e75b0083422ac85493d315fdf Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 19:26:16 +0100 Subject: [PATCH 121/289] fmt --- boa/src/exec/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/src/exec/tests.rs b/boa/src/exec/tests.rs index a4aeea7b9a8..6407e7e1d90 100644 --- a/boa/src/exec/tests.rs +++ b/boa/src/exec/tests.rs @@ -1182,4 +1182,4 @@ fn array_pop_benchmark() { "#; assert_eq!(forward(&mut engine, init), "[]"); -} \ No newline at end of file +} From 76e8431d5c7b8e4ebdb20499292699bedc46e08a Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sat, 4 Jul 2020 19:48:26 +0100 Subject: [PATCH 122/289] Fixed bug with line terminators being rejected in object arguments --- boa/src/exec/tests.rs | 19 ++++++++++++++++++- .../expression/left_hand_side/arguments.rs | 1 + .../expression/left_hand_side/member.rs | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/boa/src/exec/tests.rs b/boa/src/exec/tests.rs index 6407e7e1d90..ee8c5875146 100644 --- a/boa/src/exec/tests.rs +++ b/boa/src/exec/tests.rs @@ -2,7 +2,7 @@ use crate::{ builtins::{Number, Value}, exec, exec::Interpreter, - forward, + forward, forward_val, realm::Realm, }; @@ -1183,3 +1183,20 @@ fn array_pop_benchmark() { assert_eq!(forward(&mut engine, init), "[]"); } + +#[test] +fn number_object_access_benchmark() { + let realm = Realm::create(); + let mut engine = Interpreter::new(realm); + let init = r#" + new Number( + new Number( + new Number( + new Number(100).valueOf() - 10.5 + ).valueOf() + 100 + ).valueOf() * 1.6 + ) + "#; + + assert!(forward_val(&mut engine, init).is_ok()); +} diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index e4aca6a61ba..4c581e33d6f 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -62,6 +62,7 @@ where cursor.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); loop { + cursor.skip_line_terminators(); let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; match next_token.kind() { diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 60b63a1c238..26069781382 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -63,7 +63,7 @@ where let mut lhs = if cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() == &TokenKind::Keyword(Keyword::New) { - let _ = cursor.next().expect("keyword disappeared"); + let _ = cursor.next().expect("new keyword disappeared"); let lhs = self.parse(cursor)?; let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; let call_node = Call::new(lhs, args); From c8966a148f0facd1d5fed13481ba44b439adebc8 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 10:05:43 +0100 Subject: [PATCH 123/289] Update boa/src/syntax/lexer/number.rs Co-authored-by: HalidOdat --- boa/src/syntax/lexer/number.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 72d5b2dba6d..82588cc7ca3 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -352,7 +352,7 @@ impl Tokenizer for NumberLiteral { result = result * b + digit; } - if exp_str == "" { + if exp_str.is_empty() { Numeric::Rational( result ) From 93e39b4361fd2cc3e930a56de3c649dcef1cc224 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 10:05:53 +0100 Subject: [PATCH 124/289] Update boa/src/builtins/array/tests.rs Co-authored-by: HalidOdat --- boa/src/builtins/array/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/src/builtins/array/tests.rs b/boa/src/builtins/array/tests.rs index 5441ab1ad7e..85a2f197cbf 100644 --- a/boa/src/builtins/array/tests.rs +++ b/boa/src/builtins/array/tests.rs @@ -417,7 +417,7 @@ fn fill_obj_ref() { // test object reference forward(&mut engine, "a = (new Array(3)).fill({});"); forward(&mut engine, "a[0].hi = 'hi';"); - assert_eq!(forward(&mut engine, "a[0].hi"), String::from("hi")); + assert_eq!(forward(&mut engine, "a[0].hi"), "hi"); } #[test] From a723479f11a9ed612ddafb6bd114417dc06eb93a Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 10:06:12 +0100 Subject: [PATCH 125/289] Update boa/src/syntax/lexer/cursor.rs Co-authored-by: HalidOdat --- boa/src/syntax/lexer/cursor.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index b7f8cc6a64b..65143c4d922 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -69,10 +69,7 @@ where None } }, - Some(v) => { - #[allow(clippy::redundant_closure)] // This closure is miss-reported as redundant. - v.map(|v| Ok(v)) - } + Some(v) => v.map(Ok) } } From 526674f6a06bae58948601da6c44d7117718c565 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 10:06:51 +0100 Subject: [PATCH 126/289] Update boa/src/syntax/parser/cursor.rs Co-authored-by: HalidOdat --- boa/src/syntax/parser/cursor.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index dcabcba2327..930a507439a 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -44,8 +44,7 @@ where /// Moves the cursor to the next token and returns the token. pub(super) fn next(&mut self) -> Option> { if let Some(t) = self.peeked.pop_front() { - #[allow(clippy::redundant_closure)] // This closure is miss-reported as redundant. - return t.map(|v| Ok(v)); + return t.map(Ok); } // No value has been peeked ahead already so need to go get the next value. From f4c245c5946ac890c65579e6ce1274ce0d5e6afa Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 10:07:38 +0100 Subject: [PATCH 127/289] Update boa/src/syntax/parser/expression/assignment/mod.rs Co-authored-by: HalidOdat --- boa/src/syntax/parser/expression/assignment/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 51b93100421..b74bdf51734 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -122,8 +122,8 @@ where let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - if let Some(tok) = cursor.peek() { - match tok?.kind() { + if let Some(tok) = cursor.peek()? { + match tok.kind() { TokenKind::Punctuator(Punctuator::Assign) => { cursor.next(); // Consume the token. lhs = Assign::new(lhs, self.parse(cursor)?).into(); From 7918bf810b5a2108d8b2bbfd585c9556fff953d3 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 10:08:16 +0100 Subject: [PATCH 128/289] Update boa/src/syntax/lexer/cursor.rs Co-authored-by: HalidOdat --- boa/src/syntax/lexer/cursor.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 65143c4d922..5c5c5cedf12 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -193,8 +193,7 @@ where _ => {} } - #[allow(clippy::redundant_closure)] // This closure is miss-reported as redundant. - chr.map(|v| Ok(v)) + chr.map(Ok) } } From ef01d58aa15f8aaa36cdee214b3e46935e1ad2e9 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 10:28:02 +0100 Subject: [PATCH 129/289] Fixes as per PR review --- boa/src/syntax/lexer/comment.rs | 8 ++++---- boa/src/syntax/lexer/cursor.rs | 2 +- boa/src/syntax/lexer/identifier.rs | 5 +---- boa/src/syntax/lexer/tests.rs | 14 -------------- boa/src/syntax/parser/expression/assignment/mod.rs | 4 ++-- 5 files changed, 8 insertions(+), 25 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index d0578348064..4116f730920 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -13,8 +13,8 @@ pub(super) struct SingleLineComment; /// - [ECMAScript reference][spec] /// - [MDN documentation][mdn] /// -/// [spec]: -/// [mdn]: +/// [spec]: https://tc39.es/ecma262/#sec-comments +/// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar impl Tokenizer for SingleLineComment { fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where @@ -47,8 +47,8 @@ impl Tokenizer for SingleLineComment { /// - [ECMAScript reference][spec] /// - [MDN documentation][mdn] /// -/// [spec]: -/// [mdn]: +/// [spec]: https://tc39.es/ecma262/#sec-comments +/// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar pub(super) struct BlockComment; impl Tokenizer for BlockComment { fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 5c5c5cedf12..38a29992b0b 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -69,7 +69,7 @@ where None } }, - Some(v) => v.map(Ok) + Some(v) => v.map(Ok), } } diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index 0d5a00a7d31..d29bb47091f 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -5,9 +5,7 @@ use crate::syntax::lexer::{Token, TokenKind}; use std::io::Read; use std::str::FromStr; -/// Identifier or keyword lexing. -/// -/// This currently includes boolean/NaN lexing. +/// Identifier lexing. /// /// More information: /// - [ECMAScript reference][spec] @@ -56,7 +54,6 @@ impl Tokenizer for Identifier { "true" => TokenKind::BooleanLiteral(true), "false" => TokenKind::BooleanLiteral(false), "null" => TokenKind::NullLiteral, - "NaN" => TokenKind::NumericLiteral(Numeric::Rational(f64::NAN)), slice => { if let Ok(keyword) = FromStr::from_str(slice) { TokenKind::Keyword(keyword) diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index f496d6986e8..e9bab62a0ef 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -323,20 +323,6 @@ fn check_decrement_advances_lexer_2_places() { ); } -#[test] -fn check_nan() { - let mut lexer = Lexer::new(&b"let a = NaN;"[0..]); - match lexer.nth(3) { - None | Some(Err(_)) => panic!("No token found when expecting NaN"), - Some(Ok(token)) => match token.kind() { - TokenKind::NumericLiteral(Numeric::Rational(a)) => { - assert!(a.is_nan()); - } - ref other => panic!("Incorrect token kind found for NaN: {}", other), - }, - } -} - #[test] fn single_int() { let mut lexer = Lexer::new(&b"52"[0..]); diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index b74bdf51734..51b93100421 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -122,8 +122,8 @@ where let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - if let Some(tok) = cursor.peek()? { - match tok.kind() { + if let Some(tok) = cursor.peek() { + match tok?.kind() { TokenKind::Punctuator(Punctuator::Assign) => { cursor.next(); // Consume the token. lhs = Assign::new(lhs, self.parse(cursor)?).into(); From 86dcaa457453c14ebe5958069a3da112986b98de Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 11:23:27 +0100 Subject: [PATCH 130/289] Cleaning up number lexing --- boa/src/syntax/lexer/number.rs | 50 ++++++++-------------------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 82588cc7ca3..4dc6ad54aee 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -217,7 +217,7 @@ impl Tokenizer for NumberLiteral { // Consume digits until a non-digit character is encountered or all the characters are consumed. cursor.take_until_pred(&mut buf, &|c: char| c.is_digit(kind.base()))?; - let mut exp_str = String::new(); + let exp_str = &mut String::new(); // The non-digit character could be: // 'n' To indicate a BigIntLiteralSuffix. @@ -250,47 +250,29 @@ impl Tokenizer for NumberLiteral { // The non-digit character at this point must be an 'e' or 'E' to indicate an Exponent Part. // Another '.' or 'n' is not allowed. match cursor.peek() { - Some(Ok('n')) => { - // Found BigIntLiteralSuffix after non-integer number - - // Finish lexing number. - - // return Err(Error::syntax( - // "Found BigIntLiteralSuffix after non-integer number", - // )); - } - Some(Ok('.')) => { - // Found second . within decimal number - // Finish lexing number. - - // return Err(Error::syntax("Found second . within decimal number")); - } Some(Ok('e')) | Some(Ok('E')) => { // Consume the ExponentIndicator. cursor.next(); - take_signed_integer(&mut exp_str, cursor, &kind)?; + buf.push('E'); + + take_signed_integer(&mut buf, cursor, &kind)?; } - Some(Err(_e)) => { - // todo!(); + Some(Err(e)) => { + return Err(e.into()); } Some(Ok(_)) | None => { // Finished lexing. - kind = NumericKind::Rational; } } } } Some(Ok('e')) | Some(Ok('E')) => { - // Consume the ExponentIndicator. - cursor.next(); - - // buf.push('e'); - - take_signed_integer(&mut exp_str, cursor, &kind)?; + cursor.next(); // Consume the ExponentIndicator. + take_signed_integer(exp_str, cursor, &kind)?; } - Some(Err(_e)) => { - // todo!(); + Some(Err(e)) => { + return Err(e.into()); } Some(Ok(_)) | None => { @@ -307,17 +289,7 @@ impl Tokenizer for NumberLiteral { ) } NumericKind::Rational /* base: 10 */ => { - let r = f64::from_str(&buf).map_err(|_| Error::syntax("Could not convert value to f64"))?; - if exp_str == "" { - Numeric::Rational( - r - ) - } else { - let n = f64::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; - Numeric::Rational( - r * f64::powf(10.0, n) - ) - } + Numeric::Rational(f64::from_str(&buf).unwrap()) } NumericKind::Integer(base) => { if let Ok(num) = i32::from_str_radix(&buf, base as u32) { From e3962d9fd8162a3f288616abcb629e648166b57b Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 11:38:44 +0100 Subject: [PATCH 131/289] Tidying, removed .unwrap()'s --- boa/src/syntax/lexer/identifier.rs | 1 - boa/src/syntax/lexer/number.rs | 25 ++++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index d29bb47091f..19351b1017a 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -1,6 +1,5 @@ use super::{Cursor, Error, Tokenizer}; use crate::syntax::ast::{Position, Span}; -use crate::syntax::lexer::token::Numeric; use crate::syntax::lexer::{Token, TokenKind}; use std::io::Read; use std::str::FromStr; diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 4dc6ad54aee..0d0297823d7 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -184,8 +184,7 @@ impl Tokenizer for NumberLiteral { // Remove the initial '0' from buffer. buf.pop(); - let char = cursor.next().unwrap().unwrap(); - buf.push(char); + buf.push(cursor.next().expect("'0' character vanished")?); kind = NumericKind::Integer(8); } @@ -198,8 +197,7 @@ impl Tokenizer for NumberLiteral { "Leading 0's are not allowed in strict mode.", )); } else { - let char = cursor.next().unwrap().unwrap(); - buf.push(char); + buf.push(cursor.next().expect("Number digit vanished")?); } } // Else indicates that the symbol is a non-number. } @@ -234,14 +232,12 @@ impl Tokenizer for NumberLiteral { kind = kind.to_bigint(); } Some(Ok('.')) => { - // Consume the . - if kind.base() == 10 { // Only base 10 numbers can have a decimal seperator. // Number literal lexing finished if a . is found for a number in a different base. cursor.next(); - buf.push('.'); + buf.push('.'); // Consume the . kind = NumericKind::Rational; // Consume digits until a non-digit character is encountered or all the characters are consumed. @@ -289,11 +285,14 @@ impl Tokenizer for NumberLiteral { ) } NumericKind::Rational /* base: 10 */ => { - Numeric::Rational(f64::from_str(&buf).unwrap()) + match f64::from_str(&buf) { + Ok(val) => Numeric::Rational(val), + Err(e) => return Err(Error::syntax(format!("Unable to parse rational number, reason: {}", e.to_string()))) + } } NumericKind::Integer(base) => { if let Ok(num) = i32::from_str_radix(&buf, base as u32) { - if exp_str == "" { + if exp_str.is_empty() { Numeric::Integer(num) } else { let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; @@ -320,8 +319,12 @@ impl Tokenizer for NumberLiteral { let b = f64::from(base); let mut result = 0.0_f64; for c in buf.chars() { - let digit = f64::from(c.to_digit(base as u32).unwrap()); - result = result * b + digit; + if let Some(val) = c.to_digit(base as u32) { + let digit = f64::from(val); + result = result * b + digit; + } else { + return Err(Error::syntax("Unrecognised numerical digit encountered")); + } } if exp_str.is_empty() { From db3df443be220eb6bba4d1bb0382323184fd23df Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 11:47:21 +0100 Subject: [PATCH 132/289] Tidy up based on PR reviews --- boa/src/syntax/lexer/error.rs | 15 ------------- boa/src/syntax/lexer/identifier.rs | 34 ++++++++++++------------------ boa/src/syntax/lexer/number.rs | 4 ++-- 3 files changed, 16 insertions(+), 37 deletions(-) diff --git a/boa/src/syntax/lexer/error.rs b/boa/src/syntax/lexer/error.rs index 3c1b65e2a3b..10a0c3a9249 100644 --- a/boa/src/syntax/lexer/error.rs +++ b/boa/src/syntax/lexer/error.rs @@ -4,7 +4,6 @@ use std::{error::Error as StdError, fmt, io}; pub enum Error { IO(io::Error), Syntax(Box), - StrictMode(Box), // Not 100% decided on this name. } impl From for Error { @@ -21,18 +20,6 @@ impl Error { { Self::Syntax(err.into()) } - - /// Creates a new StrictMode error. - /// - /// This error is used to represent the case where a piece of javascript - /// cannot be lexed/parsed because it is in invalid when strict mdoe is - /// enabled. - pub(super) fn strict(err: M) -> Self - where - M: Into>, - { - Self::StrictMode(err.into()) - } } impl fmt::Display for Error { @@ -40,7 +27,6 @@ impl fmt::Display for Error { match self { Self::IO(e) => write!(f, "I/O error: {}", e), Self::Syntax(e) => write!(f, "Syntax Error: {}", e), - Self::StrictMode(e) => write!(f, "Strict Mode Error: {}", e), } } } @@ -50,7 +36,6 @@ impl StdError for Error { match self { Self::IO(err) => Some(err), Self::Syntax(_) => None, - Self::StrictMode(_) => None, } } } diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index 19351b1017a..4deec6bedeb 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -1,6 +1,8 @@ use super::{Cursor, Error, Tokenizer}; -use crate::syntax::ast::{Position, Span}; -use crate::syntax::lexer::{Token, TokenKind}; +use crate::syntax::{ + ast::{Position, Span}, + lexer::{Token, TokenKind}, +}; use std::io::Read; use std::str::FromStr; @@ -10,8 +12,8 @@ use std::str::FromStr; /// - [ECMAScript reference][spec] /// - [MDN documentation][mdn] /// -/// [spec]: -/// [mdn]: +/// [spec]: https://tc39.es/ecma262/#prod-Identifier +/// [mdn]: https://developer.mozilla.org/en-US/docs/Glossary/Identifier #[derive(Debug, Clone, Copy)] pub(super) struct Identifier { init: char, @@ -31,24 +33,16 @@ impl Tokenizer for Identifier { { let mut buf = self.init.to_string(); - loop { - match cursor.peek() { - None => { - break; - } - Some(Ok(c)) => { - if c.is_alphabetic() || c.is_digit(10) || c == '_' { - let ch = cursor.next().unwrap()?; - buf.push(ch); - } else { - break; - } - } - Some(Err(_e)) => { - // TODO handle error. - } + while let Some(c) = cursor.peek() { + let c = c?; + if c.is_alphabetic() || c.is_digit(10) || c == '_' { + let ch = cursor.next().unwrap()?; + buf.push(ch); + } else { + break; } } + let tk = match buf.as_str() { "true" => TokenKind::BooleanLiteral(true), "false" => TokenKind::BooleanLiteral(false), diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 0d0297823d7..a888481fc28 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -177,7 +177,7 @@ impl Tokenizer for NumberLiteral { // LegacyOctalIntegerLiteral if self.strict_mode { // LegacyOctalIntegerLiteral is forbidden with strict mode true. - return Err(Error::strict( + return Err(Error::syntax( "Implicit octal literals are not allowed in strict mode.", )); } else { @@ -193,7 +193,7 @@ impl Tokenizer for NumberLiteral { // so therefore this must be a number with an unneeded leading 0. This is // forbidden in strict mode. if self.strict_mode { - return Err(Error::strict( + return Err(Error::syntax( "Leading 0's are not allowed in strict mode.", )); } else { From 009e8782175b761c31b02c1de3ff87424cd60e41 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 12:29:55 +0100 Subject: [PATCH 133/289] Tidying --- boa/src/syntax/lexer/identifier.rs | 4 +++- boa/src/syntax/parser/cursor.rs | 13 +++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index 4deec6bedeb..496e5f1bd96 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -36,7 +36,9 @@ impl Tokenizer for Identifier { while let Some(c) = cursor.peek() { let c = c?; if c.is_alphabetic() || c.is_digit(10) || c == '_' { - let ch = cursor.next().unwrap()?; + let ch = cursor + .next() + .expect("Character in identifier has vanished")?; buf.push(ch); } else { break; diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 930a507439a..ae4b2185bd2 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -91,14 +91,19 @@ where } } - let temp = self.peeked.pop_front().unwrap(); - let ret = self.peeked.pop_front().unwrap(); + let temp = self + .peeked + .pop_front() + .expect("Front peeked value has vanished"); + let ret = self + .peeked + .pop_front() + .expect("Back peeked value has vanished"); self.peeked.push_front(ret.clone()); self.peeked.push_front(temp); - #[allow(clippy::redundant_closure)] // This closure is miss-reported as redundant. - ret.map(|token| Ok(token)) + ret.map(Ok) } /// Takes the given token and pushes it back onto the parser token queue (at the front so the token will be returned on next .peek()). From 7367cf8dbea5c6d506416db8d919b184172a5bbb Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 12:51:45 +0100 Subject: [PATCH 134/289] Used transpose as per PR comment --- boa/src/syntax/parser/expression/assignment/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 51b93100421..3c5204c62dd 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -122,8 +122,8 @@ where let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - if let Some(tok) = cursor.peek() { - match tok?.kind() { + if let Some(tok) = cursor.peek().transpose()? { + match tok.kind() { TokenKind::Punctuator(Punctuator::Assign) => { cursor.next(); // Consume the token. lhs = Assign::new(lhs, self.parse(cursor)?).into(); From 45e7d154467c095e2579c5e64d79796f70b48796 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 14:21:44 +0100 Subject: [PATCH 135/289] Removed error returns in number in favour of immediate panic --- boa/src/syntax/lexer/number.rs | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index a888481fc28..1ab94f1dd56 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -284,12 +284,7 @@ impl Tokenizer for NumberLiteral { BigInt::from_string_radix(&buf, base as u32).expect("Could not convert to BigInt") ) } - NumericKind::Rational /* base: 10 */ => { - match f64::from_str(&buf) { - Ok(val) => Numeric::Rational(val), - Err(e) => return Err(Error::syntax(format!("Unable to parse rational number, reason: {}", e.to_string()))) - } - } + NumericKind::Rational /* base: 10 */ => Numeric::Rational(f64::from_str(&buf).expect("Failed to parse float after checks")), NumericKind::Integer(base) => { if let Ok(num) = i32::from_str_radix(&buf, base as u32) { if exp_str.is_empty() { @@ -319,12 +314,8 @@ impl Tokenizer for NumberLiteral { let b = f64::from(base); let mut result = 0.0_f64; for c in buf.chars() { - if let Some(val) = c.to_digit(base as u32) { - let digit = f64::from(val); - result = result * b + digit; - } else { - return Err(Error::syntax("Unrecognised numerical digit encountered")); - } + let digit = f64::from(c.to_digit(base as u32).expect("Couldn't parse digit after already checking validity")); + result = result * b + digit; } if exp_str.is_empty() { @@ -332,10 +323,7 @@ impl Tokenizer for NumberLiteral { result ) } else { - let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; - Numeric::Rational( - result * f64::powi(10.0, n) - ) + Numeric::Rational( result * f64::powi(10.0, i32::from_str(&exp_str).expect("Couldn't parse number after already checking validity"))) } } } From c224cd40c23a4f7533e4141d217e32e6801004bc Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Sun, 5 Jul 2020 21:15:30 +0100 Subject: [PATCH 136/289] Started refractor for cursor to return Result> --- boa/benches/lexer.rs | 12 +-- boa/src/syntax/lexer/comment.rs | 26 ++--- boa/src/syntax/lexer/cursor.rs | 86 ++++++----------- boa/src/syntax/lexer/identifier.rs | 11 +-- boa/src/syntax/lexer/mod.rs | 85 ++++++++-------- boa/src/syntax/lexer/number.rs | 48 ++++------ boa/src/syntax/lexer/operator.rs | 41 ++++---- boa/src/syntax/lexer/regex.rs | 38 +++----- boa/src/syntax/lexer/string.rs | 12 +-- boa/src/syntax/lexer/template.rs | 7 +- boa/src/syntax/lexer/tests.rs | 27 +++--- boa/src/syntax/parser/cursor.rs | 96 ++++++++----------- .../expression/assignment/arrow_function.rs | 4 +- .../expression/assignment/conditional.rs | 4 +- .../expression/assignment/exponentiation.rs | 8 +- .../parser/expression/assignment/mod.rs | 8 +- .../expression/left_hand_side/arguments.rs | 6 +- .../parser/expression/left_hand_side/call.rs | 18 ++-- .../expression/left_hand_side/member.rs | 12 +-- .../parser/expression/left_hand_side/mod.rs | 12 +-- boa/src/syntax/parser/expression/mod.rs | 4 +- .../primary/array_initializer/mod.rs | 8 +- .../syntax/parser/expression/primary/mod.rs | 2 +- .../primary/object_initializer/mod.rs | 26 ++--- boa/src/syntax/parser/expression/unary.rs | 4 +- boa/src/syntax/parser/expression/update.rs | 14 +-- boa/src/syntax/parser/function/mod.rs | 12 +-- boa/src/syntax/parser/mod.rs | 2 +- boa/src/syntax/parser/statement/block/mod.rs | 4 +- .../parser/statement/declaration/lexical.rs | 8 +- .../parser/statement/declaration/mod.rs | 4 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 4 +- .../statement/iteration/do_while_statement.rs | 21 ++-- .../statement/iteration/for_statement.rs | 6 +- boa/src/syntax/parser/statement/mod.rs | 31 +++--- boa/src/syntax/parser/statement/switch/mod.rs | 4 +- boa/src/syntax/parser/statement/throw/mod.rs | 4 +- .../syntax/parser/statement/try_stm/catch.rs | 2 +- .../syntax/parser/statement/try_stm/mod.rs | 14 +-- .../syntax/parser/statement/variable/mod.rs | 2 +- boa_cli/src/main.rs | 12 ++- 41 files changed, 318 insertions(+), 431 deletions(-) diff --git a/boa/benches/lexer.rs b/boa/benches/lexer.rs index 68def8fde3c..ecb6dd97910 100644 --- a/boa/benches/lexer.rs +++ b/boa/benches/lexer.rs @@ -17,10 +17,10 @@ static EXPRESSION: &str = r#" fn expression_lexer(c: &mut Criterion) { c.bench_function("Expression (Lexer)", move |b| { b.iter(|| { - let lexer = Lexer::new(black_box(EXPRESSION.as_bytes())); + let mut lexer = Lexer::new(black_box(EXPRESSION.as_bytes())); // Goes through and lexes entire given string. - lexer.collect::, _>>().expect("failed to lex"); + while lexer.next().expect("Failed to lex").is_some() {} }) }); } @@ -30,11 +30,11 @@ static HELLO_WORLD: &str = "let foo = 'hello world!'; foo;"; fn hello_world_lexer(c: &mut Criterion) { c.bench_function("Hello World (Lexer)", move |b| { b.iter(|| { - let lexer = Lexer::new(black_box(HELLO_WORLD.as_bytes())); + let mut lexer = Lexer::new(black_box(HELLO_WORLD.as_bytes())); // return the value into the blackbox so its not optimized away // https://gist.github.com/jasonwilliams/5325da61a794d8211dcab846d466c4fd // Goes through and lexes entire given string. - lexer.collect::, _>>().expect("failed to lex"); + while lexer.next().expect("Failed to lex").is_some() {} }) }); } @@ -54,10 +54,10 @@ for (let a = 10; a < 100; a++) { fn for_loop_lexer(c: &mut Criterion) { c.bench_function("For loop (Lexer)", move |b| { b.iter(|| { - let lexer = Lexer::new(black_box(FOR_LOOP.as_bytes())); + let mut lexer = Lexer::new(black_box(FOR_LOOP.as_bytes())); // Goes through and lexes entire given string. - lexer.collect::, _>>().expect("failed to lex"); + while lexer.next().expect("Failed to lex").is_some() {} }) }); } diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 4116f730920..f770b556c9c 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -21,15 +21,9 @@ impl Tokenizer for SingleLineComment { R: Read, { // Skip either to the end of the line or to the end of the input - while let Some(ch) = cursor.next() { - match ch { - Err(e) => { - return Err(Error::IO(e)); - } - Ok('\n') => { - break; - } - _ => {} + while let Some(ch) = cursor.next()? { + if ch == '\n' { + break; } } Ok(Token::new( @@ -56,17 +50,9 @@ impl Tokenizer for BlockComment { R: Read, { loop { - if let Some(ch) = cursor.next() { - match ch { - Err(e) => { - return Err(Error::IO(e)); - } - Ok('*') => { - if cursor.next_is('/')? { - break; - } - } - _ => {} + if let Some(ch) = cursor.next()? { + if ch == '*' && cursor.next_is('/')? { + break; } } else { return Err(Error::syntax("unterminated multiline comment")); diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index 38a29992b0b..c5015cebc5a 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -54,35 +54,26 @@ where /// Peeks the next character. #[inline] - pub(super) fn peek(&mut self) -> Option> { + pub(super) fn peek(&mut self) -> Result, Error> { let iter = &mut self.iter; - - match self.peeked { - None => match iter.next() { - Some(Err(e)) => Some(Err(e)), - Some(Ok(c)) => { - self.peeked = Some(Some(c)); - Some(Ok(c)) - } - None => { - self.peeked = Some(None); - None - } - }, - Some(v) => v.map(Ok), + if let Some(v) = self.peeked { + Ok(v) + } else { + let val = iter.next().transpose()?; + self.peeked = Some(val); + Ok(val) } } /// Compares the character passed in to the next character, if they match true is returned and the buffer is incremented #[inline] pub(super) fn next_is(&mut self, peek: char) -> io::Result { - Ok(match self.peek() { - None => false, - Some(Ok(next)) if next == peek => { + Ok(match self.peek()? { + Some(next) if next == peek => { let _ = self.peeked.take(); true } - _ => false, + None | _ => false, }) } @@ -95,10 +86,10 @@ where where F: Fn(char) -> bool, { - Ok(match self.peek() { - None => false, - Some(Ok(peek)) => pred(peek), - Some(Err(e)) => return Err(e), + Ok(if let Some(peek) = self.peek()? { + pred(peek) + } else { + false }) } @@ -110,19 +101,13 @@ where if self.next_is(stop)? { return Ok(()); } else { - match self.next() { - None => { - return Err(io::Error::new( - ErrorKind::UnexpectedEof, - format!("Unexpected end of file when looking for character {}", stop), - )); - } - Some(Err(e)) => { - return Err(e); - } - Some(Ok(ch)) => { - buf.push(ch); - } + if let Some(ch) = self.next()? { + buf.push(ch); + } else { + return Err(io::Error::new( + ErrorKind::UnexpectedEof, + format!("Unexpected end of file when looking for character {}", stop), + )); } } } @@ -140,17 +125,11 @@ where if !self.next_is_pred(pred)? { return Ok(()); } else { - match self.next() { - None => { - // next_is_pred will return false if the next value is None so the None case should already be handled. - unreachable!(); - } - Some(Err(e)) => { - return Err(e); - } - Some(Ok(ch)) => { - buf.push(ch); - } + if let Some(ch) = self.next()? { + buf.push(ch); + } else { + // next_is_pred will return false if the next value is None so the None case should already be handled. + unreachable!(); } } } @@ -160,23 +139,16 @@ where pub(super) fn fill_bytes(&mut self, buf: &mut [u8]) -> io::Result<()> { unimplemented!("Lexer::cursor::fill_bytes {:?}", buf) } -} - -impl Iterator for Cursor -where - R: Read, -{ - type Item = io::Result; #[inline] - fn next(&mut self) -> Option { + pub(crate) fn next(&mut self) -> Result, Error> { let chr = match self.peeked.take() { Some(v) => v, None => { if let Some(n) = self.iter.next() { match n { Err(e) => { - return Some(Err(e)); + return Err(e); } Ok(c) => Some(c), } @@ -193,7 +165,7 @@ where _ => {} } - chr.map(Ok) + Ok(chr) } } diff --git a/boa/src/syntax/lexer/identifier.rs b/boa/src/syntax/lexer/identifier.rs index 496e5f1bd96..b85f199b1a2 100644 --- a/boa/src/syntax/lexer/identifier.rs +++ b/boa/src/syntax/lexer/identifier.rs @@ -33,13 +33,12 @@ impl Tokenizer for Identifier { { let mut buf = self.init.to_string(); - while let Some(c) = cursor.peek() { - let c = c?; + while let Some(c) = cursor.peek()? { if c.is_alphabetic() || c.is_digit(10) || c == '_' { - let ch = cursor - .next() - .expect("Character in identifier has vanished")?; - buf.push(ch); + cursor + .next()? + .expect("Character in identifier has vanished"); + buf.push(c); } else { break; } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 1934259a68c..43329b511da 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -102,8 +102,8 @@ impl Lexer { where R: Read, { - if let Some(c) = self.cursor.peek() { - match c? { + if let Some(c) = self.cursor.peek()? { + match c { '/' => { self.cursor.next(); // Consume the SingleLineComment.lex(&mut self.cursor, start) @@ -139,47 +139,23 @@ impl Lexer { } } } else { - Err(Error::syntax("Expecting Token /,*,= or regex")) + Err(Error::syntax("Abrupt end: Expecting Token /,*,= or regex")) } } -} -/// ECMAScript goal symbols. -/// -/// -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum InputElement { - Div, - RegExp, - RegExpOrTemplateTail, - TemplateTail, -} - -impl Default for InputElement { - fn default() -> Self { - InputElement::RegExpOrTemplateTail - // Decided on InputElementDiv as default for now based on documentation from - // - } -} - -impl Iterator for Lexer -where - R: Read, -{ - type Item = Result; - - fn next(&mut self) -> Option { + pub fn next(&mut self) -> Result, Error> + where + R: Read, + { let (start, next_chr) = loop { let start = self.cursor.pos(); - let next_chr = match self.cursor.next()? { - Ok(c) => c, - Err(e) => return Some(Err(e.into())), - }; - - // Ignore whitespace - if !Self::is_whitespace(next_chr) { - break (start, next_chr); + if let Some(next_chr) = self.cursor.next()? { + // Ignore whitespace + if !Self::is_whitespace(next_chr) { + break (start, next_chr); + } + } else { + return Ok(None); } }; @@ -253,17 +229,32 @@ where ); Err(Error::syntax(details)) } - }; + }?; - if let Ok(t) = token { - if t.kind() == &TokenKind::Comment { - // Skip comment - self.next() - } else { - Some(Ok(t)) - } + if token.kind() == &TokenKind::Comment { + // Skip comment + self.next() } else { - Some(token) + Ok(Some(token)) } } } + +/// ECMAScript goal symbols. +/// +/// +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InputElement { + Div, + RegExp, + RegExpOrTemplateTail, + TemplateTail, +} + +impl Default for InputElement { + fn default() -> Self { + InputElement::RegExpOrTemplateTail + // Decided on InputElementDiv as default for now based on documentation from + // + } +} diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 1ab94f1dd56..335f2ceb2f9 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -68,35 +68,30 @@ where { // The next part must be SignedInteger. // This is optionally a '+' or '-' followed by 1 or more DecimalDigits. - match cursor.next() { - Some(Ok('+')) => { + match cursor.next()? { + Some('+') => { buf.push('+'); if !cursor.next_is_pred(&|c: char| c.is_digit(kind.base()))? { // A digit must follow the + or - symbol. return Err(Error::syntax("No digit found after + symbol")); } } - Some(Ok('-')) => { + Some('-') => { buf.push('-'); if !cursor.next_is_pred(&|c: char| c.is_digit(kind.base()))? { // A digit must follow the + or - symbol. return Err(Error::syntax("No digit found after - symbol")); } } - Some(Ok(c)) if c.is_digit(kind.base()) => { - buf.push(c); - } - Some(Ok(c)) => { + Some(c) if c.is_digit(kind.base()) => buf.push(c), + Some(c) => { return Err(Error::syntax(format!( "When lexing exponential value found unexpected char: '{}'", c ))); } - Some(Err(e)) => { - return Err(e.into()); - } None => { - return Err(Error::syntax("No exponential value found")); + return Err(Error::syntax("Abrupt end: No exponential value found")); } } @@ -137,8 +132,8 @@ impl Tokenizer for NumberLiteral { let c = cursor.peek(); if self.init == '0' { - if let Some(ch) = c { - match ch? { + if let Some(ch) = c? { + match ch { 'x' | 'X' => { // Remove the initial '0' from buffer. cursor.next(); @@ -184,7 +179,7 @@ impl Tokenizer for NumberLiteral { // Remove the initial '0' from buffer. buf.pop(); - buf.push(cursor.next().expect("'0' character vanished")?); + buf.push(cursor.next()?.expect("'0' character vanished")); kind = NumericKind::Integer(8); } @@ -197,7 +192,7 @@ impl Tokenizer for NumberLiteral { "Leading 0's are not allowed in strict mode.", )); } else { - buf.push(cursor.next().expect("Number digit vanished")?); + buf.push(cursor.next()?.expect("Number digit vanished")); } } // Else indicates that the symbol is a non-number. } @@ -221,8 +216,8 @@ impl Tokenizer for NumberLiteral { // 'n' To indicate a BigIntLiteralSuffix. // '.' To indicate a decimal seperator. // 'e' | 'E' To indicate an ExponentPart. - match cursor.peek() { - Some(Ok('n')) => { + match cursor.peek()? { + Some('n') => { // DecimalBigIntegerLiteral // Lexing finished. @@ -231,7 +226,7 @@ impl Tokenizer for NumberLiteral { kind = kind.to_bigint(); } - Some(Ok('.')) => { + Some('.') => { if kind.base() == 10 { // Only base 10 numbers can have a decimal seperator. // Number literal lexing finished if a . is found for a number in a different base. @@ -245,8 +240,8 @@ impl Tokenizer for NumberLiteral { // The non-digit character at this point must be an 'e' or 'E' to indicate an Exponent Part. // Another '.' or 'n' is not allowed. - match cursor.peek() { - Some(Ok('e')) | Some(Ok('E')) => { + match cursor.peek()? { + Some('e') | Some('E') => { // Consume the ExponentIndicator. cursor.next(); @@ -254,24 +249,17 @@ impl Tokenizer for NumberLiteral { take_signed_integer(&mut buf, cursor, &kind)?; } - Some(Err(e)) => { - return Err(e.into()); - } - Some(Ok(_)) | None => { + Some(_) | None => { // Finished lexing. } } } } - Some(Ok('e')) | Some(Ok('E')) => { + Some('e') | Some('E') => { cursor.next(); // Consume the ExponentIndicator. take_signed_integer(exp_str, cursor, &kind)?; } - Some(Err(e)) => { - return Err(e.into()); - } - - Some(Ok(_)) | None => { + Some(_) | None => { // Indicates lexing finished. } } diff --git a/boa/src/syntax/lexer/operator.rs b/boa/src/syntax/lexer/operator.rs index 5e12890f27d..7669a03922d 100644 --- a/boa/src/syntax/lexer/operator.rs +++ b/boa/src/syntax/lexer/operator.rs @@ -8,25 +8,21 @@ use std::io::Read; /// If the next value is not an assignment operation it will pattern match the provided values and return the corresponding token. macro_rules! vop { ($cursor:ident, $assign_op:expr, $op:expr) => ({ - match $cursor.peek() { - None | Some(Err(_)) => { - Err(Error::syntax("could not preview next value")) - } - Some(Ok('=')) => { + match $cursor.peek()? { + None => Err(Error::syntax("Abrupt end: could not preview next value as part of operator")), + Some('=') => { $cursor.next(); $cursor.next_column(); $assign_op } - Some(Ok(_)) => $op, + Some(_) => $op, } }); ($cursor:ident, $assign_op:expr, $op:expr, {$($case:pat => $block:expr), +}) => ({ // let punc = $cursor.peek().ok_or_else(|| Error::syntax("could not preview next value"))?; - match $cursor.peek() { - None | Some(Err(_)) => { - Err(Error::syntax("could not preview next value")) - } - Some(Ok('=')) => { + match $cursor.peek()? { + None => Err(Error::syntax("Abrupt end: could not preview next value as part of operator")), + Some('=') => { $cursor.next(); $cursor.next_column(); $assign_op @@ -36,7 +32,6 @@ macro_rules! vop { $cursor.next_column(); $block })+, - _ => $op, } }); @@ -61,9 +56,9 @@ macro_rules! op { )) }); ($cursor:ident, $start_pos:expr, $assign_op:expr, $op:expr, {$($case:pat => $block:expr),+}) => ({ - let punc: Result = vop!($cursor, $assign_op, $op, {$($case => $block),+}); + let punc: Punctuator = vop!($cursor, $assign_op, $op, {$($case => $block),+})?; Ok(Token::new( - punc?.into(), + punc.into(), Span::new($start_pos, $cursor.pos()), )) }); @@ -98,13 +93,13 @@ impl Tokenizer for Operator { { match self.init { '*' => op!(cursor, start_pos, Ok(Punctuator::AssignMul), Ok(Punctuator::Mul), { - Some(Ok('*')) => vop!(cursor, Ok(Punctuator::AssignPow), Ok(Punctuator::Exp)) + Some('*') => vop!(cursor, Ok(Punctuator::AssignPow), Ok(Punctuator::Exp)) }), '+' => op!(cursor, start_pos, Ok(Punctuator::AssignAdd), Ok(Punctuator::Add), { - Some(Ok('+')) => Ok(Punctuator::Inc) + Some('+') => Ok(Punctuator::Inc) }), '-' => op!(cursor, start_pos, Ok(Punctuator::AssignSub), Ok(Punctuator::Sub), { - Some(Ok('-')) => { + Some('-') => { Ok(Punctuator::Dec) } }), @@ -115,10 +110,10 @@ impl Tokenizer for Operator { Ok(Punctuator::Mod) ), '|' => op!(cursor, start_pos, Ok(Punctuator::AssignOr), Ok(Punctuator::Or), { - Some(Ok('|')) => Ok(Punctuator::BoolOr) + Some('|') => Ok(Punctuator::BoolOr) }), '&' => op!(cursor, start_pos, Ok(Punctuator::AssignAnd), Ok(Punctuator::And), { - Some(Ok('&')) => Ok(Punctuator::BoolAnd) + Some('&') => Ok(Punctuator::BoolAnd) }), '^' => op!( cursor, @@ -131,17 +126,17 @@ impl Tokenizer for Operator { } else { Ok(Punctuator::Eq) }, Ok(Punctuator::Assign), { - Some(Ok('>')) => { + Some('>') => { Ok(Punctuator::Arrow) } }), '<' => op!(cursor, start_pos, Ok(Punctuator::LessThanOrEq), Ok(Punctuator::LessThan), { - Some(Ok('<')) => vop!(cursor, Ok(Punctuator::AssignLeftSh), Ok(Punctuator::LeftSh)) + Some('<') => vop!(cursor, Ok(Punctuator::AssignLeftSh), Ok(Punctuator::LeftSh)) }), '>' => { op!(cursor, start_pos, Ok(Punctuator::GreaterThanOrEq), Ok(Punctuator::GreaterThan), { - Some(Ok('>')) => vop!(cursor, Ok(Punctuator::AssignRightSh), Ok(Punctuator::RightSh), { - Some(Ok('>')) => vop!(cursor, Ok(Punctuator::AssignURightSh), Ok(Punctuator::URightSh)) + Some('>') => vop!(cursor, Ok(Punctuator::AssignRightSh), Ok(Punctuator::RightSh), { + Some('>') => vop!(cursor, Ok(Punctuator::AssignURightSh), Ok(Punctuator::URightSh)) }) }) } diff --git a/boa/src/syntax/lexer/regex.rs b/boa/src/syntax/lexer/regex.rs index 0bf0e2f52f1..165b5e184ff 100644 --- a/boa/src/syntax/lexer/regex.rs +++ b/boa/src/syntax/lexer/regex.rs @@ -28,15 +28,12 @@ impl Tokenizer for RegexLiteral { // Lex RegularExpressionBody. loop { - match cursor.next() { + match cursor.next()? { None => { // Abrupt end. return Err(Error::syntax("Abrupt end, regex not terminated")); } - Some(Err(e)) => { - return Err(Error::from(e)); - } - Some(Ok(c)) => { + Some(c) => { match c { '/' => break, // RegularExpressionBody finished. '\n' | '\r' | '\u{2028}' | '\u{2029}' => { @@ -46,28 +43,19 @@ impl Tokenizer for RegexLiteral { '\\' => { // Escape sequence body.push('\\'); - match cursor.next() { - None => { - // Abrupt end of regex. - return Err(Error::syntax("Abrupt end, regex not terminated")); - } - Some(Err(_)) => { - return Err(Error::from(io::Error::new( - ErrorKind::Interrupted, - "Failed to peek next character", - ))) - } - Some(Ok(sc)) => { - match sc { - '\n' | '\r' | '\u{2028}' | '\u{2029}' => { - // Not allowed in Regex literal. - return Err(Error::syntax( - "Encountered new line during regex", - )); - } - ch => body.push(ch), + if let Some(sc) = cursor.next()? { + match sc { + '\n' | '\r' | '\u{2028}' | '\u{2029}' => { + // Not allowed in Regex literal. + return Err(Error::syntax( + "Encountered new line during regex", + )); } + ch => body.push(ch), } + } else { + // Abrupt end of regex. + return Err(Error::syntax("Abrupt end, regex not terminated")); } } _ => body.push(c), diff --git a/boa/src/syntax/lexer/string.rs b/boa/src/syntax/lexer/string.rs index d3b3556e71d..fa11192fc7c 100644 --- a/boa/src/syntax/lexer/string.rs +++ b/boa/src/syntax/lexer/string.rs @@ -44,12 +44,12 @@ impl Tokenizer for StringLiteral { let mut buf = String::new(); loop { let next_chr_start = cursor.pos(); - let next_chr = cursor.next().ok_or_else(|| { + let next_chr = cursor.next()?.ok_or_else(|| { Error::from(io::Error::new( ErrorKind::UnexpectedEof, "unterminated string literal", )) - })??; + })?; match next_chr { '\'' if self.terminator == StringTerminator::SingleQuote => { @@ -59,12 +59,12 @@ impl Tokenizer for StringLiteral { break; } '\\' => { - let escape = cursor.next().ok_or_else(|| { + let escape = cursor.next()?.ok_or_else(|| { Error::from(io::Error::new( ErrorKind::UnexpectedEof, "unterminated escape sequence in string literal", )) - })??; + })?; if escape != '\n' { let escaped_ch = match escape { 'n' => '\n', @@ -77,12 +77,12 @@ impl Tokenizer for StringLiteral { // TODO: optimize by getting just bytes let mut nums = String::with_capacity(2); for _ in 0_u8..2 { - let next = cursor.next().ok_or_else(|| { + let next = cursor.next()?.ok_or_else(|| { Error::from(io::Error::new( ErrorKind::UnexpectedEof, "unterminated escape sequence in string literal", )) - })??; + })?; nums.push(next); } let as_num = match u64::from_str_radix(&nums, 16) { diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs index f85d3dee659..7679d52c24e 100644 --- a/boa/src/syntax/lexer/template.rs +++ b/boa/src/syntax/lexer/template.rs @@ -23,16 +23,15 @@ impl Tokenizer for TemplateLiteral { { let mut buf = String::new(); loop { - match cursor.next() { + match cursor.next()? { None => { return Err(Error::from(io::Error::new( ErrorKind::UnexpectedEof, "Unterminated template literal", ))); } - Some(Err(e)) => return Err(Error::from(e)), - Some(Ok('`')) => break, // Template literal finished. - Some(Ok(next_ch)) => buf.push(next_ch), // TODO when there is an expression inside the literal + Some('`') => break, // Template literal finished. + Some(next_ch) => buf.push(next_ch), // TODO when there is an expression inside the literal } } diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index e9bab62a0ef..d64656dfc4e 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -18,7 +18,7 @@ where } assert!( - lexer.next().is_none(), + lexer.next().unwrap().is_none(), "Unexpected extra token lexed at end of input" ); } @@ -76,10 +76,7 @@ fn check_template_literal_unterminated() { let s = "`I'm a template"; let mut lexer = Lexer::new(s.as_bytes()); - match lexer.next() { - Some(Err(Error::IO(_))) => {} - _ => panic!("Lexer did not handle unterminated literal with error"), - } + lexer.next().expect_err("Lexer did not handle unterminated literal with error"); } #[test] @@ -306,19 +303,21 @@ fn check_line_numbers() { #[test] fn check_decrement_advances_lexer_2_places() { // Here we want an example of decrementing an integer - let lexer = Lexer::new(&b"let a = b--;"[0..]); + let mut lexer = Lexer::new(&b"let a = b--;"[0..]); - let mut iter = lexer.skip(4); + for _ in 0..4 { + lexer.next().unwrap(); + } assert_eq!( - iter.next().unwrap().unwrap().kind(), + lexer.next().unwrap().unwrap().kind(), &TokenKind::Punctuator(Punctuator::Dec) ); // Decrementing means adding 2 characters '--', the lexer should consume it as a single token // and move the curser forward by 2, meaning the next token should be a semicolon assert_eq!( - iter.next().unwrap().unwrap().kind(), + lexer.next().unwrap().unwrap().kind(), &TokenKind::Punctuator(Punctuator::Semicolon) ); } @@ -419,7 +418,7 @@ fn hexadecimal_edge_case() { #[test] fn single_number_without_semicolon() { let mut lexer = Lexer::new(&b"1"[0..]); - if let Some(Ok(x)) = lexer.next() { + if let Some(x) = lexer.next().unwrap() { assert_eq!(x.kind(), &TokenKind::numeric_literal(Numeric::Integer(1))); } else { panic!("Failed to lex 1 without semicolon"); @@ -583,20 +582,20 @@ fn illegal_following_numeric_literal() { // Decimal Digit let mut lexer = Lexer::new(&b"11.6n3"[0..]); assert!( - lexer.next().unwrap().err().is_some(), + lexer.next().err().is_some(), "DecimalDigit following NumericLiteral not rejected as expected" ); // Identifier Start let mut lexer = Lexer::new(&b"17.4$"[0..]); assert!( - lexer.next().unwrap().err().is_some(), + lexer.next().err().is_some(), "IdentifierStart '$' following NumericLiteral not rejected as expected" ); let mut lexer = Lexer::new(&b"17.4_"[0..]); assert!( - lexer.next().unwrap().err().is_some(), + lexer.next().err().is_some(), "IdentifierStart '_' following NumericLiteral not rejected as expected" ); } @@ -608,7 +607,7 @@ fn illegal_code_point_following_numeric_literal() { // be immediately followed by an IdentifierStart where the IdentifierStart let mut lexer = Lexer::new(r#"17.4\u{{2764}}"#.as_bytes()); assert!( - lexer.next().unwrap().err().is_some(), + lexer.next().err().is_some(), "IdentifierStart \\u{{2764}} following NumericLiteral not rejected as expected" ); } diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index ae4b2185bd2..3db2f625be3 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -42,51 +42,34 @@ where } /// Moves the cursor to the next token and returns the token. - pub(super) fn next(&mut self) -> Option> { + pub(super) fn next(&mut self) -> Result, ParseError> { if let Some(t) = self.peeked.pop_front() { - return t.map(Ok); + return Ok(t); } // No value has been peeked ahead already so need to go get the next value. - if let Some(t) = self.lexer.next() { - Some(t.map_err(|e| e.into())) - } else { - None - } + Ok(self.lexer.next()?) } /// Peeks the next token without moving the cursor. - pub(super) fn peek(&mut self) -> Option> { + pub(super) fn peek(&mut self) -> Result, ParseError> { if let Some(v) = self.peeked.front() { - if let Some(t) = v { - return Some(Ok(t.clone())); - } else { - return None; - } + return Ok(v.clone()); } // No value has been peeked ahead already so need to go get the next value. - match self.next() { - Some(Ok(token)) => { - self.peeked.push_back(Some(token.clone())); - Some(Ok(token)) - } - Some(Err(e)) => Some(Err(e)), - None => { - self.peeked.push_back(None); - None - } - } + let val = self.next()?; + self.peeked.push_back(val.clone()); + Ok(val) } /// Peeks the token after the next token. /// i.e. if there are tokens A, B, C and peek() returns A then peek_skip(1) will return B. - pub(super) fn peek_skip(&mut self) -> Option> { + pub(super) fn peek_skip(&mut self) -> Result, ParseError> { // Add elements to the peeked buffer upto the amount required to skip the given amount ahead. while self.peeked.len() < 2 { - match self.lexer.next() { - Some(Ok(token)) => self.peeked.push_back(Some(token.clone())), - Some(Err(e)) => return Some(Err(ParseError::lex(e))), + match self.lexer.next()? { + Some(token) => self.peeked.push_back(Some(token.clone())), None => self.peeked.push_back(None), } } @@ -103,7 +86,7 @@ where self.peeked.push_front(ret.clone()); self.peeked.push_front(temp); - ret.map(Ok) + Ok(ret) } /// Takes the given token and pushes it back onto the parser token queue (at the front so the token will be returned on next .peek()). @@ -118,7 +101,7 @@ where where K: Into, { - let next_token = self.peek().ok_or(ParseError::AbruptEnd)??; + let next_token = self.peek()?.ok_or(ParseError::AbruptEnd)?; let kind = kind.into(); if next_token.kind() == &kind { @@ -135,15 +118,14 @@ where /// /// [spec]: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion pub(super) fn peek_semicolon(&mut self) -> Result<(bool, Option), ParseError> { - match self.peek() { - Some(Ok(tk)) => match tk.kind() { + match self.peek()? { + Some(tk) => match tk.kind() { TokenKind::Punctuator(Punctuator::Semicolon) => Ok((true, Some(tk))), TokenKind::LineTerminator | TokenKind::Punctuator(Punctuator::CloseBlock) => { Ok((true, Some(tk))) } _ => Ok((false, Some(tk))), }, - Some(Err(e)) => Err(e), None => Ok((true, None)), } } @@ -181,43 +163,43 @@ where /// /// If skip is true then the token after the peek() token is checked instead. pub(super) fn peek_expect_no_lineterminator(&mut self, skip: bool) -> Result<(), ParseError> { - let token = if skip { self.peek_skip() } else { self.peek() }; - - match token { - Some(Ok(t)) => { - if t.kind() == &TokenKind::LineTerminator { - Err(ParseError::unexpected(t, None)) - } else { - Ok(()) - } + let token = if skip { self.peek_skip()? } else { self.peek()? }; + + if let Some(t) = token { + if t.kind() == &TokenKind::LineTerminator { + Err(ParseError::unexpected(t, None)) + } else { + Ok(()) } - Some(Err(e)) => Err(e), - None => Err(ParseError::AbruptEnd), + } else { + Err(ParseError::AbruptEnd) } } /// Advance the cursor to the next token and retrieve it, only if it's of `kind` type. /// /// When the next token is a `kind` token, get the token, otherwise return `None`. - pub(super) fn next_if(&mut self, kind: K) -> Option> + /// + /// No next token also returns None. + pub(super) fn next_if(&mut self, kind: K) -> Result, ParseError> where K: Into, { - match self.peek() { - Some(Ok(token)) => { - if token.kind() == &kind.into() { - self.next() - } else { - None - } + Ok(if let Some(token) = self.peek()? { + if token.kind() == &kind.into() { + self.next()? + } else { + None } - Some(Err(e)) => Some(Err(e)), - None => None, - } + } else { + None + }) } /// Advance the cursor to skip 0, 1 or more line terminators. - pub(super) fn skip_line_terminators(&mut self) { - while self.next_if(TokenKind::LineTerminator).is_some() {} + #[inline] + pub(super) fn skip_line_terminators(&mut self) -> Result<(),ParseError>{ + while self.next_if(TokenKind::LineTerminator)?.is_some() {} + Ok(()) } } diff --git a/boa/src/syntax/parser/expression/assignment/arrow_function.rs b/boa/src/syntax/parser/expression/assignment/arrow_function.rs index 183ebbd8854..7f8a4c27acb 100644 --- a/boa/src/syntax/parser/expression/assignment/arrow_function.rs +++ b/boa/src/syntax/parser/expression/assignment/arrow_function.rs @@ -70,7 +70,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("ArrowFunction", "Parsing"); - let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; let params = if let TokenKind::Punctuator(Punctuator::OpenParen) = &next_token.kind() { // CoverParenthesizedExpressionAndArrowParameterList @@ -131,7 +131,7 @@ where type Output = StatementList; fn parse(self, cursor: &mut Cursor) -> Result { - match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { + match cursor.peek()?.ok_or(ParseError::AbruptEnd)?.kind() { TokenKind::Punctuator(Punctuator::OpenBlock) => { let _ = cursor.next(); let body = FunctionBody::new(false, false).parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index ee2e0d2b23d..82d587b4c4e 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -68,8 +68,8 @@ where let lhs = LogicalORExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - if let Some(tok) = cursor.peek() { - if tok?.kind() == &TokenKind::Punctuator(Punctuator::Question) { + if let Some(tok) = cursor.peek()? { + if tok.kind() == &TokenKind::Punctuator(Punctuator::Question) { cursor.next(); // Consume the token. let then_clause = AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index f910d1e4dd5..4145c64ce28 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -59,8 +59,8 @@ fn is_unary_expression(cursor: &mut Cursor) -> Result where R: Read, { - Ok(if let Some(tok) = cursor.peek() { - match tok?.kind() { + Ok(if let Some(tok) = cursor.peek()? { + match tok.kind() { TokenKind::Keyword(Keyword::Delete) | TokenKind::Keyword(Keyword::Void) | TokenKind::Keyword(Keyword::TypeOf) @@ -88,8 +88,8 @@ where } let lhs = UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.peek() { - if let TokenKind::Punctuator(Punctuator::Exp) = tok?.kind() { + if let Some(tok) = cursor.peek()? { + if let TokenKind::Punctuator(Punctuator::Exp) = tok.kind() { cursor.next(); // Consume the token. return Ok(BinOp::new(NumOp::Exp, lhs, self.parse(cursor)?).into()); } diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index 3c5204c62dd..a1933b0431d 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -83,14 +83,14 @@ where cursor.set_goal(InputElement::Div); // Arrow function - match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { + match cursor.peek()?.ok_or(ParseError::AbruptEnd)?.kind() { // a=>{} TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield) | TokenKind::Keyword(Keyword::Await) => { if cursor.peek_expect_no_lineterminator(true).is_ok() { - if let Some(tok) = cursor.peek_skip() { - if tok?.kind() == &TokenKind::Punctuator(Punctuator::Arrow) { + if let Some(tok) = cursor.peek_skip()? { + if tok.kind() == &TokenKind::Punctuator(Punctuator::Arrow) { return ArrowFunction::new( self.allow_in, self.allow_yield, @@ -122,7 +122,7 @@ where let mut lhs = ConditionalExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; - if let Some(tok) = cursor.peek().transpose()? { + if let Some(tok) = cursor.peek()? { match tok.kind() { TokenKind::Punctuator(Punctuator::Assign) => { cursor.next(); // Consume the token. diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index 4c581e33d6f..88c11f66d72 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -63,7 +63,7 @@ where let mut args = Vec::new(); loop { cursor.skip_line_terminators(); - let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; match next_token.kind() { TokenKind::Punctuator(Punctuator::CloseParen) => { @@ -77,7 +77,7 @@ where return Err(ParseError::unexpected(next_token.clone(), None)); } - if cursor.next_if(Punctuator::CloseParen).is_some() { + if cursor.next_if(Punctuator::CloseParen)?.is_some() { break; } } @@ -96,7 +96,7 @@ where } } - if cursor.next_if(Punctuator::Spread).is_some() { + if cursor.next_if(Punctuator::Spread)?.is_some() { args.push( Spread::new( AssignmentExpression::new(true, self.allow_yield, self.allow_await) diff --git a/boa/src/syntax/parser/expression/left_hand_side/call.rs b/boa/src/syntax/parser/expression/left_hand_side/call.rs index c107886cf58..23adde85155 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/call.rs @@ -65,35 +65,35 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("CallExpression", "Parsing"); - let tk = cursor.peek(); + let tk = cursor.peek()?; let mut lhs = match tk { - Some(_) if tk.unwrap()?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { + Some(_) if tk.unwrap().kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; Node::from(Call::new(self.first_member_expr, args)) } _ => { - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)?; + let next_token = cursor.next()?.ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::OpenParen)], - next_token?, + next_token, "call expression", )); } }; - while let Some(tok) = cursor.peek() { - let token = tok?.clone(); + while let Some(tok) = cursor.peek()? { + let token = tok.clone(); match token.kind() { TokenKind::Punctuator(Punctuator::OpenParen) => { let args = Arguments::new(self.allow_yield, self.allow_await).parse(cursor)?; lhs = Node::from(Call::new(lhs, args)); } TokenKind::Punctuator(Punctuator::Dot) => { - cursor.next().ok_or(ParseError::AbruptEnd)??; // We move the parser forward. + cursor.next()?.ok_or(ParseError::AbruptEnd)?; // We move the parser forward. - match &cursor.next().ok_or(ParseError::AbruptEnd)??.kind() { + match &cursor.next()?.ok_or(ParseError::AbruptEnd)?.kind() { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into(); } @@ -110,7 +110,7 @@ where } } TokenKind::Punctuator(Punctuator::OpenBracket) => { - let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser. + let _ = cursor.next()?.ok_or(ParseError::AbruptEnd)?; // We move the parser. let idx = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; cursor.expect(Punctuator::CloseBracket, "call expression")?; diff --git a/boa/src/syntax/parser/expression/left_hand_side/member.rs b/boa/src/syntax/parser/expression/left_hand_side/member.rs index 26069781382..f7adf298b4a 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/member.rs @@ -60,7 +60,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("MemberExpression", "Parsing"); - let mut lhs = if cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() + let mut lhs = if cursor.peek()?.ok_or(ParseError::AbruptEnd)?.kind() == &TokenKind::Keyword(Keyword::New) { let _ = cursor.next().expect("new keyword disappeared"); @@ -72,13 +72,13 @@ where } else { PrimaryExpression::new(self.allow_yield, self.allow_await).parse(cursor)? }; - while let Some(tok) = cursor.peek() { - let token = tok?.clone(); + while let Some(tok) = cursor.peek()? { + let token = tok.clone(); match token.kind() { TokenKind::Punctuator(Punctuator::Dot) => { - cursor.next().ok_or(ParseError::AbruptEnd)??; // We move the parser forward. + cursor.next()?.ok_or(ParseError::AbruptEnd)?; // We move the parser forward. - match cursor.next().ok_or(ParseError::AbruptEnd)??.kind() { + match cursor.next()?.ok_or(ParseError::AbruptEnd)?.kind() { TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, name.clone()).into() } @@ -95,7 +95,7 @@ where } } TokenKind::Punctuator(Punctuator::OpenBracket) => { - let _ = cursor.next().ok_or(ParseError::AbruptEnd)?; // We move the parser forward. + let _ = cursor.next()?.ok_or(ParseError::AbruptEnd)?; // We move the parser forward. let idx = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; cursor.expect(Punctuator::CloseBracket, "member expression")?; diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index ea96487066d..9a0d96a1c7e 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -65,15 +65,11 @@ where // TODO: Implement NewExpression: new MemberExpression let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - match cursor.peek() { - Some(tok) => { - if tok?.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) { - CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor) - } else { - Ok(lhs) - } + if let Some(tok) = cursor.peek()?{ + if tok.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) { + return CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor) } - _ => Ok(lhs), // TODO: is this correct? } + Ok(lhs) } } diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index c64901d1700..deca6218183 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -69,8 +69,8 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo } let mut lhs = $lower::new($( self.$low_param ),*).parse(cursor)?; - while let Some(tok) = cursor.peek() { - match *tok?.kind() { + while let Some(tok) = cursor.peek()? { + match *tok.kind() { TokenKind::Punctuator(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index dd510d7ed13..d702a9abf39 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -67,17 +67,17 @@ where loop { cursor.skip_line_terminators(); // TODO: Support all features. - while cursor.next_if(Punctuator::Comma).is_some() { + while cursor.next_if(Punctuator::Comma)?.is_some() { elements.push(Node::Const(Const::Undefined)); } - if cursor.next_if(Punctuator::CloseBracket).is_some() { + if cursor.next_if(Punctuator::CloseBracket)?.is_some() { break; } - let _ = cursor.peek().ok_or(ParseError::AbruptEnd)?; // Check that there are more tokens to read. + let _ = cursor.peek()?.ok_or(ParseError::AbruptEnd); // Check that there are more tokens to read. - if cursor.next_if(Punctuator::Spread).is_some() { + if cursor.next_if(Punctuator::Spread)?.is_some() { let node = AssignmentExpression::new(true, self.allow_yield, self.allow_await) .parse(cursor)?; elements.push(Spread::new(node).into()); diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 7c86c6b0887..75e7f9835bc 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -65,7 +65,7 @@ where type Output = Node; fn parse(self, cursor: &mut Cursor) -> ParseResult { - let tok = cursor.next().ok_or(ParseError::AbruptEnd)??; + let tok = cursor.next()?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { TokenKind::Keyword(Keyword::This) => Ok(Node::This), diff --git a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs index 2e448111b1d..b499e09c96a 100644 --- a/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -65,11 +65,11 @@ where let mut elements = Vec::new(); loop { - if cursor.next_if(Punctuator::CloseBlock).is_some() { + if cursor.next_if(Punctuator::CloseBlock)?.is_some() { break; } - if cursor.next_if(TokenKind::LineTerminator).is_some() { + if cursor.next_if(TokenKind::LineTerminator)?.is_some() { // Skip line terminators. continue; } @@ -77,17 +77,17 @@ where elements .push(PropertyDefinition::new(self.allow_yield, self.allow_await).parse(cursor)?); - if cursor.next_if(Punctuator::CloseBlock).is_some() { + if cursor.next_if(Punctuator::CloseBlock)?.is_some() { break; } - if cursor.next_if(TokenKind::LineTerminator).is_some() { + if cursor.next_if(TokenKind::LineTerminator)?.is_some() { // Skip line terminators. continue; } - if cursor.next_if(Punctuator::Comma).is_none() { - let next_token = cursor.next().ok_or(ParseError::AbruptEnd)??; + if cursor.next_if(Punctuator::Comma)?.is_none() { + let next_token = cursor.next()?.ok_or(ParseError::AbruptEnd)?; return Err(ParseError::expected( vec![ TokenKind::Punctuator(Punctuator::Comma), @@ -136,21 +136,21 @@ where type Output = node::PropertyDefinition; fn parse(self, cursor: &mut Cursor) -> Result { - if cursor.next_if(Punctuator::Spread).is_some() { + if cursor.next_if(Punctuator::Spread)?.is_some() { let node = AssignmentExpression::new(true, self.allow_yield, self.allow_await) .parse(cursor)?; return Ok(node::PropertyDefinition::SpreadObject(node)); } - let prop_name = cursor.next().ok_or(ParseError::AbruptEnd)??.to_string(); - if cursor.next_if(Punctuator::Colon).is_some() { + let prop_name = cursor.next()?.ok_or(ParseError::AbruptEnd)?.to_string(); + if cursor.next_if(Punctuator::Colon)?.is_some() { let val = AssignmentExpression::new(true, self.allow_yield, self.allow_await) .parse(cursor)?; return Ok(node::PropertyDefinition::property(prop_name, val)); } if cursor - .next_if(TokenKind::Punctuator(Punctuator::OpenParen)) + .next_if(TokenKind::Punctuator(Punctuator::OpenParen))? .is_some() || ["get", "set"].contains(&prop_name.as_str()) { @@ -158,7 +158,7 @@ where .parse(cursor); } - let pos = cursor.peek().ok_or(ParseError::AbruptEnd)??.span().start(); + let pos = cursor.peek()?.ok_or(ParseError::AbruptEnd)?.span().start(); Err(ParseError::general("expected property definition", pos)) } } @@ -201,12 +201,12 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let (methodkind, prop_name, params) = match self.identifier.as_str() { idn @ "get" | idn @ "set" => { - let prop_name = cursor.next().ok_or(ParseError::AbruptEnd)??.to_string(); + let prop_name = cursor.next()?.ok_or(ParseError::AbruptEnd)?.to_string(); cursor.expect( TokenKind::Punctuator(Punctuator::OpenParen), "property method definition", )?; - let first_param = cursor.peek().expect("current token disappeared")?; + let first_param = cursor.peek()?.expect("current token disappeared"); let params = FormalParameters::new(false, false).parse(cursor)?; cursor.expect(Punctuator::CloseParen, "method definition")?; if idn == "get" { diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index e65f27c6717..7ff8ea95ecc 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -57,8 +57,8 @@ where type Output = Node; fn parse(self, cursor: &mut Cursor) -> ParseResult { - let tok = cursor.peek().ok_or(ParseError::AbruptEnd)?; - match tok?.kind() { + let tok = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; + match tok.kind() { TokenKind::Keyword(Keyword::Delete) => { cursor.next(); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Delete, self.parse(cursor)?).into()) diff --git a/boa/src/syntax/parser/expression/update.rs b/boa/src/syntax/parser/expression/update.rs index 90bfd07629d..e8793942a46 100644 --- a/boa/src/syntax/parser/expression/update.rs +++ b/boa/src/syntax/parser/expression/update.rs @@ -47,10 +47,10 @@ where type Output = Node; fn parse(self, cursor: &mut Cursor) -> ParseResult { - let tok = cursor.peek().ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek()?.ok_or(ParseError::AbruptEnd); match tok?.kind() { TokenKind::Punctuator(Punctuator::Inc) => { - cursor.next().expect("token disappeared")?; + cursor.next()?.expect("Punctuator::Inc token disappeared"); return Ok(node::UnaryOp::new( UnaryOp::IncrementPre, LeftHandSideExpression::new(self.allow_yield, self.allow_await) @@ -59,7 +59,7 @@ where .into()); } TokenKind::Punctuator(Punctuator::Dec) => { - cursor.next().expect("token disappeared")?; + cursor.next()?.expect("Punctuator::Dec token disappeared"); return Ok(node::UnaryOp::new( UnaryOp::DecrementPre, LeftHandSideExpression::new(self.allow_yield, self.allow_await) @@ -71,14 +71,14 @@ where } let lhs = LeftHandSideExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.peek() { - match tok?.kind() { + if let Some(tok) = cursor.peek()? { + match tok.kind() { TokenKind::Punctuator(Punctuator::Inc) => { - cursor.next().expect("token disappeared")?; + cursor.next()?.expect("Punctuator::Inc token disappeared"); return Ok(node::UnaryOp::new(UnaryOp::IncrementPost, lhs).into()); } TokenKind::Punctuator(Punctuator::Dec) => { - cursor.next().expect("token disappeared")?; + cursor.next()?.expect("Punctuator::Dec token disappeared"); return Ok(node::UnaryOp::new(UnaryOp::DecrementPost, lhs).into()); } _ => {} diff --git a/boa/src/syntax/parser/function/mod.rs b/boa/src/syntax/parser/function/mod.rs index edda574a734..ecf83d1f53b 100644 --- a/boa/src/syntax/parser/function/mod.rs +++ b/boa/src/syntax/parser/function/mod.rs @@ -64,7 +64,7 @@ where let mut params = Vec::new(); - if cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() + if cursor.peek()?.ok_or(ParseError::AbruptEnd)?.kind() == &TokenKind::Punctuator(Punctuator::CloseParen) { return Ok(params.into_boxed_slice()); @@ -73,14 +73,14 @@ where loop { let mut rest_param = false; - params.push(if cursor.next_if(Punctuator::Spread).is_some() { + params.push(if cursor.next_if(Punctuator::Spread)?.is_some() { rest_param = true; FunctionRestParameter::new(self.allow_yield, self.allow_await).parse(cursor)? } else { FormalParameter::new(self.allow_yield, self.allow_await).parse(cursor)? }); - if cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() + if cursor.peek()?.ok_or(ParseError::AbruptEnd)?.kind() == &TokenKind::Punctuator(Punctuator::CloseParen) { break; @@ -88,7 +88,7 @@ where if rest_param { return Err(ParseError::unexpected( - cursor.peek().expect("Peek token disappeared")?, + cursor.peek()?.expect("Peek token disappeared"), "rest parameter must be the last formal parameter", )); } @@ -241,8 +241,8 @@ where type Output = node::StatementList; fn parse(self, cursor: &mut Cursor) -> Result { - if let Some(tk) = cursor.peek() { - if tk?.kind() == &Punctuator::CloseBlock.into() { + if let Some(tk) = cursor.peek()? { + if tk.kind() == &Punctuator::CloseBlock.into() { return Ok(Vec::new().into()); } } diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index 1c5ec933b71..fbf48a27ca0 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -130,7 +130,7 @@ where type Output = StatementList; fn parse(self, cursor: &mut Cursor) -> Result { - if cursor.peek().is_some() { + if cursor.peek()?.is_some() { ScriptBody.parse(cursor) } else { Ok(StatementList::from(Vec::new())) diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index b33e68d98c6..96eb7cfb52d 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -71,8 +71,8 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Block", "Parsing"); cursor.expect(Punctuator::OpenBlock, "block")?; - if let Some(tk) = cursor.peek() { - if tk?.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) { + if let Some(tk) = cursor.peek()? { + if tk.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) { cursor.next(); return Ok(node::Block::from(vec![])); } diff --git a/boa/src/syntax/parser/statement/declaration/lexical.rs b/boa/src/syntax/parser/statement/declaration/lexical.rs index e8a4737868e..b271326ed86 100644 --- a/boa/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa/src/syntax/parser/statement/declaration/lexical.rs @@ -61,9 +61,9 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("LexicalDeclaration", "Parsing"); - let tok = cursor.next().ok_or(ParseError::AbruptEnd)?; + let tok = cursor.next()?.ok_or(ParseError::AbruptEnd)?; - match tok?.kind() { + match tok.kind() { TokenKind::Keyword(Keyword::Const) => { BindingList::new(self.allow_in, self.allow_yield, self.allow_await, true) .parse(cursor) @@ -134,7 +134,7 @@ where } else { return Err(ParseError::expected( vec![TokenKind::Punctuator(Punctuator::Assign)], - cursor.next().ok_or(ParseError::AbruptEnd)??, + cursor.next()?.ok_or(ParseError::AbruptEnd)?, "const declaration", )); } @@ -153,7 +153,7 @@ where TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - cursor.next().ok_or(ParseError::AbruptEnd)??, + cursor.next()?.ok_or(ParseError::AbruptEnd)?, "lexical declaration", )) } diff --git a/boa/src/syntax/parser/statement/declaration/mod.rs b/boa/src/syntax/parser/statement/declaration/mod.rs index 5e5a5465e5b..0f3c8765eca 100644 --- a/boa/src/syntax/parser/statement/declaration/mod.rs +++ b/boa/src/syntax/parser/statement/declaration/mod.rs @@ -58,9 +58,9 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Declaration", "Parsing"); - let tok = cursor.peek().ok_or(ParseError::AbruptEnd)?; + let tok = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; - match tok?.kind() { + match tok.kind() { TokenKind::Keyword(Keyword::Function) => { HoistableDeclaration::new(self.allow_yield, self.allow_await, false).parse(cursor) } diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index e00851fc918..957a18f786d 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -68,10 +68,10 @@ where let then_stm = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let else_tok = cursor.peek(); + let else_tok = cursor.peek()?; let else_stm = match else_tok { - Some(_) if else_tok.unwrap()?.kind() == &TokenKind::Keyword(Keyword::Else) => { + Some(_) if else_tok.unwrap().kind() == &TokenKind::Keyword(Keyword::Else) => { cursor.next(); Some( Statement::new(self.allow_yield, self.allow_await, self.allow_return) diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index fdc58febb32..f82c80bc382 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -73,7 +73,7 @@ where cursor.skip_line_terminators(); - let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; if next_token.kind() != &TokenKind::Keyword(Keyword::While) { return Err(ParseError::expected( @@ -116,20 +116,13 @@ where R: Read, { // The previous token is already known to be a CloseParan as this is checked as part of the dowhile parsing. - // This means that + // This means that a semicolon is always automatically inserted if one isn't present. - match cursor.peek() { - None => { - // If a do while statement ends a stream then a semicolon is automatically inserted. - cursor.next(); // Consume value. - Ok(()) + if let Some(tk) = cursor.peek()? { + if tk.kind() == &TokenKind::Punctuator(Punctuator::Semicolon) { + cursor.next(); // Consume semicolon. } - Some(Ok(tk)) => { - if tk.kind() == &TokenKind::Punctuator(Punctuator::Semicolon) { - cursor.next(); // Consume semicolon. - } - Ok(()) - } - Some(Err(e)) => Err(e), } + + Ok(()) } diff --git a/boa/src/syntax/parser/statement/iteration/for_statement.rs b/boa/src/syntax/parser/statement/iteration/for_statement.rs index 75aee805256..41141870f14 100644 --- a/boa/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/for_statement.rs @@ -72,7 +72,7 @@ where cursor.expect(Keyword::For, "for statement")?; cursor.expect(Punctuator::OpenParen, "for statement")?; - let init = match cursor.peek().ok_or(ParseError::AbruptEnd)??.kind() { + let init = match cursor.peek()?.ok_or(ParseError::AbruptEnd)?.kind() { TokenKind::Keyword(Keyword::Var) => Some( VariableDeclarationList::new(false, self.allow_yield, self.allow_await) .parse(cursor) @@ -87,7 +87,7 @@ where cursor.expect(Punctuator::Semicolon, "for statement")?; - let cond = if cursor.next_if(Punctuator::Semicolon).is_some() { + let cond = if cursor.next_if(Punctuator::Semicolon)?.is_some() { Const::from(true).into() } else { let step = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; @@ -95,7 +95,7 @@ where step }; - let step = if cursor.next_if(Punctuator::CloseParen).is_some() { + let step = if cursor.next_if(Punctuator::CloseParen)?.is_some() { None } else { let step = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index eb33ca0ffa3..dd3f804a76e 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -103,7 +103,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Statement", "Parsing"); // TODO: add BreakableStatement and divide Whiles, fors and so on to another place. - let tok = cursor.peek().ok_or(ParseError::AbruptEnd)??; + let tok = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { TokenKind::Keyword(Keyword::If) => { @@ -239,24 +239,22 @@ impl StatementList { loop { cursor.skip_line_terminators(); - match cursor.peek() { - Some(token) => { - if break_nodes.contains(&token?.kind()) { - break; - } + if let Some(token) = cursor.peek()? { + if break_nodes.contains(token.kind()) { + break; } - None => return Err(ParseError::AbruptEnd), + } else { + return Err(ParseError::AbruptEnd) } let item = StatementListItem::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor)?; - println!("Item: {:?}", item); items.push(item); // move the cursor forward for any consecutive semicolon. - while cursor.next_if(Punctuator::Semicolon).is_some() {} + while cursor.next_if(Punctuator::Semicolon)?.is_some() {} } items.sort_by(Node::hoistable_order); @@ -276,13 +274,13 @@ where let mut items = Vec::new(); loop { - match cursor.peek() { - Some(Ok(token)) if token.kind() == &TokenKind::LineTerminator => { + match cursor.peek()? { + Some(token) if token.kind() == &TokenKind::LineTerminator => { // Skip line terminators. cursor.next(); continue; } - Some(Ok(token)) + Some(token) if token.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => { if self.break_when_closingbraces { @@ -291,9 +289,6 @@ where return Err(ParseError::unexpected(token, None)); } } - Some(Err(e)) => { - return Err(e); - } None => { if self.break_when_closingbraces { return Err(ParseError::AbruptEnd); @@ -310,7 +305,7 @@ where items.push(item); // move the cursor forward for any consecutive semicolon. - while cursor.next_if(Punctuator::Semicolon).is_some() {} + while cursor.next_if(Punctuator::Semicolon)?.is_some() {} } items.sort_by(Node::hoistable_order); @@ -360,7 +355,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("StatementListItem", "Parsing"); - let tok = cursor.peek().ok_or(ParseError::AbruptEnd)??; + let tok = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; match tok.kind { TokenKind::Keyword(Keyword::Function) @@ -421,7 +416,7 @@ where let _timer = BoaProfiler::global().start_event("BindingIdentifier", "Parsing"); // TODO: strict mode. - let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; match next_token.kind() { TokenKind::Identifier(ref s) => { diff --git a/boa/src/syntax/parser/statement/switch/mod.rs b/boa/src/syntax/parser/statement/switch/mod.rs index 9d9fefb3fb7..d9547206410 100644 --- a/boa/src/syntax/parser/statement/switch/mod.rs +++ b/boa/src/syntax/parser/statement/switch/mod.rs @@ -160,7 +160,7 @@ where }) => { // Default statement. // Consume the default token. - cursor.next().expect("Default token vanished")?; + cursor.next()?.expect("Default token vanished"); if default.is_some() { // If default has already been defined then it cannot be defined again and to do so is an error. @@ -194,7 +194,7 @@ where context: _, }) => { // End of switch block. - cursor.next().expect("Switch close block symbol vanished")?; // Consume the switch close block. + cursor.next()?.expect("Switch close block symbol vanished"); // Consume the switch close block. break; } Err(e) => { diff --git a/boa/src/syntax/parser/statement/throw/mod.rs b/boa/src/syntax/parser/statement/throw/mod.rs index e4f685d0bbc..fb7b9855c82 100644 --- a/boa/src/syntax/parser/statement/throw/mod.rs +++ b/boa/src/syntax/parser/statement/throw/mod.rs @@ -53,8 +53,8 @@ where cursor.peek_expect_no_lineterminator(false)?; let expr = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.peek() { - if tok?.kind() == &TokenKind::Punctuator(Punctuator::Semicolon) { + if let Some(tok) = cursor.peek()? { + if tok.kind() == &TokenKind::Punctuator(Punctuator::Semicolon) { let _ = cursor.next(); } } diff --git a/boa/src/syntax/parser/statement/try_stm/catch.rs b/boa/src/syntax/parser/statement/try_stm/catch.rs index 701a1b52a98..23bdcaecb0c 100644 --- a/boa/src/syntax/parser/statement/try_stm/catch.rs +++ b/boa/src/syntax/parser/statement/try_stm/catch.rs @@ -54,7 +54,7 @@ where fn parse(self, cursor: &mut Cursor) -> Result { let _timer = BoaProfiler::global().start_event("Catch", "Parsing"); cursor.expect(Keyword::Catch, "try statement")?; - let catch_param = if cursor.next_if(Punctuator::OpenParen).is_some() { + let catch_param = if cursor.next_if(Punctuator::OpenParen)?.is_some() { let catch_param = CatchParameter::new(self.allow_yield, self.allow_await).parse(cursor)?; cursor.expect(Punctuator::CloseParen, "catch in try statement")?; diff --git a/boa/src/syntax/parser/statement/try_stm/mod.rs b/boa/src/syntax/parser/statement/try_stm/mod.rs index d79a31f3c24..af4ea15417f 100644 --- a/boa/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa/src/syntax/parser/statement/try_stm/mod.rs @@ -63,7 +63,7 @@ where let try_clause = Block::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - let next_token = cursor.peek().ok_or(ParseError::AbruptEnd)??; + let next_token = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; if next_token.kind() != &TokenKind::Keyword(Keyword::Catch) && next_token.kind() != &TokenKind::Keyword(Keyword::Finally) @@ -84,17 +84,17 @@ where None }; - let next_token = cursor.peek(); - let finally_block = match next_token { - Some(token) => match token?.kind() { + let next_token = cursor.peek()?; + let finally_block = if let Some(token) = next_token { + match token.kind() { TokenKind::Keyword(Keyword::Finally) => Some( Finally::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor)?, ), _ => None, - }, - - None => None, + } + } else { + None }; Ok(Try::new(try_clause, catch, finally_block)) diff --git a/boa/src/syntax/parser/statement/variable/mod.rs b/boa/src/syntax/parser/statement/variable/mod.rs index 0d5c2324f28..ac277969647 100644 --- a/boa/src/syntax/parser/statement/variable/mod.rs +++ b/boa/src/syntax/parser/statement/variable/mod.rs @@ -126,7 +126,7 @@ where TokenKind::Punctuator(Punctuator::Semicolon), TokenKind::LineTerminator, ], - cursor.next().ok_or(ParseError::AbruptEnd)??, + cursor.next()?.ok_or(ParseError::AbruptEnd)?, "lexical declaration", )) } diff --git a/boa_cli/src/main.rs b/boa_cli/src/main.rs index f92775e2d02..6febf3b752b 100644 --- a/boa_cli/src/main.rs +++ b/boa_cli/src/main.rs @@ -112,12 +112,16 @@ arg_enum! { /// Returns a error of type String with a message, /// if the source has a syntax error. fn lex_source(src: &str) -> Result, String> { - let lexer = Lexer::new(src.as_bytes()); + let mut lexer = Lexer::new(src.as_bytes()); // Goes through and lexes entire given string. - lexer - .collect::, _>>() - .map_err(|e| format!("Lexing Error: {}", e)) + let mut res = Vec::new(); + + while let Some(tk) = lexer.next().expect("Failed to lex") { + res.push(tk); + } + + Ok(res) } /// Parses the the token stream into a ast and returns it. From f5f19762ce5038932792dbbf37ad7e0c4155c7e7 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 6 Jul 2020 00:01:11 +0100 Subject: [PATCH 137/289] Number lexing change --- boa/src/syntax/lexer/number.rs | 77 +++++++++++-------- boa/src/syntax/lexer/template.rs | 2 +- boa/src/syntax/lexer/tests.rs | 14 ++-- boa/src/syntax/parser/cursor.rs | 8 +- .../parser/expression/left_hand_side/mod.rs | 4 +- boa/src/syntax/parser/statement/mod.rs | 6 +- 6 files changed, 62 insertions(+), 49 deletions(-) diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index 335f2ceb2f9..e6f25d73f2b 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -210,8 +210,6 @@ impl Tokenizer for NumberLiteral { // Consume digits until a non-digit character is encountered or all the characters are consumed. cursor.take_until_pred(&mut buf, &|c: char| c.is_digit(kind.base()))?; - let exp_str = &mut String::new(); - // The non-digit character could be: // 'n' To indicate a BigIntLiteralSuffix. // '.' To indicate a decimal seperator. @@ -256,8 +254,10 @@ impl Tokenizer for NumberLiteral { } } Some('e') | Some('E') => { + kind = NumericKind::Rational; cursor.next(); // Consume the ExponentIndicator. - take_signed_integer(exp_str, cursor, &kind)?; + buf.push('E'); + take_signed_integer(&mut buf, cursor, &kind)?; } Some(_) | None => { // Indicates lexing finished. @@ -272,32 +272,42 @@ impl Tokenizer for NumberLiteral { BigInt::from_string_radix(&buf, base as u32).expect("Could not convert to BigInt") ) } - NumericKind::Rational /* base: 10 */ => Numeric::Rational(f64::from_str(&buf).expect("Failed to parse float after checks")), + NumericKind::Rational /* base: 10 */ => { + let val = f64::from_str(&buf).expect("Failed to parse float after checks"); + let int_val = val as i32; + if (int_val as f64) == val { + Numeric::Integer(int_val) + } else { + Numeric::Rational(val) + } + }, NumericKind::Integer(base) => { if let Ok(num) = i32::from_str_radix(&buf, base as u32) { - if exp_str.is_empty() { - Numeric::Integer(num) - } else { - let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; - - if n < 0 { // A negative exponent is expected to produce a decimal value. - Numeric::Rational( - (num as f64) * f64::powi(10.0, n) - ) - } else if let Some(exp) = i32::checked_pow(10, n as u32) { - if let Some(val) = i32::checked_mul(num, exp) { - Numeric::Integer(val) - } else { - Numeric::Rational( - (num as f64) * (exp as f64) - ) - } - } else { - Numeric::Rational( - (num as f64) * f64::powi(10.0, n) - ) - } - } + Numeric::Integer(num) + + // if exp_str.is_empty() { + + // } else { + // let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; + + // if n < 0 { // A negative exponent is expected to produce a decimal value. + // Numeric::Rational( + // (num as f64) * f64::powi(10.0, n) + // ) + // } else if let Some(exp) = i32::checked_pow(10, n as u32) { + // if let Some(val) = i32::checked_mul(num, exp) { + // Numeric::Integer(val) + // } else { + // Numeric::Rational( + // (num as f64) * (exp as f64) + // ) + // } + // } else { + // Numeric::Rational( + // (num as f64) * f64::powi(10.0, n) + // ) + // } + // } } else { let b = f64::from(base); let mut result = 0.0_f64; @@ -305,14 +315,13 @@ impl Tokenizer for NumberLiteral { let digit = f64::from(c.to_digit(base as u32).expect("Couldn't parse digit after already checking validity")); result = result * b + digit; } + Numeric::Rational(result) - if exp_str.is_empty() { - Numeric::Rational( - result - ) - } else { - Numeric::Rational( result * f64::powi(10.0, i32::from_str(&exp_str).expect("Couldn't parse number after already checking validity"))) - } + // if exp_str.is_empty() { + + // } else { + // Numeric::Rational( result * f64::powi(10.0, i32::from_str(&exp_str).expect("Couldn't parse number after already checking validity"))) + // } } } }; diff --git a/boa/src/syntax/lexer/template.rs b/boa/src/syntax/lexer/template.rs index 7679d52c24e..99357017d8f 100644 --- a/boa/src/syntax/lexer/template.rs +++ b/boa/src/syntax/lexer/template.rs @@ -30,7 +30,7 @@ impl Tokenizer for TemplateLiteral { "Unterminated template literal", ))); } - Some('`') => break, // Template literal finished. + Some('`') => break, // Template literal finished. Some(next_ch) => buf.push(next_ch), // TODO when there is an expression inside the literal } } diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index d64656dfc4e..60d2489d643 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -76,7 +76,9 @@ fn check_template_literal_unterminated() { let s = "`I'm a template"; let mut lexer = Lexer::new(s.as_bytes()); - lexer.next().expect_err("Lexer did not handle unterminated literal with error"); + lexer + .next() + .expect_err("Lexer did not handle unterminated literal with error"); } #[test] @@ -344,18 +346,18 @@ fn numbers() { TokenKind::numeric_literal(52), TokenKind::numeric_literal(46), TokenKind::numeric_literal(7.89), - TokenKind::numeric_literal(42.0), + TokenKind::numeric_literal(42), TokenKind::numeric_literal(5000), TokenKind::numeric_literal(5000), TokenKind::numeric_literal(0.005), TokenKind::numeric_literal(2), TokenKind::numeric_literal(83), TokenKind::numeric_literal(999), - TokenKind::numeric_literal(10.0), + TokenKind::numeric_literal(10), TokenKind::numeric_literal(0.1), - TokenKind::numeric_literal(10.0), TokenKind::numeric_literal(10), - TokenKind::numeric_literal(0.0), + TokenKind::numeric_literal(10), + TokenKind::numeric_literal(0), TokenKind::numeric_literal(0.12), TokenKind::Punctuator(Punctuator::Sub), TokenKind::numeric_literal(32), @@ -430,7 +432,7 @@ fn number_followed_by_dot() { let mut lexer = Lexer::new(&b"1.."[0..]); let expected = [ - TokenKind::numeric_literal(1.0), + TokenKind::numeric_literal(1), TokenKind::Punctuator(Punctuator::Dot), ]; diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 3db2f625be3..4fb1d6d0391 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -163,7 +163,11 @@ where /// /// If skip is true then the token after the peek() token is checked instead. pub(super) fn peek_expect_no_lineterminator(&mut self, skip: bool) -> Result<(), ParseError> { - let token = if skip { self.peek_skip()? } else { self.peek()? }; + let token = if skip { + self.peek_skip()? + } else { + self.peek()? + }; if let Some(t) = token { if t.kind() == &TokenKind::LineTerminator { @@ -198,7 +202,7 @@ where /// Advance the cursor to skip 0, 1 or more line terminators. #[inline] - pub(super) fn skip_line_terminators(&mut self) -> Result<(),ParseError>{ + pub(super) fn skip_line_terminators(&mut self) -> Result<(), ParseError> { while self.next_if(TokenKind::LineTerminator)?.is_some() {} Ok(()) } diff --git a/boa/src/syntax/parser/expression/left_hand_side/mod.rs b/boa/src/syntax/parser/expression/left_hand_side/mod.rs index 9a0d96a1c7e..3c28eca4431 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/mod.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/mod.rs @@ -65,9 +65,9 @@ where // TODO: Implement NewExpression: new MemberExpression let lhs = MemberExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; - if let Some(tok) = cursor.peek()?{ + if let Some(tok) = cursor.peek()? { if tok.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) { - return CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor) + return CallExpression::new(self.allow_yield, self.allow_await, lhs).parse(cursor); } } Ok(lhs) diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index dd3f804a76e..14ec2e1e645 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -244,7 +244,7 @@ impl StatementList { break; } } else { - return Err(ParseError::AbruptEnd) + return Err(ParseError::AbruptEnd); } let item = @@ -280,9 +280,7 @@ where cursor.next(); continue; } - Some(token) - if token.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => - { + Some(token) if token.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => { if self.break_when_closingbraces { break; } else { From f06c23b25b5d39cc104e6489a1faf5bddd4ca0a4 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 6 Jul 2020 00:16:48 +0100 Subject: [PATCH 138/289] Fixing unused result warnings --- boa/src/syntax/lexer/mod.rs | 6 +-- boa/src/syntax/lexer/number.rs | 45 ++++--------------- boa/src/syntax/lexer/operator.rs | 6 +-- boa/src/syntax/lexer/regex.rs | 2 +- boa/src/syntax/lexer/string.rs | 4 +- boa/src/syntax/parser/cursor.rs | 4 +- .../expression/assignment/conditional.rs | 2 +- .../expression/assignment/exponentiation.rs | 2 +- .../parser/expression/assignment/mod.rs | 4 +- .../expression/left_hand_side/arguments.rs | 8 ++-- .../primary/array_initializer/mod.rs | 6 +-- boa/src/syntax/parser/expression/unary.rs | 14 +++--- boa/src/syntax/parser/statement/block/mod.rs | 2 +- boa/src/syntax/parser/statement/if_stm/mod.rs | 2 +- .../statement/iteration/do_while_statement.rs | 14 +++--- .../statement/iteration/while_statement.rs | 8 ++-- boa/src/syntax/parser/statement/mod.rs | 8 +--- 17 files changed, 52 insertions(+), 85 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 43329b511da..e52bdd8af97 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -105,11 +105,11 @@ impl Lexer { if let Some(c) = self.cursor.peek()? { match c { '/' => { - self.cursor.next(); // Consume the + self.cursor.next()?.expect("/ token vanished"); // Consume the '/' SingleLineComment.lex(&mut self.cursor, start) } '*' => { - self.cursor.next(); + self.cursor.next()?.expect("* token vanished"); // Consume the '*' BlockComment.lex(&mut self.cursor, start) } ch => { @@ -119,7 +119,7 @@ impl Lexer { if ch == '=' { // Indicates this is an AssignDiv. - self.cursor.next(); // Consume the '=' + self.cursor.next()?.expect("= token vanished"); // Consume the '=' Ok(Token::new( Punctuator::AssignDiv.into(), Span::new(start, self.cursor.pos()), diff --git a/boa/src/syntax/lexer/number.rs b/boa/src/syntax/lexer/number.rs index e6f25d73f2b..129221c7e89 100644 --- a/boa/src/syntax/lexer/number.rs +++ b/boa/src/syntax/lexer/number.rs @@ -136,7 +136,7 @@ impl Tokenizer for NumberLiteral { match ch { 'x' | 'X' => { // Remove the initial '0' from buffer. - cursor.next(); + cursor.next()?.expect("x or X character vanished"); buf.pop(); // HexIntegerLiteral @@ -144,7 +144,7 @@ impl Tokenizer for NumberLiteral { } 'o' | 'O' => { // Remove the initial '0' from buffer. - cursor.next(); + cursor.next()?.expect("o or O character vanished"); buf.pop(); // OctalIntegerLiteral @@ -152,14 +152,14 @@ impl Tokenizer for NumberLiteral { } 'b' | 'B' => { // Remove the initial '0' from buffer. - cursor.next(); + cursor.next()?.expect("b or B character vanished"); buf.pop(); // BinaryIntegerLiteral kind = NumericKind::Integer(2); } 'n' => { - cursor.next(); + cursor.next()?.expect("n character vanished"); // DecimalBigIntegerLiteral '0n' return Ok(Token::new( @@ -220,7 +220,7 @@ impl Tokenizer for NumberLiteral { // Lexing finished. // Consume the n - cursor.next(); + cursor.next()?.expect("n character vanished"); kind = kind.to_bigint(); } @@ -229,7 +229,7 @@ impl Tokenizer for NumberLiteral { // Only base 10 numbers can have a decimal seperator. // Number literal lexing finished if a . is found for a number in a different base. - cursor.next(); + cursor.next()?.expect(". token vanished"); buf.push('.'); // Consume the . kind = NumericKind::Rational; @@ -241,7 +241,7 @@ impl Tokenizer for NumberLiteral { match cursor.peek()? { Some('e') | Some('E') => { // Consume the ExponentIndicator. - cursor.next(); + cursor.next()?.expect("e or E token vanished"); buf.push('E'); @@ -276,6 +276,7 @@ impl Tokenizer for NumberLiteral { let val = f64::from_str(&buf).expect("Failed to parse float after checks"); let int_val = val as i32; if (int_val as f64) == val { + // For performance reasons we attempt to store values as integers if possible. Numeric::Integer(int_val) } else { Numeric::Rational(val) @@ -284,30 +285,6 @@ impl Tokenizer for NumberLiteral { NumericKind::Integer(base) => { if let Ok(num) = i32::from_str_radix(&buf, base as u32) { Numeric::Integer(num) - - // if exp_str.is_empty() { - - // } else { - // let n = i32::from_str(&exp_str).map_err(|_| Error::syntax("Could not convert value to f64"))?; - - // if n < 0 { // A negative exponent is expected to produce a decimal value. - // Numeric::Rational( - // (num as f64) * f64::powi(10.0, n) - // ) - // } else if let Some(exp) = i32::checked_pow(10, n as u32) { - // if let Some(val) = i32::checked_mul(num, exp) { - // Numeric::Integer(val) - // } else { - // Numeric::Rational( - // (num as f64) * (exp as f64) - // ) - // } - // } else { - // Numeric::Rational( - // (num as f64) * f64::powi(10.0, n) - // ) - // } - // } } else { let b = f64::from(base); let mut result = 0.0_f64; @@ -316,12 +293,6 @@ impl Tokenizer for NumberLiteral { result = result * b + digit; } Numeric::Rational(result) - - // if exp_str.is_empty() { - - // } else { - // Numeric::Rational( result * f64::powi(10.0, i32::from_str(&exp_str).expect("Couldn't parse number after already checking validity"))) - // } } } }; diff --git a/boa/src/syntax/lexer/operator.rs b/boa/src/syntax/lexer/operator.rs index 7669a03922d..e7cf08e65df 100644 --- a/boa/src/syntax/lexer/operator.rs +++ b/boa/src/syntax/lexer/operator.rs @@ -11,7 +11,7 @@ macro_rules! vop { match $cursor.peek()? { None => Err(Error::syntax("Abrupt end: could not preview next value as part of operator")), Some('=') => { - $cursor.next(); + $cursor.next()?.expect("= token vanished"); $cursor.next_column(); $assign_op } @@ -23,12 +23,12 @@ macro_rules! vop { match $cursor.peek()? { None => Err(Error::syntax("Abrupt end: could not preview next value as part of operator")), Some('=') => { - $cursor.next(); + $cursor.next()?.expect("= token vanished"); $cursor.next_column(); $assign_op }, $($case => { - $cursor.next(); + $cursor.next()?.expect("Token vanished"); $cursor.next_column(); $block })+, diff --git a/boa/src/syntax/lexer/regex.rs b/boa/src/syntax/lexer/regex.rs index 165b5e184ff..3a52d0ce778 100644 --- a/boa/src/syntax/lexer/regex.rs +++ b/boa/src/syntax/lexer/regex.rs @@ -2,7 +2,7 @@ use super::{Cursor, Error, Span, Tokenizer}; use crate::syntax::ast::Position; use crate::syntax::lexer::Token; use crate::syntax::lexer::TokenKind; -use std::io::{self, ErrorKind, Read}; +use std::io::Read; /// Regex literal lexing. /// diff --git a/boa/src/syntax/lexer/string.rs b/boa/src/syntax/lexer/string.rs index fa11192fc7c..bb0633cd30a 100644 --- a/boa/src/syntax/lexer/string.rs +++ b/boa/src/syntax/lexer/string.rs @@ -108,13 +108,13 @@ impl Tokenizer for StringLiteral { // Support \u{X..X} (Unicode Codepoint) if cursor.next_is('{')? { - cursor.next(); // Consume the '{'. + cursor.next()?.expect("{ character vanished"); // Consume the '{'. // The biggest code point is 0x10FFFF let mut code_point = String::with_capacity(6); cursor.take_until('}', &mut code_point)?; - cursor.next(); // Consume the '}'. + cursor.next()?.expect("} character vanished"); // Consume the '}'. // We know this is a single unicode codepoint, convert to u32 let as_num = diff --git a/boa/src/syntax/parser/cursor.rs b/boa/src/syntax/parser/cursor.rs index 4fb1d6d0391..08a1e600493 100644 --- a/boa/src/syntax/parser/cursor.rs +++ b/boa/src/syntax/parser/cursor.rs @@ -105,7 +105,7 @@ where let kind = kind.into(); if next_token.kind() == &kind { - self.next(); + self.next()?.expect("Token vanished"); Ok(next_token) } else { Err(ParseError::expected(vec![kind], next_token, context)) @@ -142,7 +142,7 @@ where match self.peek_semicolon()? { (true, Some(tk)) => match tk.kind() { TokenKind::Punctuator(Punctuator::Semicolon) | TokenKind::LineTerminator => { - self.next(); // Consume the token. + self.next()?.expect("Token vanished"); // Consume the token. Ok(Some(tk)) } _ => Ok(Some(tk)), diff --git a/boa/src/syntax/parser/expression/assignment/conditional.rs b/boa/src/syntax/parser/expression/assignment/conditional.rs index 82d587b4c4e..de0f8b45f16 100644 --- a/boa/src/syntax/parser/expression/assignment/conditional.rs +++ b/boa/src/syntax/parser/expression/assignment/conditional.rs @@ -70,7 +70,7 @@ where if let Some(tok) = cursor.peek()? { if tok.kind() == &TokenKind::Punctuator(Punctuator::Question) { - cursor.next(); // Consume the token. + cursor.next()?.expect("? character vanished"); // Consume the token. let then_clause = AssignmentExpression::new(self.allow_in, self.allow_yield, self.allow_await) .parse(cursor)?; diff --git a/boa/src/syntax/parser/expression/assignment/exponentiation.rs b/boa/src/syntax/parser/expression/assignment/exponentiation.rs index 4145c64ce28..6ce891df97f 100644 --- a/boa/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa/src/syntax/parser/expression/assignment/exponentiation.rs @@ -90,7 +90,7 @@ where let lhs = UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor)?; if let Some(tok) = cursor.peek()? { if let TokenKind::Punctuator(Punctuator::Exp) = tok.kind() { - cursor.next(); // Consume the token. + cursor.next()?.expect("** token vanished"); // Consume the token. return Ok(BinOp::new(NumOp::Exp, lhs, self.parse(cursor)?).into()); } } diff --git a/boa/src/syntax/parser/expression/assignment/mod.rs b/boa/src/syntax/parser/expression/assignment/mod.rs index a1933b0431d..3f9f12431e8 100644 --- a/boa/src/syntax/parser/expression/assignment/mod.rs +++ b/boa/src/syntax/parser/expression/assignment/mod.rs @@ -125,11 +125,11 @@ where if let Some(tok) = cursor.peek()? { match tok.kind() { TokenKind::Punctuator(Punctuator::Assign) => { - cursor.next(); // Consume the token. + cursor.next()?.expect("= token vanished"); // Consume the token. lhs = Assign::new(lhs, self.parse(cursor)?).into(); } TokenKind::Punctuator(p) if p.as_binop().is_some() => { - cursor.next(); // Consume the token. + cursor.next()?.expect("Token vanished"); // Consume the token. let expr = self.parse(cursor)?; let binop = p.as_binop().expect("binop disappeared"); diff --git a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs index 88c11f66d72..21ec6e2a88e 100644 --- a/boa/src/syntax/parser/expression/left_hand_side/arguments.rs +++ b/boa/src/syntax/parser/expression/left_hand_side/arguments.rs @@ -62,16 +62,16 @@ where cursor.expect(Punctuator::OpenParen, "arguments")?; let mut args = Vec::new(); loop { - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; let next_token = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; match next_token.kind() { TokenKind::Punctuator(Punctuator::CloseParen) => { - cursor.next(); // Consume the token. + cursor.next()?.expect(") token vanished"); // Consume the token. break; } TokenKind::Punctuator(Punctuator::Comma) => { - cursor.next(); // Consume the token. + cursor.next()?.expect(", token vanished"); // Consume the token. if args.is_empty() { return Err(ParseError::unexpected(next_token.clone(), None)); @@ -83,7 +83,7 @@ where } _ => { if !args.is_empty() { - cursor.next(); // Consume the token. + cursor.next()?.expect("Token vanished"); // Consume the token. return Err(ParseError::expected( vec![ TokenKind::Punctuator(Punctuator::Comma), diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs index d702a9abf39..be21feaf0e4 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/mod.rs @@ -65,7 +65,7 @@ where let mut elements = Vec::new(); loop { - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; // TODO: Support all features. while cursor.next_if(Punctuator::Comma)?.is_some() { elements.push(Node::Const(Const::Undefined)); @@ -87,8 +87,8 @@ where .parse(cursor)?, ); } - cursor.skip_line_terminators(); - cursor.next_if(Punctuator::Comma); + cursor.skip_line_terminators()?; + cursor.next_if(Punctuator::Comma)?; } Ok(elements.into()) diff --git a/boa/src/syntax/parser/expression/unary.rs b/boa/src/syntax/parser/expression/unary.rs index 7ff8ea95ecc..43dc4e9f93c 100644 --- a/boa/src/syntax/parser/expression/unary.rs +++ b/boa/src/syntax/parser/expression/unary.rs @@ -60,31 +60,31 @@ where let tok = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { TokenKind::Keyword(Keyword::Delete) => { - cursor.next(); // Consume the token. + cursor.next()?.expect("Delete keyword vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Delete, self.parse(cursor)?).into()) } TokenKind::Keyword(Keyword::Void) => { - cursor.next(); // Consume the token. + cursor.next()?.expect("Void keyword vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Void, self.parse(cursor)?).into()) } TokenKind::Keyword(Keyword::TypeOf) => { - cursor.next(); // Consume the token. + cursor.next()?.expect("TypeOf keyword vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::TypeOf, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Add) => { - cursor.next(); // Consume the token. + cursor.next()?.expect("+ token vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Plus, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Sub) => { - cursor.next(); // Consume the token. + cursor.next()?.expect("- token vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Minus, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Neg) => { - cursor.next(); // Consume the token. + cursor.next()?.expect("~ token vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Tilde, self.parse(cursor)?).into()) } TokenKind::Punctuator(Punctuator::Not) => { - cursor.next(); // Consume the token. + cursor.next()?.expect("! token vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Not, self.parse(cursor)?).into()) } _ => UpdateExpression::new(self.allow_yield, self.allow_await).parse(cursor), diff --git a/boa/src/syntax/parser/statement/block/mod.rs b/boa/src/syntax/parser/statement/block/mod.rs index 96eb7cfb52d..86d51cd056a 100644 --- a/boa/src/syntax/parser/statement/block/mod.rs +++ b/boa/src/syntax/parser/statement/block/mod.rs @@ -73,7 +73,7 @@ where cursor.expect(Punctuator::OpenBlock, "block")?; if let Some(tk) = cursor.peek()? { if tk.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) { - cursor.next(); + cursor.next()?.expect("} token vanished"); return Ok(node::Block::from(vec![])); } } diff --git a/boa/src/syntax/parser/statement/if_stm/mod.rs b/boa/src/syntax/parser/statement/if_stm/mod.rs index 957a18f786d..f8a009b2bb8 100644 --- a/boa/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa/src/syntax/parser/statement/if_stm/mod.rs @@ -72,7 +72,7 @@ where let else_stm = match else_tok { Some(_) if else_tok.unwrap().kind() == &TokenKind::Keyword(Keyword::Else) => { - cursor.next(); + cursor.next()?.expect("Else token vanished"); Some( Statement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor)?, diff --git a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs index f82c80bc382..310bc5a59bf 100644 --- a/boa/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -66,12 +66,12 @@ where cursor.expect(Keyword::Do, "do while statement")?; // There can be space between the Do and the body. - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; let body = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; let next_token = cursor.peek()?.ok_or(ParseError::AbruptEnd)?; @@ -83,19 +83,19 @@ where )); } - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; cursor.expect(Keyword::While, "do while statement")?; - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; cursor.expect(Punctuator::OpenParen, "do while statement")?; - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; cursor.expect(Punctuator::CloseParen, "do while statement")?; @@ -120,7 +120,7 @@ where if let Some(tk) = cursor.peek()? { if tk.kind() == &TokenKind::Punctuator(Punctuator::Semicolon) { - cursor.next(); // Consume semicolon. + cursor.next()?.expect("; token vanished"); // Consume semicolon. } } diff --git a/boa/src/syntax/parser/statement/iteration/while_statement.rs b/boa/src/syntax/parser/statement/iteration/while_statement.rs index 211afd6bc03..5801ca8f4d3 100644 --- a/boa/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa/src/syntax/parser/statement/iteration/while_statement.rs @@ -57,19 +57,19 @@ where cursor.expect(Keyword::While, "while statement")?; // Line terminators can exist between a While and the condition. - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; cursor.expect(Punctuator::OpenParen, "while statement")?; - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; let cond = Expression::new(true, self.allow_yield, self.allow_await).parse(cursor)?; - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; cursor.expect(Punctuator::CloseParen, "while statement")?; - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; let body = Statement::new(self.allow_yield, self.allow_await, self.allow_return).parse(cursor)?; diff --git a/boa/src/syntax/parser/statement/mod.rs b/boa/src/syntax/parser/statement/mod.rs index 14ec2e1e645..e8ee553d11a 100644 --- a/boa/src/syntax/parser/statement/mod.rs +++ b/boa/src/syntax/parser/statement/mod.rs @@ -238,7 +238,7 @@ impl StatementList { let mut items = Vec::new(); loop { - cursor.skip_line_terminators(); + cursor.skip_line_terminators()?; if let Some(token) = cursor.peek()? { if break_nodes.contains(token.kind()) { break; @@ -274,12 +274,8 @@ where let mut items = Vec::new(); loop { + cursor.skip_line_terminators()?; match cursor.peek()? { - Some(token) if token.kind() == &TokenKind::LineTerminator => { - // Skip line terminators. - cursor.next(); - continue; - } Some(token) if token.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => { if self.break_when_closingbraces { break; From 717b379874370cc782914a3fa34b8b0cbffbb326 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Mon, 6 Jul 2020 00:41:10 +0100 Subject: [PATCH 139/289] Finished refractor --- boa/src/syntax/lexer/cursor.rs | 26 ++++++++----------- boa/src/syntax/lexer/mod.rs | 1 + boa/src/syntax/lexer/number.rs | 6 ++++- boa/src/syntax/parser/statement/mod.rs | 6 ++--- boa/src/syntax/parser/statement/switch/mod.rs | 20 +++++++------- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/boa/src/syntax/lexer/cursor.rs b/boa/src/syntax/lexer/cursor.rs index c5015cebc5a..c916f5e2c1d 100644 --- a/boa/src/syntax/lexer/cursor.rs +++ b/boa/src/syntax/lexer/cursor.rs @@ -73,7 +73,7 @@ where let _ = self.peeked.take(); true } - None | _ => false, + _ => false, }) } @@ -100,15 +100,13 @@ where loop { if self.next_is(stop)? { return Ok(()); + } else if let Some(ch) = self.next()? { + buf.push(ch); } else { - if let Some(ch) = self.next()? { - buf.push(ch); - } else { - return Err(io::Error::new( - ErrorKind::UnexpectedEof, - format!("Unexpected end of file when looking for character {}", stop), - )); - } + return Err(io::Error::new( + ErrorKind::UnexpectedEof, + format!("Unexpected end of file when looking for character {}", stop), + )); } } } @@ -124,13 +122,11 @@ where loop { if !self.next_is_pred(pred)? { return Ok(()); + } else if let Some(ch) = self.next()? { + buf.push(ch); } else { - if let Some(ch) = self.next()? { - buf.push(ch); - } else { - // next_is_pred will return false if the next value is None so the None case should already be handled. - unreachable!(); - } + // next_is_pred will return false if the next value is None so the None case should already be handled. + unreachable!(); } } } diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index e52bdd8af97..c6b6432ab5d 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -143,6 +143,7 @@ impl Lexer { } } + #[allow(clippy::should_implement_trait)] // We intentionally don't implement Iterator trait as Result