From 7408fe74ff697126bb616dcca4df82bf737ee87d Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Sun, 3 Oct 2021 22:43:03 -0400 Subject: [PATCH 01/21] initial hashbang sup logic in Lexer new() --- boa/src/syntax/lexer/mod.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 35c700e850c..aae63305f0e 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -107,9 +107,19 @@ impl Lexer { where R: Read, { - Self { - cursor: Cursor::new(reader), - goal_symbol: Default::default(), + let mut init_cursor = Cursor::new(reader); + match init_cursor.peek_n(2) { + Ok(hashbang_peek) if hashbang_peek == 0x2123 => { + init_cursor.next_line(); + Self { + cursor: init_cursor, + goal_symbol: Default::default(), + } + } + _ => Self { + cursor: init_cursor, + goal_symbol: Default::default(), + }, } } From edd14a0555dd179caa88bc30a0d8aa77a3fa4cf8 Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Mon, 4 Oct 2021 15:11:09 -0400 Subject: [PATCH 02/21] implemented basic concept for hashbang lexing --- boa/src/syntax/lexer/comment.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 88e5238839e..3386d1fc391 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -90,3 +90,36 @@ impl Tokenizer for MultiLineComment { )) } } + +pub(super) struct Hashbang; + +impl Tokenizer for Hashbang { + fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result + where + R: Read, + { + let _timer = BoaProfiler::global().start_event("Hashbang", "Lexing"); + + loop { + if let Some(ch) = cursor.peek()? { + if ch == b'\r' || ch == b'\n' { + //still want to consume the byte to move to next line + cursor.next_byte()?.expect("No byte returned") + break; + } else { + cursor.next_byte()?.expect("No byte returned"); + } + } else { + return Err(Error::syntax( + "unterminated hashbang", + cursor.pos()) + ); + } + } + + Ok(Token::new( + TokenKind::Comment, + Span::new(start_pos, cursor.pos()) + )) + } +} \ No newline at end of file From e6c0a3e26bfad04d031b2df966b6be9f2368f9f6 Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Mon, 4 Oct 2021 15:11:45 -0400 Subject: [PATCH 03/21] minor changes - missed semi-colon --- boa/src/syntax/lexer/comment.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 3386d1fc391..03bf838bda5 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -104,22 +104,19 @@ impl Tokenizer for Hashbang { if let Some(ch) = cursor.peek()? { if ch == b'\r' || ch == b'\n' { //still want to consume the byte to move to next line - cursor.next_byte()?.expect("No byte returned") + cursor.next_byte()?.expect("No byte returned"); break; } else { cursor.next_byte()?.expect("No byte returned"); } } else { - return Err(Error::syntax( - "unterminated hashbang", - cursor.pos()) - ); + return Err(Error::syntax("unterminated hashbang", cursor.pos())); } } Ok(Token::new( TokenKind::Comment, - Span::new(start_pos, cursor.pos()) + Span::new(start_pos, cursor.pos()), )) } -} \ No newline at end of file +} From 2eb3e74fcf926f6d9451191263c8efa1be042c1b Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Mon, 4 Oct 2021 17:02:29 -0400 Subject: [PATCH 04/21] cleaning calls in Hashbang.lex() --- boa/src/syntax/lexer/comment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 03bf838bda5..bf047251dc0 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -104,7 +104,7 @@ impl Tokenizer for Hashbang { if let Some(ch) = cursor.peek()? { if ch == b'\r' || ch == b'\n' { //still want to consume the byte to move to next line - cursor.next_byte()?.expect("No byte returned"); + cursor.next_byte()?; break; } else { cursor.next_byte()?.expect("No byte returned"); From 65278fa200e44e0fb90179fbcd31cd2604d0cf35 Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Mon, 4 Oct 2021 17:03:27 -0400 Subject: [PATCH 05/21] move hashbang lex from new() to next() --- boa/src/syntax/lexer/mod.rs | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index aae63305f0e..aa985226a78 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -30,7 +30,7 @@ pub mod token; mod tests; use self::{ - comment::{MultiLineComment, SingleLineComment}, + comment::{Hashbang, MultiLineComment, SingleLineComment}, cursor::Cursor, identifier::Identifier, number::NumberLiteral, @@ -107,19 +107,9 @@ impl Lexer { where R: Read, { - let mut init_cursor = Cursor::new(reader); - match init_cursor.peek_n(2) { - Ok(hashbang_peek) if hashbang_peek == 0x2123 => { - init_cursor.next_line(); - Self { - cursor: init_cursor, - goal_symbol: Default::default(), - } - } - _ => Self { - cursor: init_cursor, - goal_symbol: Default::default(), - }, + Self { + cursor: Cursor::new(reader), + goal_symbol: Default::default(), } } @@ -201,6 +191,15 @@ impl Lexer { } }; + if start.column_number() == 1 && start.line_number() == 1 && next_ch == 0x23 { + if let Some(hashbang_peek) = self.cursor.peek()? { + if hashbang_peek == 0x21 { + let _token = Hashbang.lex(&mut self.cursor, start); + return self.next(); + } + } + }; + if let Ok(c) = char::try_from(next_ch) { let token = match c { '\r' | '\n' | '\u{2028}' | '\u{2029}' => Ok(Token::new( From 318e9d022769fcdc65250059fb93fdda537e7e0d Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Mon, 4 Oct 2021 17:08:44 -0400 Subject: [PATCH 06/21] add comment to to hashbang block --- boa/src/syntax/lexer/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index aa985226a78..1753faf520d 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -191,6 +191,8 @@ impl Lexer { } }; + //handle hashbang here so the below match block still throws error on + //# if position isn't (1, 1) if start.column_number() == 1 && start.line_number() == 1 && next_ch == 0x23 { if let Some(hashbang_peek) = self.cursor.peek()? { if hashbang_peek == 0x21 { From 820b0651b3a5df9cca88aad1904b87aa013777cc Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Mon, 4 Oct 2021 19:48:45 -0400 Subject: [PATCH 07/21] Hashbang to HashbangComment - more comments added --- boa/src/syntax/lexer/comment.rs | 11 +++++++++-- boa/src/syntax/lexer/mod.rs | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index bf047251dc0..cc1fd280ea4 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -91,9 +91,16 @@ impl Tokenizer for MultiLineComment { } } -pub(super) struct Hashbang; +///Lexes a first line Hashbang comment +/// +/// More information: +/// - [ECMAScript reference][spec] +/// +/// [spec]: https://tc39.es/ecma262/#sec-ecmascript-language-lexical-grammar + +pub(super) struct HashbangComment; -impl Tokenizer for Hashbang { +impl Tokenizer for HashbangComment { fn lex(&mut self, cursor: &mut Cursor, start_pos: Position) -> Result where R: Read, diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 1753faf520d..e2ed8227e65 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -30,7 +30,7 @@ pub mod token; mod tests; use self::{ - comment::{Hashbang, MultiLineComment, SingleLineComment}, + comment::{HashbangComment, MultiLineComment, SingleLineComment}, cursor::Cursor, identifier::Identifier, number::NumberLiteral, @@ -196,7 +196,7 @@ impl Lexer { if start.column_number() == 1 && start.line_number() == 1 && next_ch == 0x23 { if let Some(hashbang_peek) = self.cursor.peek()? { if hashbang_peek == 0x21 { - let _token = Hashbang.lex(&mut self.cursor, start); + let _token = HashbangComment.lex(&mut self.cursor, start); return self.next(); } } From 6eb79c3255c6b0d85a714bccf77c61353fd22733 Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Mon, 4 Oct 2021 21:57:48 -0400 Subject: [PATCH 08/21] changed Hashbang lex - added support for u2028 and u2029 --- boa/src/syntax/lexer/comment.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index cc1fd280ea4..97bf215aa96 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -8,6 +8,7 @@ use crate::{ lexer::{Token, TokenKind}, }, }; +use core::convert::TryFrom; use std::io::Read; /// Lexes a single line comment. @@ -108,11 +109,14 @@ impl Tokenizer for HashbangComment { let _timer = BoaProfiler::global().start_event("Hashbang", "Lexing"); loop { - if let Some(ch) = cursor.peek()? { - if ch == b'\r' || ch == b'\n' { - //still want to consume the byte to move to next line - cursor.next_byte()?; - break; + if let Some(ch) = cursor.next_char()? { + if let Ok(c) = char::try_from(ch) { + if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' { + //still want to consume the byte to move to next line + println!("Found line break char: {}", c); + cursor.next_byte()?; + break; + } } else { cursor.next_byte()?.expect("No byte returned"); } From 0eef98039442e6eb6385b111802ae0e7dd29df1a Mon Sep 17 00:00:00 2001 From: nekevss <46825870+nekevss@users.noreply.github.com> Date: Mon, 4 Oct 2021 22:12:03 -0400 Subject: [PATCH 09/21] removing debug print statement --- boa/src/syntax/lexer/comment.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 97bf215aa96..d8a959cbb64 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -113,7 +113,6 @@ impl Tokenizer for HashbangComment { if let Ok(c) = char::try_from(ch) { if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' { //still want to consume the byte to move to next line - println!("Found line break char: {}", c); cursor.next_byte()?; break; } From a4491b317adda2a55e36c3f9a9e61f7c2df6946a Mon Sep 17 00:00:00 2001 From: nekevss Date: Tue, 5 Oct 2021 19:08:01 -0400 Subject: [PATCH 10/21] Removing byte consumption from hashbang.lex break statement --- boa/src/syntax/lexer/comment.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index d8a959cbb64..3414d0a5807 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -113,7 +113,6 @@ impl Tokenizer for HashbangComment { if let Ok(c) = char::try_from(ch) { if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' { //still want to consume the byte to move to next line - cursor.next_byte()?; break; } } else { From acbf3c9a2b4bfb8d77c1b0522d8ea22bd3639cf1 Mon Sep 17 00:00:00 2001 From: nekevss Date: Tue, 5 Oct 2021 19:09:42 -0400 Subject: [PATCH 11/21] removing comment linked to byte consumption --- boa/src/syntax/lexer/comment.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 3414d0a5807..7c39a1d50a7 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -112,7 +112,6 @@ impl Tokenizer for HashbangComment { if let Some(ch) = cursor.next_char()? { if let Ok(c) = char::try_from(ch) { if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' { - //still want to consume the byte to move to next line break; } } else { From 5688a7c316b1aabdac856878eb595f82f1461073 Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 18:49:48 -0400 Subject: [PATCH 12/21] Added some hashbang tests to parser/tests.rs --- boa/src/syntax/parser/tests.rs | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 12fae2072e4..fdc09f864f3 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -372,3 +372,39 @@ fn empty_statement() { ], ); } + +#[test] +fn hashbang_use_strict_no_with() { + check_parser( + r#"#!\"use strict" + "#, + vec![] + ); +} + +#[test] +fn hashbang_use_strict_with_with_statement() { + check_parser( + r#"#!\"use strict" + + with({}) {} + "#, + vec![] + ); +} + +#[test] +fn hashbang_comment() { + check_parser( + r"#!Comment Here", + vec![] + ); +} + +#[test] +fn hashbang_line_separator_test() { + check_parser( + r"#!This contains a line separator
", + vec![] + ); +} \ No newline at end of file From 44186dff537178c715867caa4cb17c4cfc68b000 Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 19:58:04 -0400 Subject: [PATCH 13/21] removed line separator hashbang test --- boa/src/syntax/parser/tests.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index fdc09f864f3..7aabd725f89 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -399,12 +399,4 @@ fn hashbang_comment() { r"#!Comment Here", vec![] ); -} - -#[test] -fn hashbang_line_separator_test() { - check_parser( - r"#!This contains a line separator
", - vec![] - ); } \ No newline at end of file From 2dfa9d476b5936ede30b467bc676d9c4fb4e35c7 Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 20:10:38 -0400 Subject: [PATCH 14/21] switched loop if let to while let --- boa/src/syntax/lexer/comment.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 7c39a1d50a7..583f93d7f2e 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -108,17 +108,13 @@ impl Tokenizer for HashbangComment { { let _timer = BoaProfiler::global().start_event("Hashbang", "Lexing"); - loop { - if let Some(ch) = cursor.next_char()? { - if let Ok(c) = char::try_from(ch) { - if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' { - break; - } - } else { - cursor.next_byte()?.expect("No byte returned"); + while let Some(ch) = cursor.next_char()? { + if let Ok(c) = char::try_from(ch) { + if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' { + break; } } else { - return Err(Error::syntax("unterminated hashbang", cursor.pos())); + cursor.next_byte()?.expect("No byte returned"); } } From 48c5eacb511cc74897234059298c0596ae9b2961 Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 20:18:29 -0400 Subject: [PATCH 15/21] comment out failing use strict with test for now --- boa/src/syntax/parser/tests.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 7aabd725f89..e0bf2cab966 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -382,6 +382,8 @@ fn hashbang_use_strict_no_with() { ); } +//commenting out this test for when with is enabled. +/* #[test] fn hashbang_use_strict_with_with_statement() { check_parser( @@ -392,6 +394,7 @@ fn hashbang_use_strict_with_with_statement() { vec![] ); } +*/ #[test] fn hashbang_comment() { From 57fd8cd1405c0e5e783c6ff44b6a0a2f2979b7da Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 21:53:40 -0400 Subject: [PATCH 16/21] changed comment out to ignore --- boa/src/syntax/parser/tests.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index e0bf2cab966..7dacd666ced 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -382,9 +382,8 @@ fn hashbang_use_strict_no_with() { ); } -//commenting out this test for when with is enabled. -/* #[test] +#[ignore] fn hashbang_use_strict_with_with_statement() { check_parser( r#"#!\"use strict" @@ -394,7 +393,6 @@ fn hashbang_use_strict_with_with_statement() { vec![] ); } -*/ #[test] fn hashbang_comment() { From 257db0392b0327fbdbc9d05ab60ab27cf1034c97 Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 21:55:11 -0400 Subject: [PATCH 17/21] adding new line to end of file --- boa/src/syntax/parser/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 7dacd666ced..8379cf686a9 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -400,4 +400,4 @@ fn hashbang_comment() { r"#!Comment Here", vec![] ); -} \ No newline at end of file +} From 62d1e74e714f8715071c6e5f38646d9312c3682a Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 22:23:54 -0400 Subject: [PATCH 18/21] running cargo fmt --- boa/src/syntax/parser/tests.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 8379cf686a9..f8999a0244c 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -377,8 +377,8 @@ fn empty_statement() { fn hashbang_use_strict_no_with() { check_parser( r#"#!\"use strict" - "#, - vec![] + "#, + vec![], ); } @@ -389,15 +389,12 @@ fn hashbang_use_strict_with_with_statement() { r#"#!\"use strict" with({}) {} - "#, - vec![] + "#, + vec![], ); } #[test] fn hashbang_comment() { - check_parser( - r"#!Comment Here", - vec![] - ); + check_parser(r"#!Comment Here", vec![]); } From 756fc46237c8f56a291df2ac46fce2c6d72dc252 Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 23:31:38 -0400 Subject: [PATCH 19/21] MultiLineComment/Hashbang match update --- boa/src/syntax/lexer/comment.rs | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 583f93d7f2e..87489ab06e5 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -66,19 +66,13 @@ impl Tokenizer for MultiLineComment { let _timer = BoaProfiler::global().start_event("MultiLineComment", "Lexing"); let mut new_line = false; - loop { - if let Some(ch) = cursor.next_byte()? { - if ch == b'*' && cursor.next_is(b'/')? { - break; - } else if ch == b'\n' { - new_line = true; - } - } else { - return Err(Error::syntax( - "unterminated multiline comment", - cursor.pos(), - )); - } + while let Some(ch) = cursor.next_char()? { + let tried_ch = char::try_from(ch); + match tried_ch { + Ok(c) if c == '*' && cursor.next_is(b'/')? => break, + Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => new_line = true, + _ => {}, + }; } Ok(Token::new( @@ -109,13 +103,11 @@ impl Tokenizer for HashbangComment { let _timer = BoaProfiler::global().start_event("Hashbang", "Lexing"); while let Some(ch) = cursor.next_char()? { - if let Ok(c) = char::try_from(ch) { - if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' { - break; - } - } else { - cursor.next_byte()?.expect("No byte returned"); - } + let tried_ch = char::try_from(ch); + match tried_ch { + Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => break, + _ => {}, + }; } Ok(Token::new( From f765c43913265db3c561c25ca65527d68aca3010 Mon Sep 17 00:00:00 2001 From: nekevss Date: Wed, 6 Oct 2021 23:34:00 -0400 Subject: [PATCH 20/21] cargo fmt update --- boa/src/syntax/lexer/comment.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 87489ab06e5..719c1359bc5 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -67,11 +67,13 @@ impl Tokenizer for MultiLineComment { let mut new_line = false; while let Some(ch) = cursor.next_char()? { - let tried_ch = char::try_from(ch); + let tried_ch = char::try_from(ch); match tried_ch { Ok(c) if c == '*' && cursor.next_is(b'/')? => break, - Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => new_line = true, - _ => {}, + Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => { + new_line = true + } + _ => {} }; } @@ -106,7 +108,7 @@ impl Tokenizer for HashbangComment { let tried_ch = char::try_from(ch); match tried_ch { Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => break, - _ => {}, + _ => {} }; } From 0b3ec3c9a6763b3ac95f5135c16729a3e3d1dc42 Mon Sep 17 00:00:00 2001 From: nekevss Date: Thu, 7 Oct 2021 00:22:46 -0400 Subject: [PATCH 21/21] Rework to include unterminated error --- boa/src/syntax/lexer/comment.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/boa/src/syntax/lexer/comment.rs b/boa/src/syntax/lexer/comment.rs index 719c1359bc5..24c94c1903a 100644 --- a/boa/src/syntax/lexer/comment.rs +++ b/boa/src/syntax/lexer/comment.rs @@ -69,7 +69,16 @@ impl Tokenizer for MultiLineComment { while let Some(ch) = cursor.next_char()? { let tried_ch = char::try_from(ch); match tried_ch { - Ok(c) if c == '*' && cursor.next_is(b'/')? => break, + Ok(c) if c == '*' && cursor.next_is(b'/')? => { + return Ok(Token::new( + if new_line { + TokenKind::LineTerminator + } else { + TokenKind::Comment + }, + Span::new(start_pos, cursor.pos()), + )) + } Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => { new_line = true } @@ -77,13 +86,9 @@ impl Tokenizer for MultiLineComment { }; } - Ok(Token::new( - if new_line { - TokenKind::LineTerminator - } else { - TokenKind::Comment - }, - Span::new(start_pos, cursor.pos()), + Err(Error::syntax( + "unterminated multiline comment", + cursor.pos(), )) } }