From 177f973d88eb9528c0b9c53dfa3439b5645dc988 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 21 Dec 2024 15:36:03 -0500 Subject: [PATCH] Support leading logical operators --- src/prism.c | 83 ++++++++++++++++++++++-- test/prism/fixtures/leading_logical.txt | 15 +++++ test/prism/fixtures_test.rb | 2 + test/prism/lex_test.rb | 4 ++ test/prism/ruby/ripper_test.rb | 3 + test/prism/ruby/ruby_parser_test.rb | 1 + test/prism/snapshots/leading_logical.txt | 83 ++++++++++++++++++++++++ 7 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 test/prism/fixtures/leading_logical.txt create mode 100644 test/prism/snapshots/leading_logical.txt diff --git a/src/prism.c b/src/prism.c index ab8de969757..c40582c1810 100644 --- a/src/prism.c +++ b/src/prism.c @@ -10829,14 +10829,37 @@ parser_lex(pm_parser_t *parser) { following = next_newline(following, parser->end - following); } - // If the lex state was ignored, or we hit a '.' or a '&.', - // we will lex the ignored newline + // If the lex state was ignored, we will lex the + // ignored newline. + if (lex_state_ignored_p(parser)) { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lexed_comment = false; + goto lex_next_token; + } + + // If we hit a '.' or a '&.' we will lex the ignored + // newline. + if (following && ( + (peek_at(parser, following) == '.') || + (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '.') + )) { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lexed_comment = false; + goto lex_next_token; + } + + + // If we are parsing as CRuby 3.5 or later and we + // hit a '&&' or a '||' then we will lex the ignored + // newline. if ( - lex_state_ignored_p(parser) || - (following && ( - (peek_at(parser, following) == '.') || - (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '.') - )) + (parser->version == PM_OPTIONS_VERSION_LATEST) && + following && ( + (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '&') || + (peek_at(parser, following) == '|' && peek_at(parser, following + 1) == '|') || + (peek_at(parser, following) == 'a' && peek_at(parser, following + 1) == 'n' && peek_at(parser, following + 2) == 'd') || + (peek_at(parser, following) == 'o' && peek_at(parser, following + 1) == 'r') + ) ) { if (!lexed_comment) parser_lex_ignored_newline(parser); lexed_comment = false; @@ -10876,6 +10899,52 @@ parser_lex(pm_parser_t *parser) { parser->next_start = NULL; LEX(PM_TOKEN_AMPERSAND_DOT); } + + if (parser->version == PM_OPTIONS_VERSION_LATEST) { + // If we hit an && then we are in a logical chain + // and we need to return the logical operator. + if (peek_at(parser, next_content) == '&' && peek_at(parser, next_content + 1) == '&') { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lex_state_set(parser, PM_LEX_STATE_BEG); + parser->current.start = next_content; + parser->current.end = next_content + 2; + parser->next_start = NULL; + LEX(PM_TOKEN_AMPERSAND_AMPERSAND); + } + + // If we hit an 'and' then we are in a logical chain + // and we need to return the logical operator. + if (peek_at(parser, next_content) == 'a' && peek_at(parser, next_content + 1) == 'n' && peek_at(parser, next_content + 2) == 'd') { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lex_state_set(parser, PM_LEX_STATE_BEG); + parser->current.start = next_content; + parser->current.end = next_content + 3; + parser->next_start = NULL; + LEX(PM_TOKEN_KEYWORD_AND); + } + + // If we hit a || then we are in a logical chain and + // we need to return the logical operator. + if (peek_at(parser, next_content) == '|' && peek_at(parser, next_content + 1) == '|') { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lex_state_set(parser, PM_LEX_STATE_BEG); + parser->current.start = next_content; + parser->current.end = next_content + 2; + parser->next_start = NULL; + LEX(PM_TOKEN_PIPE_PIPE); + } + + // If we hit a 'or' then we are in a logical chain + // and we need to return the logical operator. + if (peek_at(parser, next_content) == 'o' && peek_at(parser, next_content + 1) == 'r') { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lex_state_set(parser, PM_LEX_STATE_BEG); + parser->current.start = next_content; + parser->current.end = next_content + 2; + parser->next_start = NULL; + LEX(PM_TOKEN_KEYWORD_OR); + } + } } // At this point we know this is a regular newline, and we can set the diff --git a/test/prism/fixtures/leading_logical.txt b/test/prism/fixtures/leading_logical.txt new file mode 100644 index 00000000000..b029c367b10 --- /dev/null +++ b/test/prism/fixtures/leading_logical.txt @@ -0,0 +1,15 @@ +1 +&& 2 +&& 3 + +1 +|| 2 +|| 3 + +1 +and 2 +and 3 + +1 +or 2 +or 3 diff --git a/test/prism/fixtures_test.rb b/test/prism/fixtures_test.rb index 7225b4ac66c..0d75b7282f4 100644 --- a/test/prism/fixtures_test.rb +++ b/test/prism/fixtures_test.rb @@ -14,6 +14,8 @@ class FixturesTest < TestCase # https://bugs.ruby-lang.org/issues/19539 except << "heredocs_leading_whitespace.txt" if RUBY_VERSION < "3.3.0" + except << "leading_logical.txt" if RUBY_VERSION < "3.5.0" + Fixture.each(except: except) do |fixture| define_method(fixture.test_name) { assert_valid_syntax(fixture.read) } end diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb index 7eac677ef70..03212ad94ca 100644 --- a/test/prism/lex_test.rb +++ b/test/prism/lex_test.rb @@ -30,6 +30,10 @@ class LexTest < TestCase except << "heredocs_leading_whitespace.txt" end + if RUBY_VERSION < "3.5.0" + except << "leading_logical.txt" + end + Fixture.each(except: except) do |fixture| define_method(fixture.test_name) { assert_lex(fixture) } end diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 8db47da3d35..2c85f88ada6 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -8,6 +8,9 @@ module Prism class RipperTest < TestCase # Skip these tests that Ripper is reporting the wrong results for. incorrect = [ + # Not yet supported. + "leading_logical.txt", + # Ripper incorrectly attributes the block to the keyword. "seattlerb/block_break.txt", "seattlerb/block_next.txt", diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb index a13daeeb849..5df4177c10c 100644 --- a/test/prism/ruby/ruby_parser_test.rb +++ b/test/prism/ruby/ruby_parser_test.rb @@ -48,6 +48,7 @@ class RubyParserTest < TestCase "alias.txt", "dos_endings.txt", "heredocs_with_ignored_newlines.txt", + "leading_logical.txt", "method_calls.txt", "methods.txt", "multi_write.txt", diff --git a/test/prism/snapshots/leading_logical.txt b/test/prism/snapshots/leading_logical.txt new file mode 100644 index 00000000000..c8ecaa40ba4 --- /dev/null +++ b/test/prism/snapshots/leading_logical.txt @@ -0,0 +1,83 @@ +@ ProgramNode (location: (1,0)-(15,4)) +├── flags: ∅ +├── locals: [] +└── statements: + @ StatementsNode (location: (1,0)-(15,4)) + ├── flags: ∅ + └── body: (length: 4) + ├── @ AndNode (location: (1,0)-(3,4)) + │ ├── flags: newline + │ ├── left: + │ │ @ AndNode (location: (1,0)-(2,4)) + │ │ ├── flags: ∅ + │ │ ├── left: + │ │ │ @ IntegerNode (location: (1,0)-(1,1)) + │ │ │ ├── flags: static_literal, decimal + │ │ │ └── value: 1 + │ │ ├── right: + │ │ │ @ IntegerNode (location: (2,3)-(2,4)) + │ │ │ ├── flags: static_literal, decimal + │ │ │ └── value: 2 + │ │ └── operator_loc: (2,0)-(2,2) = "&&" + │ ├── right: + │ │ @ IntegerNode (location: (3,3)-(3,4)) + │ │ ├── flags: static_literal, decimal + │ │ └── value: 3 + │ └── operator_loc: (3,0)-(3,2) = "&&" + ├── @ OrNode (location: (5,0)-(7,4)) + │ ├── flags: newline + │ ├── left: + │ │ @ OrNode (location: (5,0)-(6,4)) + │ │ ├── flags: ∅ + │ │ ├── left: + │ │ │ @ IntegerNode (location: (5,0)-(5,1)) + │ │ │ ├── flags: static_literal, decimal + │ │ │ └── value: 1 + │ │ ├── right: + │ │ │ @ IntegerNode (location: (6,3)-(6,4)) + │ │ │ ├── flags: static_literal, decimal + │ │ │ └── value: 2 + │ │ └── operator_loc: (6,0)-(6,2) = "||" + │ ├── right: + │ │ @ IntegerNode (location: (7,3)-(7,4)) + │ │ ├── flags: static_literal, decimal + │ │ └── value: 3 + │ └── operator_loc: (7,0)-(7,2) = "||" + ├── @ AndNode (location: (9,0)-(11,5)) + │ ├── flags: newline + │ ├── left: + │ │ @ AndNode (location: (9,0)-(10,5)) + │ │ ├── flags: ∅ + │ │ ├── left: + │ │ │ @ IntegerNode (location: (9,0)-(9,1)) + │ │ │ ├── flags: static_literal, decimal + │ │ │ └── value: 1 + │ │ ├── right: + │ │ │ @ IntegerNode (location: (10,4)-(10,5)) + │ │ │ ├── flags: static_literal, decimal + │ │ │ └── value: 2 + │ │ └── operator_loc: (10,0)-(10,3) = "and" + │ ├── right: + │ │ @ IntegerNode (location: (11,4)-(11,5)) + │ │ ├── flags: static_literal, decimal + │ │ └── value: 3 + │ └── operator_loc: (11,0)-(11,3) = "and" + └── @ OrNode (location: (13,0)-(15,4)) + ├── flags: newline + ├── left: + │ @ OrNode (location: (13,0)-(14,4)) + │ ├── flags: ∅ + │ ├── left: + │ │ @ IntegerNode (location: (13,0)-(13,1)) + │ │ ├── flags: static_literal, decimal + │ │ └── value: 1 + │ ├── right: + │ │ @ IntegerNode (location: (14,3)-(14,4)) + │ │ ├── flags: static_literal, decimal + │ │ └── value: 2 + │ └── operator_loc: (14,0)-(14,2) = "or" + ├── right: + │ @ IntegerNode (location: (15,3)-(15,4)) + │ ├── flags: static_literal, decimal + │ └── value: 3 + └── operator_loc: (15,0)-(15,2) = "or"