From 47b826dde20d96ecce2fad50cad53e7dda6c41d8 Mon Sep 17 00:00:00 2001 From: llby Date: Mon, 2 May 2016 10:14:47 +0900 Subject: [PATCH 1/2] [#772] Support multiline string in " quoted string --- lib/fluent/config/basic_parser.rb | 5 +++++ lib/fluent/config/literal_parser.rb | 6 +++++ lib/fluent/config/v1_parser.rb | 22 ++++++++++++++++++ test/config/test_config_parser.rb | 35 +++++++++++++++++++++++++++++ test/config/test_literal_parser.rb | 3 ++- 5 files changed, 70 insertions(+), 1 deletion(-) diff --git a/lib/fluent/config/basic_parser.rb b/lib/fluent/config/basic_parser.rb index 2a92f6b9b7..01b04e5228 100644 --- a/lib/fluent/config/basic_parser.rb +++ b/lib/fluent/config/basic_parser.rb @@ -28,6 +28,7 @@ def initialize(strscan) SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/ ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/ SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/ + LINE_END_WITHOUT_SPACING_AND_COMMENT = /(?:\z|[\r\n])/ module ClassMethods def symbol(string) @@ -71,6 +72,10 @@ def prev_match @ss[0] end + def check(pattern) + @ss.check(pattern) + end + def line_end skip(LINE_END) end diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index 403b7a12ee..ae9e9ab0ee 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -81,6 +81,12 @@ def scan_double_quoted_string while true if skip(/\"/) return string.join + elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/) + if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/) + string << s + end + skip(/[^"]/) + return string.join elsif s = scan(/\\./) string << eval_escape_char(s[1,1]) elsif skip(/\#\{/) diff --git a/lib/fluent/config/v1_parser.rb b/lib/fluent/config/v1_parser.rb index 9f6a73aed9..fb0cecddd2 100644 --- a/lib/fluent/config/v1_parser.rb +++ b/lib/fluent/config/v1_parser.rb @@ -56,7 +56,26 @@ def parse! RESERVED_PARAMS = %W(@type @id @label @log_level) def parse_element(root_element, elem_name, attrs = {}, elems = []) + multiline_key = "" while true + if !multiline_key.empty? + k = multiline_key + v = scan_nonquoted_string(/("|\\#{LINE_END_WITHOUT_SPACING_AND_COMMENT})/) + if s = check(LINE_END_WITHOUT_SPACING_AND_COMMENT) + v << s + elsif check(/\\/) + skip(/\\/) + elsif check(/"/) + multiline_key = "" + skip(/"/) + end + attrs[k] << v + unless line_end + parse_error! "expected end of line" + end + next + end + spacing if eof? if root_element @@ -127,6 +146,9 @@ def parse_element(root_element, elem_name, attrs = {}, elems = []) end end + if check(/\"[^"]*#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/) + multiline_key = k + end v = parse_literal unless line_end parse_error! "expected end of line" diff --git a/test/config/test_config_parser.rb b/test/config/test_config_parser.rb index 7012f16b08..11f48994cd 100644 --- a/test/config/test_config_parser.rb +++ b/test/config/test_config_parser.rb @@ -5,6 +5,7 @@ require "fluent/config/basic_parser" require "fluent/config/literal_parser" require "fluent/config/v1_parser" +require 'fluent/config/parser' module Fluent::Config module V1TestHelper @@ -129,6 +130,7 @@ def parse_text(text) test "requires escaping double quote" do assert_text_parsed_as(e('ROOT', '', {"k1" => '"'}), ' k1 "\\""') assert_parse_error(' k1 """') + assert_parse_error(' k1 ""\'') end test "removes backslash in front of a normal character" do @@ -138,6 +140,39 @@ def parse_text(text) test "accepts escape characters" do assert_text_parsed_as(e('ROOT', '', {"k1" => "\n"}), ' k1 "\\n"') end + + test "support multiline string" do + assert_text_parsed_as(e('ROOT', '', + {"k1" => %[line1 + line2] + }), + %[k1 "line1 + line2"] + ) + assert_text_parsed_as(e('ROOT', '', + {"k1" => %[line1 line2] + }), + %[k1 "line1\\ + line2"] + ) + assert_text_parsed_as(e('ROOT', '', + {"k1" => %[line1 + line2 + line3] + }), + %[k1 "line1 + line2 + line3"] + ) + assert_text_parsed_as(e('ROOT', '', + {"k1" => %[line1 + line2 line3] + }), + %[k1 "line1 + line2\\ + line3"] + ) + end end sub_test_case 'single quoted string' do diff --git a/test/config/test_literal_parser.rb b/test/config/test_literal_parser.rb index 75218b77aa..c6c5b3a87b 100644 --- a/test/config/test_literal_parser.rb +++ b/test/config/test_literal_parser.rb @@ -108,7 +108,8 @@ def test_falseX test('"\\z"') { assert_parse_error('"\\z"') } # unknown escaped character test('"\\0"') { assert_parse_error('"\\0"') } # unknown escaped character test('"\\1"') { assert_parse_error('"\\1"') } # unknown escaped character - test('"t') { assert_parse_error('"t') } # non-terminated quoted character + test("\"t\n") { assert_text_parsed_as("t\n", "\"t\n" ) } # multiline string + test("\"t\\\n") { assert_text_parsed_as('t', "\"t\\\n" ) } # multiline string test('t"') { assert_text_parsed_as('t"', 't"') } test('"."') { assert_text_parsed_as('.', '"."') } test('"*"') { assert_text_parsed_as('*', '"*"') } From 9fd7e8dfee21db1a959ecd26321c5bf76b0cf143 Mon Sep 17 00:00:00 2001 From: llby Date: Mon, 2 May 2016 21:00:35 +0900 Subject: [PATCH 2/2] [#772] Support multiline string in " quoted string another simple way --- lib/fluent/config/literal_parser.rb | 3 +-- lib/fluent/config/v1_parser.rb | 22 ---------------------- test/config/test_literal_parser.rb | 5 +++-- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index ae9e9ab0ee..67db779461 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -85,8 +85,7 @@ def scan_double_quoted_string if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/) string << s end - skip(/[^"]/) - return string.join + skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/) elsif s = scan(/\\./) string << eval_escape_char(s[1,1]) elsif skip(/\#\{/) diff --git a/lib/fluent/config/v1_parser.rb b/lib/fluent/config/v1_parser.rb index fb0cecddd2..9f6a73aed9 100644 --- a/lib/fluent/config/v1_parser.rb +++ b/lib/fluent/config/v1_parser.rb @@ -56,26 +56,7 @@ def parse! RESERVED_PARAMS = %W(@type @id @label @log_level) def parse_element(root_element, elem_name, attrs = {}, elems = []) - multiline_key = "" while true - if !multiline_key.empty? - k = multiline_key - v = scan_nonquoted_string(/("|\\#{LINE_END_WITHOUT_SPACING_AND_COMMENT})/) - if s = check(LINE_END_WITHOUT_SPACING_AND_COMMENT) - v << s - elsif check(/\\/) - skip(/\\/) - elsif check(/"/) - multiline_key = "" - skip(/"/) - end - attrs[k] << v - unless line_end - parse_error! "expected end of line" - end - next - end - spacing if eof? if root_element @@ -146,9 +127,6 @@ def parse_element(root_element, elem_name, attrs = {}, elems = []) end end - if check(/\"[^"]*#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/) - multiline_key = k - end v = parse_literal unless line_end parse_error! "expected end of line" diff --git a/test/config/test_literal_parser.rb b/test/config/test_literal_parser.rb index c6c5b3a87b..3e495fe332 100644 --- a/test/config/test_literal_parser.rb +++ b/test/config/test_literal_parser.rb @@ -108,8 +108,9 @@ def test_falseX test('"\\z"') { assert_parse_error('"\\z"') } # unknown escaped character test('"\\0"') { assert_parse_error('"\\0"') } # unknown escaped character test('"\\1"') { assert_parse_error('"\\1"') } # unknown escaped character - test("\"t\n") { assert_text_parsed_as("t\n", "\"t\n" ) } # multiline string - test("\"t\\\n") { assert_text_parsed_as('t', "\"t\\\n" ) } # multiline string + test('"t') { assert_parse_error('"t') } # non-terminated quoted character + test("\"t\nt\"") { assert_text_parsed_as("t\nt", "\"t\nt\"" ) } # multiline string + test("\"t\\\nt\"") { assert_text_parsed_as("tt", "\"t\\\nt\"" ) } # multiline string test('t"') { assert_text_parsed_as('t"', 't"') } test('"."') { assert_text_parsed_as('.', '"."') } test('"*"') { assert_text_parsed_as('*', '"*"') }