From 1691ab6319cfc6187bacd20c3c2488760ce7c6e7 Mon Sep 17 00:00:00 2001 From: Nguyen Duc Toan Date: Thu, 22 Dec 2022 00:37:55 +0700 Subject: [PATCH 1/3] Fix control character bug --- src/Tomlyn.Tests/ValidatorTests.cs | 15 +++++++++++++++ src/Tomlyn/Parsing/Lexer.cs | 2 +- src/Tomlyn/Text/CharHelper.cs | 5 +++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Tomlyn.Tests/ValidatorTests.cs b/src/Tomlyn.Tests/ValidatorTests.cs index bc68d63..c23fd87 100644 --- a/src/Tomlyn.Tests/ValidatorTests.cs +++ b/src/Tomlyn.Tests/ValidatorTests.cs @@ -89,5 +89,20 @@ public void TestTableArrayAndInvalidTable() Assert.True(doc.HasErrors, "The document should have errors"); } + [Test] + public void TestValidControlCharacters() + { + var input = ""; + input += "backspace='\b'\n"; + input += "tab='\t'\n"; + input += "form_feed='\f'\n"; + input += "quote='\"'\n"; + input += "backslash='\\'\n"; + input += "heart='\u2665'\n"; + input += "some_unicode='\U0002B695'\n"; + var doc = Toml.Parse(input); + Assert.False(doc.HasErrors, "The document should not have errors on valid control characters"); + } + } } \ No newline at end of file diff --git a/src/Tomlyn/Parsing/Lexer.cs b/src/Tomlyn/Parsing/Lexer.cs index 5497851..be02c4f 100644 --- a/src/Tomlyn/Parsing/Lexer.cs +++ b/src/Tomlyn/Parsing/Lexer.cs @@ -1016,7 +1016,7 @@ private void ReadStringLiteral(TextPosition start, bool allowMultiline) { AddError("Invalid newline in a string", _position, _position); } - else if (CharHelper.IsControlCharacter(_c) && (!isMultiLine || !CharHelper.IsNewLine(_c))) + else if (CharHelper.IsControlCharacter(_c) && !CharHelper.IsNonNewlineValidControlCharacter(_c) && (!isMultiLine || !CharHelper.IsNewLine(_c))) { AddError($"Invalid control character found {((char)_c).ToPrintableString()}", start, start); } diff --git a/src/Tomlyn/Text/CharHelper.cs b/src/Tomlyn/Text/CharHelper.cs index e2405cd..4cbf944 100644 --- a/src/Tomlyn/Text/CharHelper.cs +++ b/src/Tomlyn/Text/CharHelper.cs @@ -24,6 +24,11 @@ public static bool IsControlCharacter(char32 c) return c <= 0x1F || c == 0x7F; } + public static bool IsNonNewlineValidControlCharacter(char32 c) + { + return c == '\b' || c == '\t' || c == '\f'; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsKeyStart(char32 c) { From ee0df99ceb95da3fceec32462ff10560de220086 Mon Sep 17 00:00:00 2001 From: Nguyen Duc Toan Date: Thu, 22 Dec 2022 01:13:29 +0700 Subject: [PATCH 2/3] Allow tab character in normal and literal string. --- src/Tomlyn.Tests/ValidatorTests.cs | 13 ++++--------- src/Tomlyn/Parsing/Lexer.cs | 4 ++-- src/Tomlyn/Text/CharHelper.cs | 5 ----- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/Tomlyn.Tests/ValidatorTests.cs b/src/Tomlyn.Tests/ValidatorTests.cs index c23fd87..d0f1f8a 100644 --- a/src/Tomlyn.Tests/ValidatorTests.cs +++ b/src/Tomlyn.Tests/ValidatorTests.cs @@ -90,18 +90,13 @@ public void TestTableArrayAndInvalidTable() } [Test] - public void TestValidControlCharacters() + public void TestAllowTab() { var input = ""; - input += "backspace='\b'\n"; - input += "tab='\t'\n"; - input += "form_feed='\f'\n"; - input += "quote='\"'\n"; - input += "backslash='\\'\n"; - input += "heart='\u2665'\n"; - input += "some_unicode='\U0002B695'\n"; + input += "tab_normal=\"\t\"\n"; + input += "tab_literal='\t'\n"; var doc = Toml.Parse(input); - Assert.False(doc.HasErrors, "The document should not have errors on valid control characters"); + Assert.False(doc.HasErrors, "The document should not have errors on tab character"); } } diff --git a/src/Tomlyn/Parsing/Lexer.cs b/src/Tomlyn/Parsing/Lexer.cs index be02c4f..808aa7a 100644 --- a/src/Tomlyn/Parsing/Lexer.cs +++ b/src/Tomlyn/Parsing/Lexer.cs @@ -797,7 +797,7 @@ private void ReadString(TextPosition start, bool allowMultiline) { AddError("Invalid newline in a string", _position, _position); } - else if (CharHelper.IsControlCharacter(_c) && (!isMultiLine || !CharHelper.IsWhiteSpaceOrNewLine(_c))) + else if (CharHelper.IsControlCharacter(_c) && _c != '\t' && (!isMultiLine || !CharHelper.IsWhiteSpaceOrNewLine(_c))) { AddError($"Invalid control character found {((char)_c).ToPrintableString()}", start, start); } @@ -1016,7 +1016,7 @@ private void ReadStringLiteral(TextPosition start, bool allowMultiline) { AddError("Invalid newline in a string", _position, _position); } - else if (CharHelper.IsControlCharacter(_c) && !CharHelper.IsNonNewlineValidControlCharacter(_c) && (!isMultiLine || !CharHelper.IsNewLine(_c))) + else if (CharHelper.IsControlCharacter(_c) && _c != '\t' && (!isMultiLine || !CharHelper.IsNewLine(_c))) { AddError($"Invalid control character found {((char)_c).ToPrintableString()}", start, start); } diff --git a/src/Tomlyn/Text/CharHelper.cs b/src/Tomlyn/Text/CharHelper.cs index 4cbf944..e2405cd 100644 --- a/src/Tomlyn/Text/CharHelper.cs +++ b/src/Tomlyn/Text/CharHelper.cs @@ -24,11 +24,6 @@ public static bool IsControlCharacter(char32 c) return c <= 0x1F || c == 0x7F; } - public static bool IsNonNewlineValidControlCharacter(char32 c) - { - return c == '\b' || c == '\t' || c == '\f'; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsKeyStart(char32 c) { From 9b7acdd9a7815c03bfb6edcdb8bddbc90d822e44 Mon Sep 17 00:00:00 2001 From: Nguyen Duc Toan Date: Thu, 22 Dec 2022 01:30:50 +0700 Subject: [PATCH 3/3] Allow tab character in string literal without escaping --- src/Tomlyn.Tests/ValidatorTests.cs | 7 ++++--- src/Tomlyn/Parsing/Lexer.cs | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Tomlyn.Tests/ValidatorTests.cs b/src/Tomlyn.Tests/ValidatorTests.cs index d0f1f8a..ded9921 100644 --- a/src/Tomlyn.Tests/ValidatorTests.cs +++ b/src/Tomlyn.Tests/ValidatorTests.cs @@ -93,10 +93,11 @@ public void TestTableArrayAndInvalidTable() public void TestAllowTab() { var input = ""; - input += "tab_normal=\"\t\"\n"; - input += "tab_literal='\t'\n"; + input += "tab_multiline_basic=\"\"\"\t\"\"\"\n"; + input += "tab_singleline_literal='\t'\n"; + input += "tab_multiline_literal='''\t'''\n"; var doc = Toml.Parse(input); - Assert.False(doc.HasErrors, "The document should not have errors on tab character"); + Assert.False(doc.HasErrors, "The document should not have errors"); } } diff --git a/src/Tomlyn/Parsing/Lexer.cs b/src/Tomlyn/Parsing/Lexer.cs index 808aa7a..abc29d6 100644 --- a/src/Tomlyn/Parsing/Lexer.cs +++ b/src/Tomlyn/Parsing/Lexer.cs @@ -797,7 +797,7 @@ private void ReadString(TextPosition start, bool allowMultiline) { AddError("Invalid newline in a string", _position, _position); } - else if (CharHelper.IsControlCharacter(_c) && _c != '\t' && (!isMultiLine || !CharHelper.IsWhiteSpaceOrNewLine(_c))) + else if (CharHelper.IsControlCharacter(_c) && (!isMultiLine || !CharHelper.IsWhiteSpaceOrNewLine(_c))) { AddError($"Invalid control character found {((char)_c).ToPrintableString()}", start, start); }