Add comments to the public API. Fix a few lexer/parsing issues

src/Tomlyn.Tests/AssertHelper.cs

@@ -0,0 +1,23 @@
+// Copyright (c) 2019 - Alexandre Mutel. All rights reserved.
+// Licensed under the BSD-Clause 2 license. 
+// See license.txt file in the project root for full license information.
+
+using NUnit.Framework;
+
+namespace Tomlyn.Tests
+{
+    public static class AssertHelper
+    {
+        public static void AreEqualNormalizeNewLine(string expected, string actual, string message = null)
+        {
+            expected = NormalizeEndOfLine(expected);
+            actual = NormalizeEndOfLine(actual);
+            Assert.AreEqual(expected, actual, message);
+        }
+
+        public static string NormalizeEndOfLine(string text)
+        {
+            return text.Replace("\r\n", "\n");
+        }
+    }
+}

src/Tomlyn.Tests/ModelTests.cs

@@ -250,20 +250,12 @@ private static void AssertJson(string input, string expectedJson)
             var serializer = JsonSerializer.Create(new JsonSerializerSettings() { Formatting = Formatting.Indented });
             var writer = new StringWriter();
             serializer.Serialize(writer, model);
-            var jsonResult = NormalizeEndOfLine(writer.ToString());
-            expectedJson = NormalizeEndOfLine(expectedJson);
+            var jsonResult = writer.ToString();
 
             StandardTests.DisplayHeader("json");
             Console.WriteLine(jsonResult);
 
-            Assert.AreEqual(expectedJson, jsonResult);
+            AssertHelper.AreEqualNormalizeNewLine(expectedJson, jsonResult);
         }
-
-        private static string NormalizeEndOfLine(string text)
-        {
-            return text.Replace("\r\n", "\n");
-        }
-
-
     }
 }

src/Tomlyn.Tests/StandardTests.cs

@@ -119,14 +119,14 @@ public static IEnumerable ListTomlFiles(string type)
             {
                 var functionName = Path.GetFileName(file);
 
-                var input = File.ReadAllText(file);
+                var input = File.ReadAllText(file, Encoding.UTF8);
 
                 string json = null;
                 if (type == "valid")
                 {
                     var jsonFile = Path.ChangeExtension(file, "json");
                     Assert.True(File.Exists(jsonFile), $"The json file `{jsonFile}` does not exist");
-                    json = File.ReadAllText(jsonFile);
+                    json = File.ReadAllText(jsonFile, Encoding.UTF8);
                 }
                 tests.Add(new TestCaseData(functionName, input, json));
             }

src/Tomlyn.Tests/SyntaxTests.cs

@@ -0,0 +1,56 @@
+// Copyright (c) 2019 - Alexandre Mutel. All rights reserved.
+// Licensed under the BSD-Clause 2 license. 
+// See license.txt file in the project root for full license information.
+
+using NUnit.Framework;
+using Tomlyn.Syntax;
+
+namespace Tomlyn.Tests
+{
+    public class SyntaxTests
+    {
+        [Test]
+        public void TestDocument()
+        {
+            var doc = new DocumentSyntax()
+            {
+                Tables =
+                {
+                    new TableSyntax("test")
+                    {
+                        Items =
+                        {
+                            {"a", 1},
+                            {"b", true },
+                            {"c", "Check"},
+                            {"d", "ToEscape\nWithAnotherChar\t" },
+                            {"e", 12.5 },
+                            {"f", new int[] {1,2,3,4} },
+                            {"g", new string[] {"0", "1", "2"} },
+                            {"key with space", 2}
+                        }
+                    }
+                }
+            };
+
+            var docStr = doc.ToString();
+
+            var expected = @"[test]
+a = 1
+b = true
+c = ""Check""
+d = ""ToEscape\nWithAnotherChar\t""
+e = 12.5
+f = [1, 2, 3, 4]
+g = [""0"", ""1"", ""2""]
+""key with space"" = 2
+";
+
+            AssertHelper.AreEqualNormalizeNewLine(expected, docStr);
+
+            // Reparse the result and compare it again
+            var newDoc = Toml.Parse(docStr);
+            AssertHelper.AreEqualNormalizeNewLine(expected, newDoc.ToString());
+        }
+    }
+}

src/Tomlyn.sln.DotSettings

@@ -1,7 +1,10 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:String x:Key="/Default/CodeStyle/FileHeader/FileHeaderText/@EntryValue">Copyright (c) $CURRENT_YEAR$ - Alexandre Mutel. All rights reserved.&#xD;
+	<s:String x:Key="/Default/CodeStyle/FileHeader/FileHeaderText/@EntryValue">Copyright (c) Alexandre Mutel. All rights reserved.&#xD;
 Licensed under the BSD-Clause 2 license. &#xD;
 See license.txt file in the project root for full license information.</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RFC/@EntryIndexedValue">RFC</s:String>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Alexandre/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Mutel/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Toml/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/UserDictionary/Words/=Tomlyn/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Tomlyn/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=trivias/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

src/Tomlyn/Model/ObjectKind.cs

@@ -3,6 +3,9 @@
 // See license.txt file in the project root for full license information.
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Kind of an TOML object.
+    /// </summary>
     public enum ObjectKind
     {
         Table,

src/Tomlyn/Model/SyntaxTransform.cs

@@ -7,6 +7,9 @@
 
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Internal class used to transform a <see cref="DocumentSyntax"/> into a <see cref="TomlTable"/>
+    /// </summary>
     internal class SyntaxTransform : SyntaxVisitor
     {
         private readonly TomlTable _rootTable;
@@ -42,12 +45,12 @@ public override void Visit(TableArraySyntax table)
         private TomlTable SetKeyValue(KeySyntax key, object value, SyntaxKind kind)
         {
             var currentTable = _currentTable;
-            var name = GetStringFromBasic(key.Base);
-            var items = key.DotKeyItems;
+            var name = GetStringFromBasic(key.Key);
+            var items = key.DotKeys;
             for (int i = 0; i < items.ChildrenCount; i++)
             {
                 currentTable = GetTable(currentTable, name, false);
-                name = GetStringFromBasic(items.GetChildren(i).Value);
+                name = GetStringFromBasic(items.GetChildren(i).Key);
             }
 
             var isTableArray = kind == SyntaxKind.TableArray;
@@ -93,9 +96,9 @@ private TomlTable GetTable(TomlTable table, string key, bool createTableArrayIte
             return newTable;
         }
 
-        private string GetStringFromBasic(BasicValueSyntax value)
+        private string GetStringFromBasic(BareKeyOrStringValueSyntax value)
         {
-            if (value is BasicKeySyntax basicKey)
+            if (value is BareKeySyntax basicKey)
             {
                 return basicKey.Key.Text;
             }

src/Tomlyn/Model/TomlArray.cs

@@ -8,7 +8,10 @@
 
 namespace Tomlyn.Model
 {
-    public class TomlArray : TomlObject, IList<object>
+    /// <summary>
+    /// Runtime representation of a TOML array
+    /// </summary>
+    public sealed class TomlArray : TomlObject, IList<object>
     {
         private readonly List<TomlObject> _items;
 

src/Tomlyn/Model/TomlBoolean.cs

@@ -3,6 +3,9 @@
 // See license.txt file in the project root for full license information.
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Runtime representation of a TOML bool
+    /// </summary>
     public sealed class TomlBoolean : TomlValue<bool>
     {
         public TomlBoolean(bool value) : base(ObjectKind.Boolean, value)

src/Tomlyn/Model/TomlDateTime.cs

@@ -7,6 +7,9 @@
 
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Runtime representation of a TOML datetime
+    /// </summary>
     public sealed class TomlDateTime : TomlValue<DateTime>
     {
         public TomlDateTime(ObjectKind kind, DateTime value) : base(CheckDateTimeKind(kind), value)

src/Tomlyn/Model/TomlFloat.cs

@@ -3,6 +3,9 @@
 // See license.txt file in the project root for full license information.
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Runtime representation of a TOML float
+    /// </summary>
     public sealed class TomlFloat : TomlValue<double>
     {
         public TomlFloat(double value) : base(ObjectKind.Float, value)

src/Tomlyn/Model/TomlInteger.cs

@@ -3,6 +3,9 @@
 // See license.txt file in the project root for full license information.
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Runtime representation of a TOML integer
+    /// </summary>
     public sealed class TomlInteger : TomlValue<long>
     {
         public TomlInteger(long value) : base(ObjectKind.Integer, value)

src/Tomlyn/Model/TomlObject.cs

@@ -1,24 +1,22 @@
 using System;
-using Tomlyn.Syntax;
 
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Base class for the runtime representation of a TOML object
+    /// </summary>
     public abstract class TomlObject
     {
-        protected TomlObject(ObjectKind kind)
+        internal TomlObject(ObjectKind kind)
         {
             Kind = kind;
         }
 
+        /// <summary>
+        /// The kind of the object
+        /// </summary>
         public ObjectKind Kind { get; }
 
-        public SyntaxNode Node { get; internal set; }
-
-        internal static bool IsContainer(TomlObject tomlObj)
-        {
-            return tomlObj.Kind == ObjectKind.Array || tomlObj.Kind == ObjectKind.Table || tomlObj.Kind == ObjectKind.TableArray;
-        }
-
         internal static object ToObject(TomlObject tomlObj)
         {
             return tomlObj is TomlValue value ? value.ValueAsObject : tomlObj;

src/Tomlyn/Model/TomlString.cs

@@ -3,6 +3,9 @@
 // See license.txt file in the project root for full license information.
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Runtime representation of a TOML string
+    /// </summary>
     public sealed class TomlString : TomlValue<string>
     {
         public TomlString(string value) : base(ObjectKind.String, value)

src/Tomlyn/Model/TomlTable.cs

@@ -9,12 +9,21 @@
 
 namespace Tomlyn.Model
 {
-    public class TomlTable : TomlObject, IDictionary<string, object>
+    /// <summary>
+    /// Runtime representation of a TOML table
+    /// </summary>
+    /// <remarks>
+    /// This object keep the order of the inserted key=values
+    /// </remarks>
+    public sealed class TomlTable : TomlObject, IDictionary<string, object>
     {
         // TODO: optimize the internal by avoiding two structures
         private readonly List<KeyValuePair<string, TomlObject>> _order;
         private readonly Dictionary<string, TomlObject> _map;
 
+        /// <summary>
+        /// Creates an instance of a <see cref="TomlTable"/>
+        /// </summary>
         public TomlTable() : base(ObjectKind.Table)
         {
             _order = new List<KeyValuePair<string, TomlObject>>();

src/Tomlyn/Model/TomlTableArray.cs

@@ -8,7 +8,10 @@
 
 namespace Tomlyn.Model
 {
-    public class TomlTableArray : TomlObject, IList<TomlTable>
+    /// <summary>
+    /// Runtime representation of a TOML table array
+    /// </summary>
+    public sealed class TomlTableArray : TomlObject, IList<TomlTable>
     {
         private readonly List<TomlTable> _items;
 

src/Tomlyn/Model/TomlValue.cs

@@ -6,6 +6,9 @@
 
 namespace Tomlyn.Model
 {
+    /// <summary>
+    /// Base class of a TOML value (bool, string, integer, float, datetime)
+    /// </summary>
     public abstract class TomlValue : TomlObject
     {
         internal TomlValue(ObjectKind kind) : base(kind)
@@ -15,6 +18,9 @@ internal TomlValue(ObjectKind kind) : base(kind)
         public abstract object ValueAsObject { get; }
     }
 
+    /// <summary>
+    /// Base class of a TOML value (bool, string, integer, float, datetime)
+    /// </summary>
     public abstract class TomlValue<T> : TomlValue, IEquatable<TomlValue<T>> where T : IEquatable<T>
     {
         private T _value;

src/Tomlyn/Parsing/Lexer.cs

@@ -155,7 +155,7 @@ private void NextTokenForKey()
                     ReadStringLiteral(start, false);
                     break;
                 case Eof:
-                    _token = SyntaxTokenValue.Eof;
+                    _token = new SyntaxTokenValue(TokenKind.Eof, _position, _position);
                     break;
                 default:
                     // Eat any whitespace
@@ -229,7 +229,7 @@ private void NextTokenForValue()
                     ReadStringLiteral(start, true);
                     break;
                 case Eof:
-                    _token = SyntaxTokenValue.Eof;
+                    _token = new SyntaxTokenValue(TokenKind.Eof, _position, _position);
                     break;
                 default:
                     // Eat any whitespace
@@ -576,7 +576,20 @@ private void ReadNumberOrDate(char32? signPrefix = null, TextPosition? signPrefi
                 }
                 else
                 {
-                    AddError($"Unable to parse the date time/offset `{dateTimeAsString}`", start, end);
+                    // Try to recover the date using the standard C# (not necessarily RFC3339)
+                    if (DateTime.TryParse(dateTimeAsString, CultureInfo.InvariantCulture, DateTimeStyles.AllowInnerWhite, out datetime))
+                    {
+                        _token = new SyntaxTokenValue(TokenKind.LocalDateTime, start, end, datetime);
+
+                        // But we produce an error anyway
+                        AddError($"Invalid format of date time/offset `{dateTimeAsString}` not following RFC3339", start, end);
+                    }
+                    else
+                    {
+                        _token = new SyntaxTokenValue(TokenKind.LocalDateTime, start, end, new DateTime());
+                        // But we produce an error anyway
+                        AddError($"Unable to parse the date time/offset `{dateTimeAsString}`", start, end);
+                    }
                 }
 
                 return;
@@ -685,7 +698,7 @@ private void ReadDigits(ref TextPosition end, bool isPreviousDigit)
 
             if (!isPreviousDigit)
             {
-                AddError("An underscore `_` must not followed a digit", _position, _position);
+                AddError("Missing a digit after a trailing underscore `_`", _position, _position);
             }
         }
 
@@ -1074,7 +1087,8 @@ private char32 NextCharFromReader()
 
         private void CheckCharacter(char32 c)
         {
-            if (!CharHelper.IsValidUnicodeScalarValue(c))
+            // The character 0xFFFD is the replacement character and we assume that something went wrong when reading the input
+            if (!CharHelper.IsValidUnicodeScalarValue(c) || c == 0xFFFD)
             {
                 AddError($"The character `{c}` is an invalid UTF8 character", _current.Position, _current.Position);
             }