Merge pull request #14 from Pizzaandy/source-text-abstraction

refactor: use SourceText instead of string, add tests

Gobo.Cli/Program.cs

@@ -1,6 +1,7 @@
 using DocoptNet;
 using Gobo;
 using Gobo.Cli;
+using Gobo.Text;
 using System.Diagnostics;
 
 const string usage =
@@ -152,7 +153,7 @@ static async Task CheckFile(
     IDictionary<string, ArgValue> arguments
 )
 {
-    var input = await File.ReadAllTextAsync(filePath);
+    var input = SourceText.From(await File.ReadAllTextAsync(filePath));
     bool success;
 
     try

Gobo.Tests/Gml/FormattingTests/StringsAndTemplates.expected

@@ -1,3 +1,5 @@
 str = "\\";
 
 var escaped_template = $"\{{a + b}\}";
+
+var more_escapes = $"\"{link_list[link_index]}\"";

Gobo.Tests/Gml/FormattingTests/StringsAndTemplates.test

@@ -1,3 +1,5 @@
 str = "\\"
 
-var escaped_template = $"\{{a+b}\}"
+var escaped_template = $"\{{a+b}\}"
+
+var more_escapes = $"\"{link_list[link_index]}\""

Gobo.Tests/SourceTextTests.cs

@@ -0,0 +1,44 @@
+using Gobo.Text;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Gobo.Tests;
+
+/// <summary>
+/// These tests ensure that implementations of SourceText work as expected on large files
+/// </summary>
+public class SourceTextTests
+{
+    private readonly ITestOutputHelper output;
+
+    public const string TestFileExtension = ".test";
+
+    public SourceTextTests(ITestOutputHelper output)
+    {
+        this.output = output;
+    }
+
+    [Theory]
+    [ClassData(typeof(SampleFileProvider))]
+    public async Task EnsureContentEquals(TestFile test)
+    {
+        var filePath = test.FilePath;
+
+        var input = await File.ReadAllTextAsync(filePath);
+        var wrongInput = "obviously wrong input";
+
+        var sourceTextA = new StringText(input);
+        var sourceTextB = new StringText(input);
+        var sourceTextC = new StringText(wrongInput);
+
+        if (!sourceTextA.ContentEquals(sourceTextB))
+        {
+            throw new XunitException($"Comparison failed");
+        }
+
+        if (sourceTextA.ContentEquals(sourceTextC))
+        {
+            throw new XunitException($"Something has gone horribly wrong");
+        }
+    }
+}

Gobo/GmlFormatter.cs

@@ -1,6 +1,7 @@
 using Gobo.Parser;
 using Gobo.Printer.DocPrinter;
 using Gobo.SyntaxNodes;
+using Gobo.Text;
 using System.Diagnostics;
 
 namespace Gobo;
@@ -48,7 +49,12 @@ public override string ToString()
 
 public static partial class GmlFormatter
 {
-    public static FormatResult Format(string code, FormatOptions options)
+    public static FormatResult Format(string text, FormatOptions options)
+    {
+        return Format(SourceText.From(text), options);
+    }
+
+    public static FormatResult Format(SourceText code, FormatOptions options)
     {
         long parseStart = 0;
         long parseStop = 0;
@@ -134,7 +140,7 @@ public static FormatResult Format(string code, FormatOptions options)
 
             try
             {
-                updatedParseResult = new GmlParser(output).Parse();
+                updatedParseResult = new GmlParser(SourceText.From(output)).Parse();
             }
             catch (GmlSyntaxErrorException ex)
             {
@@ -176,10 +182,10 @@ out var difference
         }
     }
 
-    public static bool Check(string code, FormatOptions options)
+    public static bool Check(SourceText code, FormatOptions options)
     {
         var result = Format(code, options);
-        return result.Output == code;
+        return SourceText.From(result.Output).ContentEquals(code);
     }
 
     public static async Task FormatFileAsync(string filePath, FormatOptions options)
@@ -189,7 +195,7 @@ public static async Task FormatFileAsync(string filePath, FormatOptions options)
 
         try
         {
-            var result = Format(input, options);
+            var result = Format(SourceText.From(input), options);
             formatted = result.Output;
         }
         catch (Exception)

Gobo/Gobo.csproj

@@ -7,6 +7,7 @@
     <IsAotCompatible>true</IsAotCompatible>
     <JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
     <AssemblyName>gobolib</AssemblyName>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
 </Project>

Gobo/Parser/CommentMapper.cs

@@ -1,14 +1,15 @@
 using Gobo.SyntaxNodes;
+using Gobo.Text;
 
 namespace Gobo.Parser;
 
 internal class CommentMapper
 {
-    public string SourceText { get; set; }
+    public SourceText SourceText { get; set; }
 
     public List<CommentGroup> CommentGroups { get; set; } = new();
 
-    public CommentMapper(string sourceText, List<Token[]> triviaGroups)
+    public CommentMapper(SourceText sourceText, List<Token[]> triviaGroups)
     {
         SourceText = sourceText;
         foreach (var triviaGroup in triviaGroups)

Gobo/Parser/GmlLexer.cs

@@ -1,4 +1,5 @@
-using System.Runtime.CompilerServices;
+using Gobo.Text;
+using System.Runtime.CompilerServices;
 
 namespace Gobo.Parser;
 
@@ -14,19 +15,17 @@ internal enum LexerMode
     public bool HitEof { get; private set; } = false;
     public LexerMode Mode { get; set; } = LexerMode.Default;
 
-    private readonly string text;
+    private readonly SourceText sourceText;
     private int lineNumber;
     private int columnNumber;
     private int startIndex;
     private int index;
     private int character;
-    private string CurrentToken => text[startIndex..index];
+    private string CurrentToken => sourceText.ReadSpan(startIndex, index);
 
-    private static readonly char[] whitespaces = { '\u000B', '\u000C', '\u0020', '\u00A0', '\t' };
-
-    public GmlLexer(string text)
+    public GmlLexer(SourceText source)
     {
-        this.text = text;
+        sourceText = source;
         index = 0;
         lineNumber = 1;
     }
@@ -82,7 +81,7 @@ public Token NextToken()
             case '\t':
                 while (true)
                 {
-                    if (!MatchAny(whitespaces))
+                    if (!MatchAnyWhitespace())
                     {
                         break;
                     }
@@ -594,40 +593,45 @@ private bool Match(int expected)
         return false;
     }
 
-    private bool MatchAny(char[] expected)
+    private bool MatchAnyWhitespace()
     {
         var next = Peek();
 
-        if (Array.Exists(expected, c => next == c))
+        bool matched = next switch
+        {
+            '\u000B' or '\u000C' or '\u0020' or '\u00A0' or '\t' => true,
+            _ => false
+        };
+
+        if (matched)
         {
             Advance();
-            return true;
         }
 
-        return false;
+        return matched;
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     private int Peek(int amount = 1)
     {
         var targetIndex = index + amount - 1;
-        if (targetIndex >= text.Length)
+        if (targetIndex >= sourceText.Length)
         {
             return -1;
         }
-        return text[targetIndex];
+        return sourceText[targetIndex];
     }
 
     private void Advance()
     {
-        if (index >= text.Length)
+        if (index >= sourceText.Length)
         {
             HitEof = true;
             character = -1;
             return;
         }
 
-        character = text[index];
+        character = sourceText[index];
         index++;
 
         switch (character)

Gobo/Parser/GmlParser.cs

@@ -1,6 +1,7 @@
 using Gobo.SyntaxNodes;
 using Gobo.SyntaxNodes.Gml;
 using Gobo.SyntaxNodes.Gml.Literals;
+using Gobo.Text;
 
 namespace Gobo.Parser;
 
@@ -112,9 +113,9 @@ internal class GmlParser
 
     private delegate bool BinaryExpressionRule(out GmlSyntaxNode node);
 
-    public GmlParser(string code)
+    public GmlParser(SourceText sourceText)
     {
-        lexer = new GmlLexer(code);
+        lexer = new GmlLexer(sourceText);
         token = lexer.NextToken();
         ProcessToken(token);
     }

Gobo/PrintContext.cs

@@ -1,11 +1,13 @@
-namespace Gobo;
+using Gobo.Text;
+
+namespace Gobo;
 
 internal class PrintContext
 {
     public FormatOptions Options { get; init; }
-    public string SourceText { get; init; }
+    public SourceText SourceText { get; init; }
 
-    public PrintContext(FormatOptions options, string sourceText)
+    public PrintContext(FormatOptions options, SourceText sourceText)
     {
         Options = options;
         SourceText = sourceText;