Merge pull request #23 from NeilMacMullen/serialized-result

Add capability to serialize KustoQueryResults

Directory.Packages.Props

@@ -7,6 +7,8 @@
  <PackageVersion Include="CommandLineParser" Version="2.9.1" />
  <PackageVersion Include="Fastenshtein" Version="1.0.0.9" />
  <PackageVersion Include="JPoke" Version="1.0.0" />
+ <PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly" Version="8.0.5" />
+ <PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="8.0.5" />
  <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" />
  <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" />
  <PackageVersion Include="Microsoft.PowerShell.SDK" Version="7.4.2" />
@@ -31,7 +33,8 @@
  <PackageVersion Include="System.Text.Json" Version="8.0.3" />
  <PackageVersion Include="CsvHelper" Version="32.0.3" />
  <PackageVersion Include="geohash-dotnet" Version="2.1.0" />
- <PackageVersion Include="FluentAssertions" Version="7.0.0-alpha.3" />
+ <PackageVersion Include="FluentAssertions" Version="6.12.0" />
  <PackageVersion Include="BenchmarkDotNet" Version="0.13.12" />
+ <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.4.0" />
  </ItemGroup>
 </Project>

KustoLoco.sln

@@ -70,7 +70,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "processes", "samples\proces
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SettingsTests", "test\SettingsTests\SettingsTests.csproj", "{1EFE0E94-DB11-4BFA-96CB-28CB54DB9B71}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProcessesWpf", "samples\ProcessesWpf\ProcessesWpf.csproj", "{71B23959-1E4C-4C67-B6CD-554F697C58DC}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProcessesWpf", "samples\ProcessesWpf\ProcessesWpf.csproj", "{71B23959-1E4C-4C67-B6CD-554F697C58DC}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SerializationTests", "test\SerializationTests\SerializationTests.csproj", "{71831E12-10C8-44F1-B1D7-9ED5DBE5AC09}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Blazor-Static", "samples\Blazor-Static\Blazor-Static.csproj", "{A92A0C23-698C-4F48-96D4-B6113564A85B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebApp", "samples\WebApp\WebApp.csproj", "{A953BE43-446B-4508-B144-A79AEAC6F7E1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebAppClient", "samples\WebAppClient\WebAppClient.csproj", "{32E26ABF-C631-405E-8EFB-1AD08A30CF5E}"
 EndProject
 Global
  GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -158,6 +166,22 @@ Global
  {71B23959-1E4C-4C67-B6CD-554F697C58DC}.Debug|Any CPU.Build.0 = Debug|Any CPU
  {71B23959-1E4C-4C67-B6CD-554F697C58DC}.Release|Any CPU.ActiveCfg = Release|Any CPU
  {71B23959-1E4C-4C67-B6CD-554F697C58DC}.Release|Any CPU.Build.0 = Release|Any CPU
+ {71831E12-10C8-44F1-B1D7-9ED5DBE5AC09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {71831E12-10C8-44F1-B1D7-9ED5DBE5AC09}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {71831E12-10C8-44F1-B1D7-9ED5DBE5AC09}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {71831E12-10C8-44F1-B1D7-9ED5DBE5AC09}.Release|Any CPU.Build.0 = Release|Any CPU
+ {A92A0C23-698C-4F48-96D4-B6113564A85B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {A92A0C23-698C-4F48-96D4-B6113564A85B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {A92A0C23-698C-4F48-96D4-B6113564A85B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {A92A0C23-698C-4F48-96D4-B6113564A85B}.Release|Any CPU.Build.0 = Release|Any CPU
+ {A953BE43-446B-4508-B144-A79AEAC6F7E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {A953BE43-446B-4508-B144-A79AEAC6F7E1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {A953BE43-446B-4508-B144-A79AEAC6F7E1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {A953BE43-446B-4508-B144-A79AEAC6F7E1}.Release|Any CPU.Build.0 = Release|Any CPU
+ {32E26ABF-C631-405E-8EFB-1AD08A30CF5E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {32E26ABF-C631-405E-8EFB-1AD08A30CF5E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {32E26ABF-C631-405E-8EFB-1AD08A30CF5E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {32E26ABF-C631-405E-8EFB-1AD08A30CF5E}.Release|Any CPU.Build.0 = Release|Any CPU
  EndGlobalSection
  GlobalSection(SolutionProperties) = preSolution
  HideSolutionNode = FALSE
@@ -184,6 +208,10 @@ Global
  {8F48A377-57EC-4A30-88F5-7E3261B23DB6} = {48CD2752-9462-473A-8699-7AD8B146742F}
  {1EFE0E94-DB11-4BFA-96CB-28CB54DB9B71} = {932C804C-F6B7-4361-8360-E5238938E464}
  {71B23959-1E4C-4C67-B6CD-554F697C58DC} = {48CD2752-9462-473A-8699-7AD8B146742F}
+ {71831E12-10C8-44F1-B1D7-9ED5DBE5AC09} = {932C804C-F6B7-4361-8360-E5238938E464}
+ {A92A0C23-698C-4F48-96D4-B6113564A85B} = {48CD2752-9462-473A-8699-7AD8B146742F}
+ {A953BE43-446B-4508-B144-A79AEAC6F7E1} = {48CD2752-9462-473A-8699-7AD8B146742F}
+ {32E26ABF-C631-405E-8EFB-1AD08A30CF5E} = {48CD2752-9462-473A-8699-7AD8B146742F}
  EndGlobalSection
  GlobalSection(ExtensibilityGlobals) = postSolution
  SolutionGuid = {2C5C9CC8-3EB2-4272-BBE5-637753D5E739}

benchmarks/Benchmarks/SimpleBenchmarks.cs

@@ -13,14 +13,14 @@ public class SimpleBenchmarks
  private KustoQueryContext _context ;
 
  [GlobalSetup]
- public void Setup()
+ public async Task Setup()
  {
  var console = new SystemConsole();
  _context = new KustoQueryContext();
  var settings = new KustoSettingsProvider();
- CsvSerializer.Default(settings,console)
-
-  .Load(@"C:\temp\locations.csv", "data");
+ var tableResult =await CsvSerializer.Default(settings,console)
+ .LoadTable(@"C:\temp\locations.csv", "data");
+ _context.AddTable(tableResult.Table);
  }
 
 

libraries/FileFormats/CsvSerializer.cs

@@ -1,11 +1,11 @@
 using System.Globalization;
+using System.Text;
 using CsvHelper;
 using CsvHelper.Configuration;
 using KustoLoco.Core;
 using KustoLoco.Core.Console;
 using KustoLoco.Core.Settings;
 using KustoLoco.Core.Util;
-using NLog;
 
 namespace KustoLoco.FileFormats;
 
@@ -14,6 +14,7 @@ public class CsvSerializer : ITableSerializer
  private readonly CsvConfiguration _config;
  private readonly IKustoConsole _console;
  private readonly KustoSettingsProvider _settings;
+ private readonly bool _skipHeader = false;
 
  public CsvSerializer(CsvConfiguration config, KustoSettingsProvider settings, IKustoConsole console)
  {
@@ -23,45 +24,16 @@ public CsvSerializer(CsvConfiguration config, KustoSettingsProvider settings, IK
  settings.Register(CsvSerializerSettings.SkipTypeInference, CsvSerializerSettings.TrimCells);
  }
 
- public Task<TableLoadResult> LoadTable(string path, string tableName)
+ public async Task<TableSaveResult> SaveTable(string path, KustoQueryResult result)
  {
- try
- {
- var table = Load(path, tableName);
- if (!_settings.GetBool(CsvSerializerSettings.SkipTypeInference))
- table = TableBuilder.AutoInferColumnTypes(table, _console);
-
- return Task.FromResult(TableLoadResult.Success(table));
- }
- catch (Exception e)
- {
- return Task.FromResult(TableLoadResult.Failure(e.Message));
- }
- }
-
- public Task<TableSaveResult> SaveTable(string path, KustoQueryResult result)
- {
- WriteToCsv(path, result);
- return Task.FromResult(TableSaveResult.Success());
+ await using var stream = File.OpenWrite(path);
+ return await SaveTable(stream, result);
  }
 
 
- public static CsvSerializer Default(KustoSettingsProvider settings, IKustoConsole console)
- {
- return new CsvSerializer(new CsvConfiguration(CultureInfo.InvariantCulture), settings, console);
- }
-
- public static CsvSerializer Tsv(KustoSettingsProvider settings, IKustoConsole console)
- {
- return new CsvSerializer(new CsvConfiguration(CultureInfo.InvariantCulture)
- {
- Delimiter = "\t"
- }
- , settings, console);
- }
-
- private ITableSource Load(TextReader reader, string tableName)
+ public Task<TableLoadResult> LoadTable(Stream stream, string tableName)
  {
+ using var reader = new StreamReader(stream);
  var csv = new CsvReader(reader, _config);
  csv.Read();
  csv.ReadHeader();
@@ -85,38 +57,48 @@ string TrimIfRequired(string s)
  if (rowCount % 100_000 == 0)
  _console.ShowProgress($"loaded {rowCount} records");
  }
- 
+
  var tableBuilder = TableBuilder.CreateEmpty(tableName, rowCount);
  for (var i = 0; i < keys.Length; i++) tableBuilder.WithColumn(keys[i], builders[i].ToColumn());
 
  var tableSource = tableBuilder.ToTableSource();
  _console.CompleteProgress($"loaded {rowCount} records");
- return tableSource;
- }
-
 
- public ITableSource Load(string filename, string tableName)
- {
- using TextReader fileReader = new StreamReader(filename);
- return Load(fileReader, tableName);
+ if (!_settings.GetBool(CsvSerializerSettings.SkipTypeInference))
+ tableSource = TableBuilder.AutoInferColumnTypes(tableSource, _console);
+ return Task.FromResult(TableLoadResult.Success(tableSource));
  }
 
 
- public ITableSource LoadFromString(string csv, string tableName)
+ public async Task<TableLoadResult> LoadTable(string filename, string tableName)
  {
- var reader = new StringReader(csv.Trim());
- var table = Load(reader, tableName);
- return table;
+ try
+ {
+ await using var fileReader = File.OpenRead(filename);
+ return await LoadTable(fileReader, tableName);
+ }
+ catch (Exception e)
+ {
+ return TableLoadResult.Failure(e.Message);
+ }
  }
 
 
- public void WriteToCsvStream(KustoQueryResult result, bool skipHeader, TextWriter writer)
+ public Task<TableSaveResult> SaveTable(Stream stream, KustoQueryResult result)
  {
+ if (result.ColumnCount == 0)
+ {
+ _console.Warn("No columns in result - empty file/stream written");
+ return Task.FromResult(TableSaveResult.Success());
+ }
+
+ using var writer = new StreamWriter(stream);
  using var csv = new CsvWriter(writer, _config);
- if (!skipHeader)
+ if (!_skipHeader)
  foreach (var heading in result.ColumnNames())
  csv.WriteField(heading);
  csv.NextRecord();
+
  var rowCount = 0;
  foreach (var r in result.EnumerateRows())
  {
@@ -133,30 +115,51 @@ public void WriteToCsvStream(KustoQueryResult result, bool skipHeader, TextWrite
 
  csv.NextRecord();
  }
+
  _console.CompleteProgress($"wrote {rowCount} records");
+ return Task.FromResult(TableSaveResult.Success());
  }
 
- private void WriteToCsv(string path, KustoQueryResult result)
+
+ public static CsvSerializer Default(KustoSettingsProvider settings, IKustoConsole console)
  {
- using var writer = new StreamWriter(path);
- WriteToCsvStream(result, false, writer);
+ return new CsvSerializer(new CsvConfiguration(CultureInfo.InvariantCulture), settings, console);
  }
 
+ public static CsvSerializer Tsv(KustoSettingsProvider settings, IKustoConsole console)
+ {
+ return new CsvSerializer(new CsvConfiguration(CultureInfo.InvariantCulture)
+ {
+ Delimiter = "\t"
+ }
+ , settings, console);
+ }
+
+
+ public ITableSource LoadFromString(string csv, string tableName)
+ {
+ var stream = new MemoryStream(Encoding.UTF8.GetBytes(csv.Trim()));
+ var result = LoadTable(stream, tableName).Result;
+ return result.Table;
+ }
+
+
  private static class CsvSerializerSettings
  {
- private const string prefix = "csv";
- public static readonly KustoSettingDefinition SkipTypeInference = new(
+ private const string Prefix = "csv";
+
+ public static readonly KustoSettingDefinition SkipTypeInference = new(
  Setting("skipTypeInference"), "prevents conversion of string columns to types",
  "off",
  nameof(Boolean));
- 
+
 
  public static readonly KustoSettingDefinition TrimCells = new(Setting("TrimCells"),
  "Removes leading and trailing whitespace from string values", "true", nameof(Boolean));
 
  private static string Setting(string setting)
  {
- return $"{prefix}.{setting}";
+ return $"{Prefix}.{setting}";
  }
  }
 }

libraries/FileFormats/ITableSerializer.cs

@@ -19,8 +19,13 @@ public interface ITableSerializer
  /// </summary>
  Task<TableLoadResult> LoadTable(string path,string tableName);
 
- Task<TableSaveResult> SaveTable(string path,KustoQueryResult result);
-}
+ Task<TableLoadResult> LoadTable(Stream stream, string tableName);
 
 
+ /// <summary>
+ /// Attempts to save a table to a patch
+ /// </summary>
+ Task<TableSaveResult> SaveTable(string path,KustoQueryResult result);
 
+ Task<TableSaveResult> SaveTable(Stream stream, KustoQueryResult result);
+}

libraries/FileFormats/JsonObjectArraySerializer.cs

@@ -1,4 +1,5 @@
 using System.Collections.Specialized;
+using System.Text;
 using System.Text.Json;
 using KustoLoco.Core;
 using KustoLoco.Core.Console;
@@ -20,18 +21,30 @@ public JsonObjectArraySerializer(KustoSettingsProvider settings, IKustoConsole c
  _settings.Register(JsonSerializerSettings.SkipTypeInference);
  }
 
- public Task<TableSaveResult> SaveTable(string path, KustoQueryResult result)
+
+ public async Task<TableSaveResult> SaveTable(string path, KustoQueryResult result)
+ {
+ await using var stream = File.OpenWrite(path);
+ return await SaveTable(stream, result); 
+ }
+
+ public async Task<TableSaveResult> SaveTable(Stream stream, KustoQueryResult result)
  {
  var json = result.ToJsonString();
- File.WriteAllText(path, json);
- return Task.FromResult(TableSaveResult.Success());
+ await stream.WriteAsync(Encoding.UTF8.GetBytes(json));
+ return TableSaveResult.Success();
+ }
+ public async Task<TableLoadResult> LoadTable(string path, string tableName)
+ {
+ await using var stream = File.OpenRead(path);
+ return await LoadTable(stream, tableName); 
  }
 
 
- public Task<TableLoadResult> LoadTable(string path, string name)
+ public Task<TableLoadResult> LoadTable(Stream stream, string name)
  {
- var text = File.ReadAllText(path);
- var dict = JsonSerializer.Deserialize<OrderedDictionary[]>(text);
+
+ var dict = JsonSerializer.Deserialize<OrderedDictionary[]>(stream);
  var sdlist = new List<OrderedDictionary>();
  foreach (var d in dict!)
  {