Merge pull request #43 from NeilMacMullen/chatgpt

Provide experimental copilot support

applications/lokqlDx/Copilot.cs

@@ -0,0 +1,136 @@
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Text;
+using System.Text.Json;
+using KustoLoco.Core.Console;
+using NotNullStrings;
+
+namespace lokqlDx;
+
+public class Copilot
+{
+ public static class Roles
+ {
+ public const string User = "user";
+ public const string Assistant = "assistant";
+ public const string System = "system";
+ public const string Kql = "kql";
+ }
+ public void RenderResponses(IKustoConsole console,params string [] roles)
+ {
+ foreach(var message in context)
+ {
+ if (roles.Length > 0 && !roles.Contains(message.role))
+ {
+ continue;
+ }
+ var color = message.role switch
+ {
+ Roles.User => ConsoleColor.Green,
+ Roles.Assistant => ConsoleColor.White,
+ Roles.System => ConsoleColor.Red,
+ Roles.Kql => ConsoleColor.Yellow,
+ _ => ConsoleColor.White
+ };
+ console.ForegroundColor = color;
+ console.WriteLine(message.content);
+ console.WriteLine();
+ }
+ }
+
+ private HttpClient _client;
+
+ List<ChatMessage> context = [];
+ public Copilot(string apiToken)
+ {
+ Initialised = apiToken.IsNotBlank();
+ _client = new HttpClient();
+ _client.DefaultRequestHeaders.Authorization =
+ new AuthenticationHeaderValue("Bearer", apiToken);
+ AddSystemInstructions(@"
+You are an assistant to the user. You will be asked questions about tables of data and
+will be expected to provide answers in the form of KQL queries.
+
+This KQL engine does not support the top operator. Please avoid it in your responses.
+
+This KQL engine does not support joining on more than a single column. Please avoid doing this in your responses and
+instead use the extend operator to generate a new column that can be used for joining.
+
+This KQL implementation does not support arg_min or dynamic property types.
+
+You should provide only a brief explanation of the query and ensure that the KQL query is enclosed within the markdown code block syntax.
+Prefer to place each pipeline stage on a new line to improve readability.
+
+When asked to render a chart, use the render operator to specify the type of chart to render. The render operator should be the final operator in the query.
+The only 'with' property that the render operator supports is 'title'. Please use this to specify a suitable name for the chart. 
+
+When asked to render multi-series charts use the project operator to order columns such that the column that defines the series
+name is the final one in the list. Charts are rendered such that the first column is the x-axis, the second is the y-axis and
+the last one is the series name.
+
+In general, prefer to place any time-based column as the x-axis and quantity-based columns as the y-axis but this is not a strict requirement.
+
+Here are some common mistakes I want you to avoid:
+- Using the top operator
+- Using the arg_min operator
+- Using the dynamic property types
+- Joining on more than a single column
+- Using the render operator in the middle of the query
+- using 'm' or 'mon' to specify months in a timespan. Remember that 'm' is minutes and that 'mon' is not a valid timespan. Instead you need to convert months to a number of days.
+
+I will now give some some information about the schema of the tables that you will be asked to query.
+
+");
+ }
+
+
+ public async Task<string> Issue(string question)
+ {
+ AddUserMessage(question);
+ var requestData = new
+ {
+ model = "gpt-4",
+ messages = context
+ .Where(m=> m.role!= Roles.Kql)
+ .Select(x => new { x.role, x.content }).ToArray()
+ };
+
+ var requestContent = new StringContent(JsonSerializer.Serialize(requestData), Encoding.UTF8,
+ "application/json");
+ var response = await _client.PostAsync("https://api.openai.com/v1/chat/completions", requestContent);
+ var responseString = await response.Content.ReadAsStringAsync();
+ var responseDto = JsonSerializer.Deserialize<ChatResponse>(responseString);
+ await response.Content.ReadFromJsonAsync<ChatResponse>();
+ var assistanceResponse = responseDto?.choices?.FirstOrDefault()?.message.content ?? string.Empty;
+ AddCopilotResponse(assistanceResponse);
+ return assistanceResponse;
+ }
+
+ private void AddMessage(string role, string content) => context.Add(new ChatMessage(role, content));
+
+ private void AddUserMessage(string content) => AddMessage(Roles.User, content);
+
+ private void AddCopilotResponse(string content) => AddMessage(Roles.Assistant, content);
+ public void AddSystemInstructions(string content) => AddMessage(Roles.System, content);
+
+
+ public readonly record struct ChatMessage(string role, string content);
+
+ public class ChatResponse
+ {
+ public ChatChoice[] choices { get; set; } = [];
+ }
+
+ public class ChatChoice
+ {
+ public ChatMessage message { get; set; } = new ChatMessage();
+ }
+
+ public bool Initialised { get; private set; }
+
+ public void AddResponse(string response)
+ {
+ AddMessage(Roles.Kql,response);
+ }
+}

applications/lokqlDx/MainWindow.xaml

@@ -50,10 +50,50 @@
  <RowDefinition Height="Auto" />
  <RowDefinition Height="*" />
  </Grid.RowDefinitions>
- <local:QueryEditor x:Name="Editor" Grid.Row="0" 
+ <TabControl Grid.Row="0" >
+ <TabItem Header="Edit">
+ <local:QueryEditor x:Name="Editor" 
  RunEvent="OnQueryEditorRunTextBlock" />
+ </TabItem>
+ <TabItem Header="Copilot">
+ <DockPanel>
+ <StackPanel Orientation="Horizontal" DockPanel.Dock="Top">
+ <CheckBox Name ="TerseMode" VerticalAlignment="Center">Terse</CheckBox>
+ </StackPanel>
+ <Grid>
+ <Grid.RowDefinitions>
+ <RowDefinition Height="*" />
+ <RowDefinition Height="Auto" />
+ <RowDefinition Height="*" />
+ </Grid.RowDefinitions>
+ <RichTextBox Grid.Row="0" Name="ChatHistory" FontFamily="Consolas"
+ Background="DarkBlue"
+ FontSize="10"/>
+ <GridSplitter Grid.Row="1"
+ VerticalAlignment="Stretch"
+ Height="5"
+ HorizontalAlignment="Stretch"
+ Background="DarkGray"/>
+ <DockPanel Grid.Row="2">
+ <DockPanel DockPanel.Dock="Bottom" Margin="5" >
+ <Button Name="SubmitButton" DockPanel.Dock="Right" Width="100" HorizontalAlignment="Right" Click="SubmitToCopilot">Submit</Button>
+ <Button Name="ClearChatButton" DockPanel.Dock="Left" Width="100" HorizontalAlignment="Left" Click="ResetCopilot">Restart</Button>
+ </DockPanel>
+
+ <TextBox Name="UserChat" FontFamily="Consolas"
+ Background="LightBlue"
+ FontSize="10"
+ AcceptsReturn="True"/>
 
- <GridSplitter Grid.Row="1"
+ </DockPanel>
+
+ </Grid>
+
+ </DockPanel>
+
+ </TabItem>
+ </TabControl>
+ <GridSplitter Grid.Row="1"
  VerticalAlignment="Stretch"
  Height="5"
  HorizontalAlignment="Stretch"

applications/lokqlDx/MainWindow.xaml.cs

@@ -1,5 +1,6 @@
 using System.ComponentModel;
 using System.IO;
+using System.Text;
 using System.Windows;
 using System.Windows.Input;
 using System.Windows.Shell;
@@ -8,19 +9,21 @@
 using KustoLoco.Core.Settings;
 using KustoLoco.Rendering;
 using Lokql.Engine;
-using Lokql.Engine.Commands;
 using Microsoft.Win32;
+using NotNullStrings;
 
 namespace lokqlDx;
 
 public partial class MainWindow : Window
 {
  private readonly string[] _args;
  private readonly WpfConsole _console;
+ private readonly Size _minWindowSize = new(600, 400);
  private readonly PreferencesManager _preferenceManager = new();
  private readonly WorkspaceManager _workspaceManager;
- private InteractiveTableExplorer _explorer;
 
+ private Copilot _copilot = new(string.Empty);
+ private InteractiveTableExplorer _explorer;
  private bool isBusy;
 
 
@@ -100,20 +103,27 @@ private void UpdateUIFromWorkspace()
  Editor.SetText(_workspaceManager.UserText);
  var settings = _workspaceManager.Settings;
  var loader = new StandardFormatAdaptor(settings, _console);
- _explorer = new InteractiveTableExplorer(_console, loader, settings, CommandProcessorProvider.GetCommandProcessor());
+ _explorer = new InteractiveTableExplorer(_console, loader, settings,
+ CommandProcessorProvider.GetCommandProcessor());
  UpdateFontSize();
  Title = $"LokqlDX - {_workspaceManager.Path}";
  }
 
  private async void MainWindow_OnLoaded(object sender, RoutedEventArgs e)
  {
  _preferenceManager.Load();
- var pathToLoad = _args.Any() ? _args[0] : _preferenceManager.Preferences.LastWorkspacePath;
+ var pathToLoad = _args.Any()
+ ? _args[0]
+ : _preferenceManager.Preferences.LastWorkspacePath;
  _workspaceManager.Load(pathToLoad);
  if (Width > 100 && Height > 100 && Left > 0 && Top > 0)
  {
- Width = _preferenceManager.Preferences.WindowWidth;
- Height = _preferenceManager.Preferences.WindowHeight;
+ Width = _preferenceManager.Preferences.WindowWidth < _minWindowSize.Width
+ ? _minWindowSize.Width
+ : _preferenceManager.Preferences.WindowWidth;
+ Height = _preferenceManager.Preferences.WindowHeight < _minWindowSize.Height
+ ? _minWindowSize.Height
+ : _preferenceManager.Preferences.WindowHeight;
  Left = _preferenceManager.Preferences.WindowLeft;
  Top = _preferenceManager.Preferences.WindowTop;
  }
@@ -230,6 +240,8 @@ private void UpdateFontSize()
  Editor.SetFontSize(_preferenceManager.Preferences.FontSize);
  OutputText.FontSize = _preferenceManager.Preferences.FontSize;
  dataGrid.FontSize = _preferenceManager.Preferences.FontSize;
+ UserChat.FontSize = _preferenceManager.Preferences.FontSize;
+ ChatHistory.FontSize = _preferenceManager.Preferences.FontSize;
  }
 
  protected override void OnKeyDown(KeyEventArgs e)
@@ -266,4 +278,81 @@ private void EnableJumpList(object sender, RoutedEventArgs e)
  {
  RegistryOperations.AssociateFileType();
  }
+
+ private async void SubmitToCopilot(object sender, RoutedEventArgs e)
+ {
+ SubmitButton.IsEnabled = false;
+ if (!_copilot.Initialised)
+ {
+ _copilot = new Copilot(_explorer.Settings.GetOr("copilot", string.Empty));
+ foreach (var table in _explorer.GetCurrentContext().Tables())
+ {
+ var sb = new StringBuilder();
+ sb.AppendLine($"The table named '{table.Name}' has the following columns");
+ var cols = table.ColumnNames.Zip(table.Type.Columns)
+ .Select(z => $" {z.First} is of type {z.Second.Type.Name}").ToArray();
+ foreach (var column in cols) sb.AppendLine(column);
+ _copilot.AddSystemInstructions(sb.ToString());
+ }
+ }
+
+ var userchat = UserChat.Text;
+ UserChat.Text = string.Empty;
+ const int maxResubmissions = 3;
+ for (var i = 0; i < maxResubmissions; i++)
+ {
+ var response = await _copilot.Issue(userchat);
+
+
+ var console = new WpfConsole(ChatHistory);
+
+ //now try to extract kql...
+ var lines = response.Split('\n');
+ var kql = new StringBuilder();
+ var getting = false;
+ foreach (var line in lines)
+ {
+ if (line.StartsWith("```kql") || line.StartsWith("```kusto"))
+ {
+ kql.Clear();
+ getting = true;
+ continue;
+ }
+
+ if (line.StartsWith("```"))
+ {
+ getting = false;
+ continue;
+ }
+
+ if (getting)
+ kql.AppendLine(line.Trim());
+ }
+
+ _copilot.AddResponse(kql.ToString());
+ console.PrepareForOutput();
+ var options = new List<string> { Copilot.Roles.System, Copilot.Roles.User, Copilot.Roles.Kql };
+ if (TerseMode.IsChecked != true)
+ options.Add(Copilot.Roles.Assistant);
+ _copilot.RenderResponses(console, options.ToArray());
+
+ if (kql.ToString().IsBlank())
+ break;
+ await RunQuery(kql.ToString());
+ var lastResult = _explorer._prevResultIncludingError;
+
+ if (lastResult.Error.IsBlank())
+ break;
+ userchat = $"That query gave an error: {lastResult.Error}";
+ }
+
+
+ SubmitButton.IsEnabled = true;
+ }
+
+ private void ResetCopilot(object sender, RoutedEventArgs e)
+ {
+ _copilot = new Copilot(string.Empty);
+ ChatHistory.Document.Blocks.Clear();
+ }
 }

applications/lokqlDx/Preferences.cs

@@ -3,7 +3,7 @@
 public class Preferences
 {
  public string LastWorkspacePath { get; set; } = string.Empty;
- public double FontSize { get; set; }
+ public double FontSize { get; set; } = 20;
  public string FontFamily { get; set; } = string.Empty;
  public double WindowWidth { get; set; }
  public double WindowHeight { get; set; }

applications/lokqlDx/WorkspaceManager.cs

@@ -71,6 +71,18 @@ public void Save(string path,string userText)
 
  public void Load(string path)
  {
+ if (!File.Exists(path))
+ {
+ var rootSettingFolderPath =
+ System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
+ "kustoloco");
+ if (!Directory.Exists(rootSettingFolderPath))
+ Directory.CreateDirectory(rootSettingFolderPath);
+
+ path = System.IO.Path.Combine(rootSettingFolderPath, "settings");
+ File.WriteAllText(path, JsonSerializer.Serialize(new Workspace()));
+ }
+
  Path = path;
  try
  {

libraries/KustoLoco.Core/Evaluation/BuiltIns/BuiltInScalarFunctions.cs

@@ -148,6 +148,7 @@ static void AddCoalesce(List<ScalarOverloadInfo> overloads, Func<IScalarFunction
  StartOfYearFunction.Register(functions);
  EndOfYearFunction.Register(functions);
  DatetimeDiffFunction.Register(functions);
+ DatetimePartFunction.Register(functions);
  functions.Add(Functions.DatetimeUtcToLocal,
  new ScalarFunctionInfo(new ScalarOverloadInfo(new DateTimeUtcToLocalFunctionImpl(),
  ScalarTypes.DateTime,