Skip to content

Commit

Permalink
Further improve LG's performance (#4083)
Browse files Browse the repository at this point in the history 
* add templateBodyCache

* update

* re-do updateTemplate

* improve perf for addTemplate and DeleteTemplate

* add more tests and remove cache

* adjust test

* adjust

* fix environment newline issue

* try fix ci error

* add tests

* extract methods

* refine

* fix

* revert

* renaming and add more tests

* add more comments

* extract range offset
  • Loading branch information
@Danieladu
Danieladu authored Jun 29, 2020
1 parent 09de1f7 commit 59f8ccc
Show file tree
Hide file tree
Showing 6 changed files with 368 additions and 101 deletions.
9 changes: 0 additions & 9 deletions libraries/Microsoft.Bot.Builder.LanguageGeneration/SourceRange.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ public class SourceRange
public SourceRange(ParserRuleContext parseTree, string source = "", int offset = 0)
{
this.Source = source ?? string.Empty;
this.ParseTree = parseTree;
this.Range = parseTree.ConvertToRange(offset);
}

Expand All @@ -38,13 +37,5 @@ public SourceRange(Range range, string source = "")
/// Code source, used as the lg file path.
/// </value>
public string Source { get; set; }

/// <summary>
/// Gets or sets content parse tree form LGFileParser.g4.
/// </summary>
/// <value>
/// Content parse tree form LGFileParser.g4.
/// </value>
public ParserRuleContext ParseTree { get; set; }
}
}
161 changes: 134 additions & 27 deletions libraries/Microsoft.Bot.Builder.LanguageGeneration/Templates.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.Text.RegularExpressions;
using AdaptiveExpressions;
using AdaptiveExpressions.Memory;
using static Microsoft.Bot.Builder.LanguageGeneration.TemplatesParser;

namespace Microsoft.Bot.Builder.LanguageGeneration
{
Expand Down Expand Up @@ -280,15 +281,31 @@ public Templates UpdateTemplate(string templateName, string newTemplateName, Lis
var template = this.FirstOrDefault(u => u.Name == templateName);
if (template != null)
{
ClearDiagnostics();

var templateNameLine = BuildTemplateNameLine(newTemplateName, parameters);
var newTemplateBody = ConvertTemplateBody(templateBody);
var content = $"{templateNameLine}{newLine}{newTemplateBody}";

var startLine = template.SourceRange.Range.Start.Line - 1;
var stopLine = template.SourceRange.Range.End.Line - 1;
// update content
this.Content = ReplaceRangeContent(
this.Content,
template.SourceRange.Range.Start.Line - 1,
template.SourceRange.Range.End.Line - 1,
content);

var updatedTemplates = new Templates(content: string.Empty, id: Id, importResolver: ImportResolver, expressionParser: ExpressionParser);
updatedTemplates = new TemplatesTransformer(updatedTemplates).Transform(AntlrParseTemplates(content, Id));

var newContent = ReplaceRangeContent(Content, startLine, stopLine, content);
Initialize(ParseText(newContent, Id, ImportResolver));
var originStartLine = template.SourceRange.Range.Start.Line - 1;
AppendDiagnosticsWithOffset(updatedTemplates.Diagnostics, originStartLine);

var newTemplate = updatedTemplates.FirstOrDefault();
if (newTemplate != null)
{
AdjustRangeForUpdateTemplate(template, newTemplate);
new StaticChecker(this).Check().ForEach(u => this.Diagnostics.Add(u));
}
}

return this;
Expand All @@ -309,10 +326,29 @@ public Templates AddTemplate(string templateName, List<string> parameters, strin
throw new Exception(TemplateErrors.TemplateExist(templateName));
}

ClearDiagnostics();

var templateNameLine = BuildTemplateNameLine(templateName, parameters);
var newTemplateBody = ConvertTemplateBody(templateBody);
var newContent = $"{Content}{newLine}{templateNameLine}{newLine}{newTemplateBody}";
Initialize(ParseText(newContent, Id, ImportResolver));
var content = $"{templateNameLine}{newLine}{newTemplateBody}";

var originStartLine = GetLinesOfText(this.Content).Length;

// update content
this.Content = $"{Content}{newLine}{templateNameLine}{newLine}{newTemplateBody}";

var newTemplates = new Templates(content: string.Empty, id: Id, importResolver: ImportResolver, expressionParser: ExpressionParser);
newTemplates = new TemplatesTransformer(newTemplates).Transform(AntlrParseTemplates(content, Id));

AppendDiagnosticsWithOffset(newTemplates.Diagnostics, originStartLine);

var newTemplate = newTemplates.FirstOrDefault();
if (newTemplate != null)
{
AdjustRangeForAddTemplate(newTemplate, originStartLine);
this.Add(newTemplate);
new StaticChecker(this).Check().ForEach(u => this.Diagnostics.Add(u));
}

return this;
}
Expand All @@ -327,10 +363,15 @@ public Templates DeleteTemplate(string templateName)
var template = this.FirstOrDefault(u => u.Name == templateName);
if (template != null)
{
ClearDiagnostics();

var startLine = template.SourceRange.Range.Start.Line - 1;
var stopLine = template.SourceRange.Range.End.Line - 1;
var newContent = ReplaceRangeContent(Content, startLine, stopLine, null);
Initialize(ParseText(newContent, Id, ImportResolver));
this.Content = ReplaceRangeContent(Content, startLine, stopLine, null);

AdjustRangeForDeleteTemplate(template);
this.Remove(template);
new StaticChecker(this).Check().ForEach(u => this.Diagnostics.Add(u));
}

return this;
Expand Down Expand Up @@ -369,6 +410,74 @@ private Templates InjectToExpressionFunction()
return this;
}

private void AppendDiagnosticsWithOffset(IList<Diagnostic> diagnostics, int offset)
{
if (diagnostics != null)
{
diagnostics.ToList().ForEach(u =>
{
u.Range.Start.Line += offset;
u.Range.End.Line += offset;
this.Diagnostics.Add(u);
});
}
}

private void AdjustRangeForUpdateTemplate(Template oldTemplate, Template newTemplate)
{
var newRange = newTemplate.SourceRange.Range.End.Line - newTemplate.SourceRange.Range.Start.Line;
var oldRange = oldTemplate.SourceRange.Range.End.Line - oldTemplate.SourceRange.Range.Start.Line;
var lineOffset = newRange - oldRange;

var hasFound = false;

for (var i = 0; i < this.Count; i++)
{
if (hasFound)
{
this[i].SourceRange.Range.Start.Line += lineOffset;
this[i].SourceRange.Range.End.Line += lineOffset;
}
else if (this[i].Name == oldTemplate.Name)
{
hasFound = true;
newTemplate.SourceRange.Range.Start.Line = oldTemplate.SourceRange.Range.Start.Line;
newTemplate.SourceRange.Range.End.Line = oldTemplate.SourceRange.Range.End.Line + lineOffset;
this[i] = newTemplate;
}
}
}

private void AdjustRangeForAddTemplate(Template newTemplate, int lineOffset)
{
var lineLength = newTemplate.SourceRange.Range.End.Line - newTemplate.SourceRange.Range.Start.Line;
newTemplate.SourceRange.Range.Start.Line = lineOffset + 1;
newTemplate.SourceRange.Range.End.Line = lineLength + lineOffset + 1;
}

private void AdjustRangeForDeleteTemplate(Template oldTemplate)
{
var lineOffset = oldTemplate.SourceRange.Range.End.Line - oldTemplate.SourceRange.Range.Start.Line + 1;
var hasFound = false;
for (var i = 0; i < this.Count; i++)
{
if (hasFound)
{
this[i].SourceRange.Range.Start.Line -= lineOffset;
this[i].SourceRange.Range.End.Line -= lineOffset;
}
else if (this[i].Name == oldTemplate.Name)
{
hasFound = true;
}
}
}

private void ClearDiagnostics()
{
this.Diagnostics = new List<Diagnostic>();
}

private string ReplaceRangeContent(string originString, int startLine, int stopLine, string replaceString)
{
var originList = originString.Split(new[] { "\r\n", "\n", "\r" }, StringSplitOptions.None);
Expand All @@ -381,15 +490,20 @@ private string ReplaceRangeContent(string originString, int startLine, int stopL
var destList = new List<string>();

destList.AddRange(originList.Take(startLine));
destList.Add(replaceString);

if (replaceString != null)
{
destList.Add(replaceString);
}

destList.AddRange(originList.Skip(stopLine + 1));

return string.Join(newLine, destList);
}

private string ConvertTemplateBody(string templateBody)
{
var lines = templateBody.Split(new[] { "\r\n", "\n", "\r" }, StringSplitOptions.None);
var lines = GetLinesOfText(templateBody);
var destList = lines.Select(u =>
{
return u.TrimStart().StartsWith("#") ? $"- {u.TrimStart()}" : u;
Expand All @@ -398,6 +512,16 @@ private string ConvertTemplateBody(string templateBody)
return string.Join(newLine, destList);
}

private string[] GetLinesOfText(string text)
{
if (text == null)
{
return new string[0];
}

return text.Split(new[] { "\r\n", "\n", "\r" }, StringSplitOptions.None);
}

private string BuildTemplateNameLine(string templateName, List<string> parameters)
{
if (parameters == null)
Expand All @@ -410,23 +534,6 @@ private string BuildTemplateNameLine(string templateName, List<string> parameter
}
}

/// <summary>
/// Use an existing LG file to override current object.
/// </summary>
/// <param name="templates">Existing LG file.</param>
private void Initialize(Templates templates)
{
this.Clear();
this.AddRange(templates);
Imports = templates.Imports;
Diagnostics = templates.Diagnostics;
References = templates.References;
Content = templates.Content;
ImportResolver = templates.ImportResolver;
Id = templates.Id;
ExpressionParser = templates.ExpressionParser;
}

private void CheckErrors()
{
if (AllDiagnostics != null)
Expand Down
75 changes: 45 additions & 30 deletions libraries/Microsoft.Bot.Builder.LanguageGeneration/TemplatesParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
// Licensed under the MIT License.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -108,6 +110,34 @@ public static Templates ParseTextWithRef(string content, Templates lg)
return newLG;
}

/// <summary>
/// Parse LG content and achieve the AST.
/// </summary>
/// <param name="text">LG content.</param>
/// <param name="id">Source id.</param>
/// <returns>The abstract syntax tree of lg file.</returns>
public static IParseTree AntlrParseTemplates(string text, string id)
{
if (string.IsNullOrEmpty(text))
{
return null;
}

var input = new AntlrInputStream(text);
var lexer = new LGFileLexer(input);
lexer.RemoveErrorListeners();

var tokens = new CommonTokenStream(lexer);
var parser = new LGFileParser(tokens);
parser.RemoveErrorListeners();
var listener = new ErrorListener(id);

parser.AddErrorListener(listener);
parser.BuildParseTree = true;

return parser.file();
}

/// <summary>
/// Parser to turn lg content into a <see cref="Templates"/>.
/// </summary>
Expand Down Expand Up @@ -168,28 +198,6 @@ private static (string content, string id) DefaultFileResolver(string sourceId,
return (File.ReadAllText(importPath), importPath);
}

private static IParseTree AntlrParseTemplates(string text, string id)
{
if (string.IsNullOrEmpty(text))
{
return null;
}

var input = new AntlrInputStream(text);
var lexer = new LGFileLexer(input);
lexer.RemoveErrorListeners();

var tokens = new CommonTokenStream(lexer);
var parser = new LGFileParser(tokens);
parser.RemoveErrorListeners();
var listener = new ErrorListener(id);

parser.AddErrorListener(listener);
parser.BuildParseTree = true;

return parser.file();
}

private static IList<Templates> GetReferences(Templates file, Dictionary<string, Templates> cachedTemplates = null)
{
var resourcesFound = new HashSet<Templates>();
Expand All @@ -213,7 +221,7 @@ private static void ResolveImportResources(Templates start, HashSet<Templates> r
}
catch (Exception e)
{
var diagnostic = new Diagnostic(import.SourceRange.ParseTree.ConvertToRange(), e.Message, DiagnosticSeverity.Error, start.Id);
var diagnostic = new Diagnostic(import.SourceRange.Range, e.Message, DiagnosticSeverity.Error, start.Id);
throw new TemplateException(e.Message, new List<Diagnostic>() { diagnostic });
}

Expand All @@ -235,7 +243,10 @@ private static void ResolveImportResources(Templates start, HashSet<Templates> r
}
}

private class TemplatesTransformer : LGFileParserBaseVisitor<object>
/// <summary>
/// Templates transfeormer. Fullfill more details and body context into the templates object.
/// </summary>
public class TemplatesTransformer : LGFileParserBaseVisitor<object>
{
private static readonly Regex IdentifierRegex = new Regex(@"^[0-9a-zA-Z_]+$");
private static readonly Regex TemplateNamePartRegex = new Regex(@"^[a-zA-Z_][0-9a-zA-Z_]*$");
Expand All @@ -246,13 +257,23 @@ public TemplatesTransformer(Templates templates)
this.templates = templates;
}

/// <summary>
/// Transform the parse tree into templates.
/// </summary>
/// <param name="parseTree">Input abstract syntax tree.</param>
/// <returns>Templates.</returns>
public Templates Transform(IParseTree parseTree)
{
if (parseTree != null)
{
Visit(parseTree);
}

for (var i = 0; i < templates.Count - 1; i++)
{
templates[i].Body = RemoveTrailingNewline(templates[i].Body);
}

return this.templates;
}

Expand Down Expand Up @@ -322,12 +343,6 @@ public override object VisitTemplateDefinition([NotNull] LGFileParser.TemplateDe
else
{
var templateBody = context.templateBody().GetText();
var file = context.Parent.Parent as LGFileParser.FileContext;
var isLastTemplate = file.paragraph().Select(u => u.templateDefinition()).Where(u => u != null).Last() == context;
if (!isLastTemplate)
{
templateBody = RemoveTrailingNewline(templateBody);
}

var sourceRange = new SourceRange(context, this.templates.Id);
var template = new Template(templateName, parameters, templateBody, sourceRange);
Expand Down
12 changes: 12 additions & 0 deletions tests/Microsoft.Bot.Builder.LanguageGeneration.Tests/Examples/CrudInit.lg
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
> this is the lg file that is used to test the template crud
>
# template1
> this is an in-template comment
- Hi
- Hello
- Hiya

# template2
- Good morning
- Good afternoon
- Good night
Loading

0 comments on commit 59f8ccc

Please sign in to comment.