Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add custom filter to process batch of resources in parallel #576

Merged
merged 1 commit into from
Nov 1, 2024
Merged

Add custom filter to process batch of resources in parallel #576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Filters/CollectionFiltersTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,30 @@ public void BatchRenderTests()
maxIterations: 0,
formatProvider: CultureInfo.InvariantCulture,
cancellationToken: CancellationToken.None);

var collection = new List<object> { 1, 2, 3 };
Assert.Equal("1 ,2 ,3 ,", Filters.BatchRender(context, collection, "foo", "i"));
var result = Filters.BatchRenderParallel(context, collection, "foo", "i");
Assert.Contains("1 ", result);
Assert.Contains("2 ", result);
Assert.Contains("3 ", result);

// Valid template file system but null collection
Assert.Equal(string.Empty, Filters.BatchRender(context, null, "foo", "i"));
Assert.Equal(string.Empty, Filters.BatchRenderParallel(context, null, "foo", "i"));

// No template file system
context = new Context(CultureInfo.InvariantCulture);
var exception = Assert.Throws<RenderException>(() => Filters.BatchRender(context, null, "foo", "bar"));
Assert.Equal(FhirConverterErrorCode.TemplateNotFound, exception.FhirConverterErrorCode);
exception = Assert.Throws<RenderException>(() => Filters.BatchRenderParallel(context, null, "foo", "bar"));
Assert.Equal(FhirConverterErrorCode.TemplateNotFound, exception.FhirConverterErrorCode);

// Valid template file system but non-existing template
exception = Assert.Throws<RenderException>(() => Filters.BatchRender(context, collection, "bar", "i"));
Assert.Equal(FhirConverterErrorCode.TemplateNotFound, exception.FhirConverterErrorCode);
exception = Assert.Throws<RenderException>(() => Filters.BatchRenderParallel(context, collection, "bar", "i"));
Assert.Equal(FhirConverterErrorCode.TemplateNotFound, exception.FhirConverterErrorCode);
}
}
}
76 changes: 67 additions & 9 deletions src/Microsoft.Health.Fhir.Liquid.Converter/Filters/CollectionFilters.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using DotLiquid;
using Microsoft.Health.Fhir.Liquid.Converter.DotLiquids;
using Microsoft.Health.Fhir.Liquid.Converter.Exceptions;
Expand Down Expand Up @@ -35,28 +37,84 @@ public static List<object> Concat(List<object> l1, List<object> l2)
return new List<object>().Concat(l1 ?? new List<object>()).Concat(l2 ?? new List<object>()).ToList();
}

public static string BatchRender(Context context, List<object> collection, string templateName, string variableName)
public static string BatchRender(Context context, List<object> collection, string templateName, string variableName, string collectionVarName = null)
{
var templateFileSystem = context.Registers["file_system"] as IFhirConverterTemplateFileSystem;
var template = templateFileSystem?.GetTemplate(templateName, context[TemplateUtility.RootTemplateParentPathScope]?.ToString());

if (template == null)
{
throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, templateName));
}
var template = GetTemplate(context, templateName);

var sb = new StringBuilder();
context.Stack(() =>
{
collection?.ForEach(entry =>
{
context[variableName] = entry;
sb.Append(template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
if (!string.IsNullOrWhiteSpace(collectionVarName))
{
context[collectionVarName] = collection;
}

var result = template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture));

sb.Append(result);
sb.Append(',');
});
});

return sb.ToString();
}

public static string BatchRenderParallel(Context context, List<object> collection, string templateName, string variableName, string collectionVarName = null)
{
var template = GetTemplate(context, templateName);

var sb = new StringBuilder();
context.Stack(() =>
{
if (collection != null && collection.Any())
{
Parallel.ForEach(collection, entry =>
{
// Create a new context for each parallel task to avoid race conditions
var localContext = new Context(context.Environments, new Hash(), context.Registers, ErrorsOutputMode.Rethrow, context.MaxIterations, CultureInfo.InvariantCulture, CancellationToken.None);

foreach (var scope in context.Scopes)
{
foreach (var key in scope.Keys)
{
localContext[key] = scope[key];
}
}

localContext[variableName] = entry;
if (!string.IsNullOrWhiteSpace(collectionVarName))
{
localContext[collectionVarName] = collection;
}

var result = template.Render(RenderParameters.FromContext(localContext, CultureInfo.InvariantCulture));

lock (sb)
{
sb.Append(result);
sb.Append(',');
}
});
}
});

return sb.ToString();
}

private static Template GetTemplate(Context context, string templateName)
{
var templateFileSystem = context.Registers["file_system"] as IFhirConverterTemplateFileSystem;
var template = templateFileSystem?.GetTemplate(templateName, context[TemplateUtility.RootTemplateParentPathScope]?.ToString());

if (template == null)
{
throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, templateName));
}

return template;
}
}
}