diff --git a/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java b/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java index 6e2b8828e..532336586 100644 --- a/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java +++ b/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java @@ -564,6 +564,10 @@ public void setPosition(int position) { } public void addError(TemplateError templateError) { + if (context.getHideInterpreterErrors()) { + // Hiding errors when resolving chunks. + return; + } // fix line numbers not matching up with source template if (!context.getCurrentPathStack().isEmpty()) { if (!templateError.getSourceTemplate().isPresent()) { diff --git a/src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java b/src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java index 94c106681..8adda97e2 100644 --- a/src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java +++ b/src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java @@ -1,8 +1,19 @@ package com.hubspot.jinjava.lib.expression; +import com.hubspot.jinjava.JinjavaConfig; import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.lib.filter.EscapeFilter; +import com.hubspot.jinjava.lib.tag.RawTag; +import com.hubspot.jinjava.lib.tag.eager.EagerStringResult; +import com.hubspot.jinjava.lib.tag.eager.EagerTagDecorator; +import com.hubspot.jinjava.lib.tag.eager.EagerToken; import com.hubspot.jinjava.tree.output.RenderedOutputNode; import com.hubspot.jinjava.tree.parse.ExpressionToken; +import com.hubspot.jinjava.tree.parse.TagToken; +import com.hubspot.jinjava.util.ChunkResolver; +import com.hubspot.jinjava.util.Logging; +import com.hubspot.jinjava.util.WhitespaceUtils; +import org.apache.commons.lang3.StringUtils; public class EagerExpressionStrategy implements ExpressionStrategy { @@ -11,6 +22,111 @@ public RenderedOutputNode interpretOutput( ExpressionToken master, JinjavaInterpreter interpreter ) { - return new DefaultExpressionStrategy().interpretOutput(master, interpreter); // TODO replace with actual functionality + EagerStringResult eagerStringResult = eagerResolveExpression(master, interpreter); + return new RenderedOutputNode( + eagerStringResult.getPrefixToPreserveState() + eagerStringResult.getResult() + ); + } + + private EagerStringResult eagerResolveExpression( + ExpressionToken master, + JinjavaInterpreter interpreter + ) { + ChunkResolver chunkResolver = new ChunkResolver( + master.getExpr(), + master, + interpreter + ); + EagerStringResult resolvedExpression = EagerTagDecorator.executeInChildContext( + eagerInterpreter -> chunkResolver.resolveChunks(), + interpreter, + true + ); + StringBuilder prefixToPreserveState = new StringBuilder( + interpreter.getContext().isProtectedMode() + ? resolvedExpression.getPrefixToPreserveState() + : "" + ); + if (chunkResolver.getDeferredWords().isEmpty()) { + String result = WhitespaceUtils.unquote(resolvedExpression.getResult()); + if ( + !StringUtils.equals(result, master.getImage()) && + ( + StringUtils.contains(result, master.getSymbols().getExpressionStart()) || + StringUtils.contains(result, master.getSymbols().getExpressionStartWithTag()) + ) + ) { + if (interpreter.getConfig().isNestedInterpretationEnabled()) { + try { + result = interpreter.renderFlat(result); + } catch (Exception e) { + Logging.ENGINE_LOG.warn("Error rendering variable node result", e); + } + } else { + // Possible macro/set tag in front of this one. Includes result + result = wrapInRawOrExpressionIfNeeded(result, interpreter); + } + } + + if (interpreter.getContext().isAutoEscape()) { + result = EscapeFilter.escapeHtmlEntities(result); + } + return new EagerStringResult(result, prefixToPreserveState.toString()); + } + prefixToPreserveState.append( + EagerTagDecorator.reconstructFromContextBeforeDeferring( + chunkResolver.getDeferredWords(), + interpreter + ) + ); + String helpers = wrapInExpression(resolvedExpression.getResult(), interpreter); + interpreter + .getContext() + .handleEagerToken( + new EagerToken( + new TagToken( + helpers, + master.getLineNumber(), + master.getStartPosition(), + master.getSymbols() + ), + chunkResolver.getDeferredWords() + ) + ); + // There is no result because it couldn't be entirely evaluated. + return new EagerStringResult( + "", + EagerTagDecorator.wrapInAutoEscapeIfNeeded( + prefixToPreserveState.toString() + helpers, + interpreter + ) + ); + } + + private static String wrapInRawOrExpressionIfNeeded( + String output, + JinjavaInterpreter interpreter + ) { + JinjavaConfig config = interpreter.getConfig(); + if ( + config.getExecutionMode().isPreserveRawTags() && + ( + output.contains(config.getTokenScannerSymbols().getExpressionStart()) || + output.contains(config.getTokenScannerSymbols().getExpressionStartWithTag()) + ) + ) { + return EagerTagDecorator.wrapInTag(output, RawTag.TAG_NAME, interpreter); + } + return output; + } + + private static String wrapInExpression(String output, JinjavaInterpreter interpreter) { + JinjavaConfig config = interpreter.getConfig(); + return String.format( + "%s %s %s", + config.getTokenScannerSymbols().getExpressionStart(), + output, + config.getTokenScannerSymbols().getExpressionEnd() + ); } } diff --git a/src/main/java/com/hubspot/jinjava/lib/fn/eager/EagerMacroFunction.java b/src/main/java/com/hubspot/jinjava/lib/fn/eager/EagerMacroFunction.java index bedbddcac..8ef465066 100644 --- a/src/main/java/com/hubspot/jinjava/lib/fn/eager/EagerMacroFunction.java +++ b/src/main/java/com/hubspot/jinjava/lib/fn/eager/EagerMacroFunction.java @@ -52,6 +52,7 @@ public Object doEvaluate( ) { Optional importFile = macroFunction.getImportFile(interpreter); try (InterpreterScopeClosable c = interpreter.enterScope()) { + interpreter.getContext().setProtectedMode(true); return macroFunction.getEvaluationResult(argMap, kwargMap, varArgs, interpreter); } finally { importFile.ifPresent(path -> interpreter.getContext().getCurrentPathStack().pop()); diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecorator.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecorator.java index e7b8c3d5e..34d416a96 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecorator.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecorator.java @@ -17,6 +17,7 @@ import com.hubspot.jinjava.lib.fn.MacroFunction; import com.hubspot.jinjava.lib.fn.eager.EagerMacroFunction; import com.hubspot.jinjava.lib.tag.AutoEscapeTag; +import com.hubspot.jinjava.lib.tag.MacroTag; import com.hubspot.jinjava.lib.tag.RawTag; import com.hubspot.jinjava.lib.tag.SetTag; import com.hubspot.jinjava.lib.tag.Tag; @@ -118,7 +119,7 @@ public String eagerInterpret(TagNode tagNode, JinjavaInterpreter interpreter) { * Render all children of this TagNode. * @param tagNode TagNode to render the children of. * @param interpreter The JinjavaInterpreter. - * @return + * @return the string output of this tag node's children. */ public String renderChildren(TagNode tagNode, JinjavaInterpreter interpreter) { StringBuilder sb = new StringBuilder(); @@ -231,6 +232,27 @@ public static EagerStringResult executeInChildContext( return new EagerStringResult(result.toString()); } + /** + * Reconstruct the macro functions and variables from the context before they + * get deferred. + * Those macro functions and variables found within {@code deferredWords} are + * reconstructed with {@link MacroTag}(s) and a {@link SetTag}, respectively to + * preserve the context within the Jinjava template itself. + * @param deferredWords set of words that will need to be deferred based on the + * previously performed operation. + * @param interpreter the Jinjava interpreter. + * @return a Jinjava-syntax string of 0 or more macro tags and 0 or 1 set tags. + */ + public static String reconstructFromContextBeforeDeferring( + Set deferredWords, + JinjavaInterpreter interpreter + ) { + return ( + reconstructMacroFunctionsBeforeDeferring(deferredWords, interpreter) + + reconstructVariablesBeforeDeferring(deferredWords, interpreter) + ); + } + /** * Build macro tag images for any macro functions that are included in deferredWords * and remove those macro functions from the deferredWords set. @@ -242,7 +264,7 @@ public static EagerStringResult executeInChildContext( * @return A jinjava-syntax string that is the images of any macro functions that must * be evaluated at a later time. */ - public static String getNewlyDeferredFunctionImages( + private static String reconstructMacroFunctionsBeforeDeferring( Set deferredWords, JinjavaInterpreter interpreter ) { @@ -279,12 +301,42 @@ public static String getNewlyDeferredFunctionImages( ) .map(EagerStringResult::toString) .collect(Collectors.joining()); + // Remove macro functions from the set because they've been fully processed now. deferredWords.removeAll(toRemove); return result; } + private static String reconstructVariablesBeforeDeferring( + Set deferredWords, + JinjavaInterpreter interpreter + ) { + if (interpreter.getContext().isProtectedMode()) { + return ""; // This will be handled outside of the protected mode. + } + Map deferredMap = new HashMap<>(); + deferredWords + .stream() + .map(w -> w.split("\\.", 2)[0]) // get base prop + .filter( + w -> + interpreter.getContext().containsKey(w) && + !(interpreter.getContext().get(w) instanceof DeferredValue) + ) + .forEach( + w -> { + try { + deferredMap.put( + w, + ChunkResolver.getValueAsJinjavaString(interpreter.getContext().get(w)) + ); + } catch (JsonProcessingException ignored) {} + } + ); + return buildSetTagForDeferredInChildContext(deferredMap, interpreter, true); + } + /** - * Build the image for a set tag which preserves the values of objects on the context + * Build the image for a {@link SetTag} which preserves the values of objects on the context * for a later rendering pass. The set tag will set the keys to the values within * the {@code deferredValuesToSet} Map. * @param deferredValuesToSet Map that specifies what the context objects should be set @@ -300,6 +352,9 @@ public static String buildSetTagForDeferredInChildContext( JinjavaInterpreter interpreter, boolean registerEagerToken ) { + if (deferredValuesToSet.size() == 0) { + return ""; + } if ( interpreter.getConfig().getDisabled().containsKey(Library.TAG) && interpreter.getConfig().getDisabled().get(Library.TAG).contains(SetTag.TAG_NAME) @@ -395,7 +450,7 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter joiner.add(resolvedChunks); } joiner.add(tagToken.getSymbols().getExpressionEndWithTag()); - String newlyDeferredFunctionImages = getNewlyDeferredFunctionImages( + String reconstructedFromContext = reconstructFromContextBeforeDeferring( chunkResolver.getDeferredWords(), interpreter ); @@ -414,7 +469,7 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter ) ); - return (newlyDeferredFunctionImages + joiner.toString()); + return (reconstructedFromContext + joiner.toString()); } public static String reconstructEnd(TagNode tagNode) { diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScannerSymbols.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScannerSymbols.java index ed82ba91a..dc046a767 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScannerSymbols.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScannerSymbols.java @@ -23,6 +23,7 @@ public abstract class TokenScannerSymbols implements Serializable { private String expressionStart = null; private String expressionStartWithTag = null; private String closingComment = null; + private String expressionEnd = null; private String expressionEndWithTag = null; public abstract char getPrefixChar(); @@ -86,6 +87,13 @@ public String getExpressionStart() { return expressionStart; } + public String getExpressionEnd() { + if (expressionEnd == null) { + expressionEnd = String.valueOf(getExprEndChar()) + getPostfixChar(); + } + return expressionEnd; + } + public String getExpressionStartWithTag() { if (expressionStartWithTag == null) { expressionStartWithTag = String.valueOf(getPrefixChar()) + getTagChar(); diff --git a/src/test/java/com/hubspot/jinjava/EagerTest.java b/src/test/java/com/hubspot/jinjava/EagerTest.java index 22c7c64c2..4516f7793 100644 --- a/src/test/java/com/hubspot/jinjava/EagerTest.java +++ b/src/test/java/com/hubspot/jinjava/EagerTest.java @@ -523,7 +523,6 @@ public void itPrependsSetIfStateChanges() { } @Test - @Ignore public void itHandlesLoopVarAgainstDeferredInLoop() { expectedTemplateInterpreter.assertExpectedOutput( "handles-loop-var-against-deferred-in-loop" @@ -585,7 +584,6 @@ public void itDefersMacroInIf() { } @Test - @Ignore public void itPutsDeferredImportedMacroInOutput() { expectedTemplateInterpreter.assertExpectedOutput( "puts-deferred-imported-macro-in-output" @@ -593,7 +591,6 @@ public void itPutsDeferredImportedMacroInOutput() { } @Test - @Ignore public void itPutsDeferredImportedMacroInOutputSecondPass() { localContext.put("deferred", 1); expectedTemplateInterpreter.assertExpectedOutput( @@ -605,7 +602,6 @@ public void itPutsDeferredImportedMacroInOutputSecondPass() { } @Test - @Ignore public void itPutsDeferredFromedMacroInOutput() { expectedTemplateInterpreter.assertExpectedOutput( "puts-deferred-fromed-macro-in-output" @@ -630,13 +626,11 @@ public void itEagerlyDefersMacroSecondPass() { } @Test - @Ignore public void itLoadsImportedMacroSyntax() { expectedTemplateInterpreter.assertExpectedOutput("loads-imported-macro-syntax"); } @Test - @Ignore public void itDefersCaller() { expectedTemplateInterpreter.assertExpectedOutput("defers-caller"); } @@ -649,7 +643,6 @@ public void itDefersCallerSecondPass() { } @Test - @Ignore public void itDefersMacroInExpression() { expectedTemplateInterpreter.assertExpectedOutput("defers-macro-in-expression"); } @@ -667,7 +660,6 @@ public void itDefersMacroInExpressionSecondPass() { } @Test - @Ignore public void itHandlesDeferredInIfchanged() { expectedTemplateInterpreter.assertExpectedOutput("handles-deferred-in-ifchanged"); } @@ -727,7 +719,6 @@ public void itHandlesNonDeferringCycles() { } @Test - @Ignore public void itHandlesAutoEscape() { localContext.put("myvar", "foo < bar"); expectedTemplateInterpreter.assertExpectedOutput("handles-auto-escape"); @@ -765,7 +756,6 @@ public void itHandlesDeferredImportVars() { } @Test - @Ignore public void itHandlesDeferredImportVarsSecondPass() { localContext.put("deferred", 1); expectedTemplateInterpreter.assertExpectedOutput( diff --git a/src/test/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategyTest.java b/src/test/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategyTest.java new file mode 100644 index 000000000..1dcbb39e1 --- /dev/null +++ b/src/test/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategyTest.java @@ -0,0 +1,86 @@ +package com.hubspot.jinjava.lib.expression; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.hubspot.jinjava.JinjavaConfig; +import com.hubspot.jinjava.interpret.DeferredValue; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.mode.EagerExecutionMode; +import com.hubspot.jinjava.objects.collections.PyList; +import com.hubspot.jinjava.tree.ExpressionNodeTest; +import java.util.ArrayList; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class EagerExpressionStrategyTest extends ExpressionNodeTest { + + @Before + public void eagerSetup() { + interpreter = + new JinjavaInterpreter( + jinjava, + context, + JinjavaConfig.newBuilder().withExecutionMode(new EagerExecutionMode()).build() + ); + JinjavaInterpreter.pushCurrent(interpreter); + context.put("deferred", DeferredValue.instance()); + } + + @After + public void teardown() { + JinjavaInterpreter.popCurrent(); + } + + @Test + public void itPreservesRawTags() { + interpreter = + new JinjavaInterpreter( + jinjava, + context, + JinjavaConfig + .newBuilder() + .withNestedInterpretationEnabled(false) + .withExecutionMode(new EagerExecutionMode()) + .build() + ); + JinjavaInterpreter.pushCurrent(interpreter); + try { + assertExpectedOutput( + "{{ '{{ foo }}' }} {{ '{% something %}' }} {{ 'not needed' }}", + "{% raw %}{{ foo }}{% endraw %} {% raw %}{% something %}{% endraw %} not needed" + ); + } finally { + JinjavaInterpreter.popCurrent(); + } + } + + @Test + public void itPreservesRawTagsNestedInterpretation() { + assertExpectedOutput( + "{{ '{{ 12345 }}' }} {{ '{% print 'bar' %}' }} {{ 'not needed' }}", + "12345 bar not needed" + ); + } + + @Test + public void itPrependsMacro() { + assertExpectedOutput( + "{% macro foo(bar) %} {{ bar }} {% endmacro %}{{ foo(deferred) }}", + "{% macro foo(bar) %} {{ bar }} {% endmacro %}{{ foo(deferred) }}" + ); + } + + @Test + public void itPrependsSet() { + context.put("foo", new PyList(new ArrayList<>())); + assertExpectedOutput( + "{{ foo.append(deferred) }}", + "{% set foo = [] %}{{ foo.append(deferred) }}" + ); + } + + private void assertExpectedOutput(String inputTemplate, String expectedOutput) { + assertThat(interpreter.render(inputTemplate)).isEqualTo(expectedOutput); + } +} diff --git a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecoratorTest.java b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecoratorTest.java index 0eb60cb39..cb56016d0 100644 --- a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecoratorTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecoratorTest.java @@ -18,6 +18,7 @@ import com.hubspot.jinjava.mode.DefaultExecutionMode; import com.hubspot.jinjava.mode.EagerExecutionMode; import com.hubspot.jinjava.mode.PreserveRawExecutionMode; +import com.hubspot.jinjava.objects.collections.PyList; import com.hubspot.jinjava.objects.collections.PyMap; import com.hubspot.jinjava.tree.TagNode; import com.hubspot.jinjava.tree.parse.DefaultTokenScannerSymbols; @@ -104,13 +105,13 @@ public void itExecutesInChildContextAndDefersNewValue() { } @Test - public void itGetsNewlyDeferredFunctionImagesFromGlobal() { + public void itReconstructsMacroFunctionsFromGlobal() { Set deferredWords = new HashSet<>(); deferredWords.add("foo"); String image = "{% macro foo(bar) %}something{% endmacro %}"; MacroFunction mockMacroFunction = getMockMacroFunction(image); context.addGlobalMacro(mockMacroFunction); - String result = EagerTagDecorator.getNewlyDeferredFunctionImages( + String result = EagerTagDecorator.reconstructFromContextBeforeDeferring( deferredWords, interpreter ); @@ -119,14 +120,14 @@ public void itGetsNewlyDeferredFunctionImagesFromGlobal() { } @Test - public void itGetsNewlyDeferredFunctionImagesFromLocal() { + public void itReconstructsMacroFunctionsFromLocal() { Set deferredWords = new HashSet<>(); deferredWords.add("local.foo"); String image = "{% macro foo(bar) %}something{% endmacro %}"; MacroFunction mockMacroFunction = getMockMacroFunction(image); Map localAlias = new PyMap(ImmutableMap.of("foo", mockMacroFunction)); context.put("local", localAlias); - String result = EagerTagDecorator.getNewlyDeferredFunctionImages( + String result = EagerTagDecorator.reconstructFromContextBeforeDeferring( deferredWords, interpreter ); @@ -134,6 +135,48 @@ public void itGetsNewlyDeferredFunctionImagesFromLocal() { assertThat(deferredWords).isEmpty(); } + @Test + public void itReconstructsVariables() { + Set deferredWords = new HashSet<>(); + deferredWords.add("foo.append"); + context.put("foo", new PyList(new ArrayList<>())); + String result = EagerTagDecorator.reconstructFromContextBeforeDeferring( + deferredWords, + interpreter + ); + assertThat(result).isEqualTo("{% set foo = [] %}"); + } + + @Test + public void itDoesntReconstructVariablesInProtectedMode() { + Set deferredWords = new HashSet<>(); + deferredWords.add("foo.append"); + context.put("foo", new PyList(new ArrayList<>())); + context.setProtectedMode(true); + String result = EagerTagDecorator.reconstructFromContextBeforeDeferring( + deferredWords, + interpreter + ); + assertThat(result).isEqualTo(""); + } + + @Test + public void itReconstructsVariablesAndMacroFunctions() { + Set deferredWords = new HashSet<>(); + deferredWords.add("bar.append"); + deferredWords.add("foo"); + String image = "{% macro foo(bar) %}something{% endmacro %}"; + MacroFunction mockMacroFunction = getMockMacroFunction(image); + context.addGlobalMacro(mockMacroFunction); + context.put("bar", new PyList(new ArrayList<>())); + String result = EagerTagDecorator.reconstructFromContextBeforeDeferring( + deferredWords, + interpreter + ); + assertThat(result) + .isEqualTo("{% macro foo(bar) %}something{% endmacro %}{% set bar = [] %}"); + } + @Test public void itBuildsSetTagForDeferredAndRegisters() { Map deferredValuesToSet = ImmutableMap.of("foo", "'bar'");