diff --git a/include/jinja2cpp/error_info.h b/include/jinja2cpp/error_info.h index 222302bf..a1edf143 100644 --- a/include/jinja2cpp/error_info.h +++ b/include/jinja2cpp/error_info.h @@ -32,6 +32,7 @@ enum class ErrorCode ExpectedToken, //!< Specific token(s) expected. ExtraParams[0] contains the actual token, rest of ExtraParams contain set of expected tokens ExpectedExpression, //!< Expression expected ExpectedEndOfStatement, //!< End of statement expected. ExtraParams[0] contains the expected end of statement tag + ExpectedRawEnd, //!< {% endraw %} expected UnexpectedToken, //!< Unexpected token. ExtraParams[0] contains the invalid token UnexpectedStatement, //!< Unexpected statement. ExtraParams[0] contains the invalid statement tag UnexpectedCommentBegin, //!< Unexpected comment block begin (`{#`) @@ -40,6 +41,8 @@ enum class ErrorCode UnexpectedExprEnd, //!< Unexpected expression block end (`}}`) UnexpectedStmtBegin, //!< Unexpected statement block begin (`{%`) UnexpectedStmtEnd, //!< Unexpected statment block end (`%}`) + UnexpectedRawBegin, //!< Unexpected raw block begin {% raw %} + UnexpectedRawEnd, //!< Unexpected raw block end {% endraw %} }; /*! diff --git a/src/error_info.cpp b/src/error_info.cpp index ede3367f..51822e44 100644 --- a/src/error_info.cpp +++ b/src/error_info.cpp @@ -176,6 +176,9 @@ void RenderErrorInfo(std::basic_string& result, const ErrorInfoTpl format_to(out, UNIVERSAL_STR("Expected end of statement, got: '{}'").GetValue(), extraParams[0]); break; } + case ErrorCode::ExpectedRawEnd: + format_to(out, UNIVERSAL_STR("Expected end of raw block").GetValue()); + break; case ErrorCode::UnexpectedToken: { auto& extraParams = errInfo.GetExtraParams(); @@ -194,6 +197,12 @@ void RenderErrorInfo(std::basic_string& result, const ErrorInfoTpl case ErrorCode::UnexpectedCommentEnd: format_to(out, UNIVERSAL_STR("Unexpected comment end").GetValue()); break; + case ErrorCode::UnexpectedRawBegin: + format_to(out, UNIVERSAL_STR("Unexpected raw block begin").GetValue()); + break; + case ErrorCode::UnexpectedRawEnd: + format_to(out, UNIVERSAL_STR("Unexpected raw block end").GetValue()); + break; case ErrorCode::UnexpectedExprBegin: format_to(out, UNIVERSAL_STR("Unexpected expression block begin").GetValue()); break; diff --git a/src/lexer.h b/src/lexer.h index 78ea48b4..82edd9fc 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -98,6 +98,8 @@ struct Token // Template control CommentBegin, CommentEnd, + RawBegin, + RawEnd, StmtBegin, StmtEnd, ExprBegin, diff --git a/src/template_parser.h b/src/template_parser.h index 82a3baa2..54bcc0ec 100644 --- a/src/template_parser.h +++ b/src/template_parser.h @@ -56,7 +56,7 @@ struct ParserTraits : public ParserTraitsBase<> { static std::regex GetRoughTokenizer() { - return std::regex(R"((\{\{)|(\}\})|(\{%)|(%\})|(\{#)|(#\})|(\n))"); + return std::regex(R"((\{\{)|(\}\})|(\{%[\+\-]?\s+raw\s+[\+\-]?%\})|(\{%[\+\-]?\s+endraw\s+[\+\-]?%\})|(\{%)|(%\})|(\{#)|(#\})|(\n))"); } static std::regex GetKeywords() { @@ -112,7 +112,7 @@ struct ParserTraits : public ParserTraitsBase<> { static std::wregex GetRoughTokenizer() { - return std::wregex(LR"((\{\{)|(\}\})|(\{%)|(%\})|(\{#)|(#\})|(\n))"); + return std::wregex(LR"((\{\{)|(\}\})|(\{%[\+\-]?\s+raw\s+[\+\-]?%\})|(\{%[\+\-]?\s+endraw\s+[\+\-]?%\})|(\{%)|(%\})|(\{#)|(#\})|(\n))"); } static std::wregex GetKeywords() { @@ -289,6 +289,8 @@ class TemplateParser : public LexerHelper RM_Unknown = 0, RM_ExprBegin = 1, RM_ExprEnd, + RM_RawBegin, + RM_RawEnd, RM_StmtBegin, RM_StmtEnd, RM_CommentBegin, @@ -308,7 +310,8 @@ class TemplateParser : public LexerHelper Expression, Statement, Comment, - LineStatement + LineStatement, + RawBlock }; struct TextBlockInfo @@ -352,6 +355,14 @@ class TemplateParser : public LexerHelper } } while (matchBegin != matchEnd); FinishCurrentLine(m_template->size()); + + if ( m_currentBlockInfo.type == TextBlockType::RawBlock) + { + nonstd::expected result = MakeParseError(ErrorCode::ExpectedRawEnd, MakeToken(Token::RawEnd, {m_template->size(), m_template->size() })); + foundErrors.push_back(result.error()); + return nonstd::make_unexpected(std::move(foundErrors)); + } + FinishCurrentBlock(m_template->size()); if (!foundErrors.empty()) @@ -395,6 +406,8 @@ class TemplateParser : public LexerHelper } break; case RM_CommentBegin: + if (m_currentBlockInfo.type == TextBlockType::RawBlock) + break; if (m_currentBlockInfo.type != TextBlockType::RawText) { FinishCurrentLine(match.position() + 2); @@ -407,6 +420,8 @@ class TemplateParser : public LexerHelper break; case RM_CommentEnd: + if (m_currentBlockInfo.type == TextBlockType::RawBlock) + break; if (m_currentBlockInfo.type != TextBlockType::Comment) { FinishCurrentLine(match.position() + 2); @@ -444,6 +459,26 @@ class TemplateParser : public LexerHelper m_currentBlockInfo.range.startOffset = FinishCurrentBlock(matchStart); break; + case RM_RawBegin: + if (m_currentBlockInfo.type == TextBlockType::RawBlock) + break; + else if (m_currentBlockInfo.type != TextBlockType::RawText && m_currentBlockInfo.type != TextBlockType::Comment) + { + FinishCurrentLine(match.position() + match.length()); + return MakeParseError(ErrorCode::UnexpectedRawBegin, MakeToken(Token::RawBegin, {matchStart, matchStart + match.length()})); + } + StartControlBlock(TextBlockType::RawBlock, matchStart); + break; + case RM_RawEnd: + if (m_currentBlockInfo.type == TextBlockType::Comment) + break; + else if (m_currentBlockInfo.type != TextBlockType::RawBlock) + { + FinishCurrentLine(match.position() + match.length()); + return MakeParseError(ErrorCode::UnexpectedRawEnd, MakeToken(Token::RawEnd, {matchStart, matchStart + match.length()})); + } + m_currentBlockInfo.range.startOffset = FinishCurrentBlock(matchStart); + break; } return nonstd::expected(); @@ -453,7 +488,7 @@ class TemplateParser : public LexerHelper { size_t startOffset = matchStart + 2; size_t endOffset = matchStart; - if (m_currentBlockInfo.type != TextBlockType::RawText) + if (m_currentBlockInfo.type != TextBlockType::RawText || m_currentBlockInfo.type == TextBlockType::RawBlock ) return; else endOffset = StripBlockLeft(m_currentBlockInfo, startOffset, endOffset); @@ -465,8 +500,53 @@ class TemplateParser : public LexerHelper (*m_template)[startOffset] == '-') ++ startOffset; } - m_currentBlockInfo.range.startOffset = startOffset; + m_currentBlockInfo.type = blockType; + + if (blockType==TextBlockType::RawBlock) + startOffset = StripBlockRight(m_currentBlockInfo, matchStart); + + m_currentBlockInfo.range.startOffset = startOffset; + } + + size_t StripBlockRight(TextBlockInfo& currentBlockInfo, size_t position) + { + bool doTrim = m_settings.trimBlocks && (m_currentBlockInfo.type == TextBlockType::Statement || m_currentBlockInfo.type == TextBlockType::RawBlock); + + if (m_currentBlockInfo.type == TextBlockType::RawBlock) + { + position+=2; + for(; position < m_template->size(); ++ position) + { + if ('%' == (*m_template)[position]) + break; + } + } + + size_t newPos = position + 2; + + if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0) + { + auto ctrlChar = (*m_template)[position - 1]; + doTrim = ctrlChar == '-' ? true : (ctrlChar == '+' ? false : doTrim); + } + + if (doTrim) + { + auto locale = std::locale(); + for (;newPos < m_template->size(); ++ newPos) + { + auto ch = (*m_template)[newPos]; + if (ch == '\n') + { + ++ newPos; + break; + } + if (!std::isspace(ch, locale)) + break; + } + } + return newPos; } size_t StripBlockLeft(TextBlockInfo& currentBlockInfo, size_t ctrlCharPos, size_t endOffset) @@ -483,7 +563,7 @@ class TemplateParser : public LexerHelper doStrip |= doTotalStrip; } - if (!doStrip || currentBlockInfo.type != TextBlockType::RawText) + if (!doStrip || (currentBlockInfo.type != TextBlockType::RawText && currentBlockInfo.type != TextBlockType::RawBlock)) return endOffset; auto locale = std::locale(); @@ -512,6 +592,7 @@ class TemplateParser : public LexerHelper switch (block.type) { + case TextBlockType::RawBlock: case TextBlockType::RawText: { if (block.range.size() == 0) @@ -648,32 +729,25 @@ class TemplateParser : public LexerHelper size_t FinishCurrentBlock(size_t position) { - bool doTrim = m_settings.trimBlocks && m_currentBlockInfo.type == TextBlockType::Statement; - size_t newPos = position + 2; + size_t newPos; - if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0) + if (m_currentBlockInfo.type == TextBlockType::RawBlock) { - auto ctrlChar = (*m_template)[position - 1]; - doTrim = ctrlChar == '-' ? true : (ctrlChar == '+' ? false : doTrim); - if (ctrlChar == '+' || ctrlChar == '-') - -- position; + size_t currentPosition = position; + position = StripBlockLeft(m_currentBlockInfo, currentPosition+2, currentPosition); + newPos = StripBlockRight(m_currentBlockInfo, currentPosition); } - - if (doTrim) + else { - auto locale = std::locale(); - for (;newPos < m_template->size(); ++ newPos) + newPos = StripBlockRight(m_currentBlockInfo, position); + if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0) { - auto ch = (*m_template)[newPos]; - if (ch == '\n') - { - ++ newPos; - break; - } - if (!std::isspace(ch, locale)) - break; + auto ctrlChar = (*m_template)[position - 1]; + if (ctrlChar == '+' || ctrlChar == '-') + -- position; } } + m_currentBlockInfo.range.endOffset = position; m_textBlocks.push_back(m_currentBlockInfo); m_currentBlockInfo.type = TextBlockType::RawText; @@ -928,6 +1002,8 @@ std::unordered_map ParserTraitsBase::s_tokens = { {Token::From, UNIVERSAL_STR("form")}, {Token::As, UNIVERSAL_STR("as")}, {Token::Do, UNIVERSAL_STR("do")}, + {Token::RawBegin, UNIVERSAL_STR("{% raw %}")}, + {Token::RawEnd, UNIVERSAL_STR("{% endraw %}")}, {Token::CommentBegin, UNIVERSAL_STR("{#")}, {Token::CommentEnd, UNIVERSAL_STR("#}")}, {Token::StmtBegin, UNIVERSAL_STR("{%")}, diff --git a/test/errors_test.cpp b/test/errors_test.cpp index 0d01fb7d..ab592748 100644 --- a/test/errors_test.cpp +++ b/test/errors_test.cpp @@ -485,7 +485,13 @@ INSTANTIATE_TEST_CASE_P(StatementsTest_2, ErrorsGenericTest, ::testing::Values( InputOutputPair{"{% if a == 42 %}{% endwith %}", "noname.j2tpl:1:20: error: Unexpected statement: 'endwith'\n{% if a == 42 %}{% endwith %}\n ---^-------"}, InputOutputPair{"{{}}", - "noname.j2tpl:1:3: error: Unexpected token: '<>'\n{{}}\n--^-------"} + "noname.j2tpl:1:3: error: Unexpected token: '<>'\n{{}}\n--^-------"}, + InputOutputPair{"{% raw %}{% raw %}{{ x }{% endraw %}{% endraw %}", + "noname.j2tpl:1:37: error: Unexpected raw block end\n{% raw %}{% raw %}{{ x }{% endraw %}{% endraw %}\n ---^-------"}, + InputOutputPair{"{% raw %}", + "noname.j2tpl:1:10: error: Expected end of raw block\n{% raw %}\n ---^-------"}, + InputOutputPair{"{{ 2 + 3 + {% raw %} }}", + "noname.j2tpl:1:12: error: Unexpected raw block begin\n{{ 2 + 3 + {% raw %}\n ---^-------"} )); INSTANTIATE_TEST_CASE_P(ExtensionStatementsTest, ErrorsGenericExtensionsTest, ::testing::Values( diff --git a/test/statements_tets.cpp b/test/statements_tets.cpp index f5e14ce6..85a5f72c 100644 --- a/test/statements_tets.cpp +++ b/test/statements_tets.cpp @@ -287,4 +287,163 @@ TEST(SetBlockStatement, MoreVarsFiltered) const auto result = tpl.RenderAsString({}).value(); EXPECT_STREQ("\n|9+8+7+6+5+4+3+2+1+0|\n|9+8+7+6+5+4+3+2+1+0|\n|9+8+7+6+5+4+3+2+1+0|\n", result.c_str()); -} \ No newline at end of file +} + +using RawTest = BasicTemplateRenderer; + +TEST(RawTest, General) +{ + const std::string source = R"( +{% raw %} + This is a raw text {{ 2 + 2 }} +{% endraw %} +)"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("\n\n This is a raw text {{ 2 + 2 }}\n", result.c_str()); +} + +TEST(RawTest, KeywordsInside) +{ + const std::string source = R"( +{% raw %} +
    + {% for item in seq %} +
  • {{ item }}
    • + {% endfor %} +
{% endraw %} +)"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("\n\n
    \n {% for item in seq %}\n
  • {{ item }}
    • \n {% endfor %}\n
", result.c_str()); +} + +TEST(RawTest, BrokenExpression) +{ + const std::string source = R"({% raw %}{{ x }{% endraw %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("{{ x }", result.c_str()); +} + +TEST(RawTest, BrokenTag) +{ + const std::string source = R"({% raw %}{% if im_broken }still work{% endraw %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("{% if im_broken }still work", result.c_str()); +} + +TEST(RawTest, ExtraSpaces) +{ + const std::string source = R"({% raw %}abc{% endraw %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("abc", result.c_str()); +} + +TEST(RawTest, ExtraSpaces2) +{ + const std::string source = R"({% raw %}abc{% endraw %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("abc", result.c_str()); +} + +TEST(RawTest, TrimPostRaw) +{ + const std::string source = R"({% raw -%} abc{% endraw %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("abc", result.c_str()); +} + +TEST(RawTest, TrimRawEndRaw) +{ + const std::string source = R"({% raw -%} abc {%- endraw %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("abc", result.c_str()); +} + +TEST(RawTest, TrimPostEndRaw) +{ + const std::string source = R"({% raw %}abc{% endraw -%} defg)"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("abcdefg", result.c_str()); +} + +TEST(RawTest, TrimBeforeEndRaw) +{ + const std::string source = R"({% raw %} abc {%- endraw %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ(" abc", result.c_str()); +} + +TEST(RawTest, TrimBeforeRaw) +{ + const std::string source = R"( {%- raw %} abc {% endraw %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ(" abc ", result.c_str()); +} + +TEST(RawTest, ForRaw) +{ + const std::string source = R"({% for i in (0, 1, 2) -%} + {%- raw %}{{ x }} {% endraw %} + {%- endfor %})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("{{ x }} {{ x }} {{ x }} ", result.c_str()); +} + +TEST(RawTest, CommentRaw) +{ + const std::string source = R"({# {% raw %} {% endraw %} #})"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + const auto result = tpl.RenderAsString({}).value(); + std::cout << result << std::endl; + EXPECT_STREQ("", result.c_str()); +}