Skip to content

Commit

Permalink
feat(lexer): add support for multiple front matter variables #16
Browse files Browse the repository at this point in the history 
Added support for multiple front matter variables in the lexer. This includes changes to the lexer's state management and whitespace handling. Also added a new test case to verify the functionality.
  • Loading branch information
@phodal
phodal committed Jun 24, 2024
1 parent cd9d28b commit 9eb9f8d
Show file tree
Hide file tree
Showing 4 changed files with 292 additions and 8 deletions.
46 changes: 38 additions & 8 deletions shirelang/src/main/grammar/ShireLexer.flex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ AND =and
private boolean isCodeStart = false;
private boolean isInsideShireTemplate = false;
private boolean isInsideQueryExpression = false;
private boolean isInsideFrontMatter = false;
private boolean isFMObjectStart = false;
%}

%{
Expand Down Expand Up @@ -193,11 +195,38 @@ AND =and

return COMMAND_PROP;
}


/** @param offset offset from currently matched token start (could be negative) */
private char getCharAtOffset(final int offset) {
final int loc = getTokenStart() + offset;
return 0 <= loc && loc < zzBuffer.length() ? zzBuffer.charAt(loc) : (char) -1;
}

private boolean isAfterEol() {
final char prev = getCharAtOffset(-1);
return prev == (char)-1 || prev == '\n';
}

private IElementType getWhitespaceType() {
if (!isInsideFrontMatter) {
return TokenType.WHITE_SPACE;
}

if (isAfterEol() && !isFMObjectStart) {
isFMObjectStart = true;
yybegin(FRONT_MATTER_VAL_OBJECT);
return INDENT;
} else {
isFMObjectStart = false;
return TokenType.WHITE_SPACE;
}
}
%}

%%
<YYINITIAL> {
"---" { yybegin(FRONT_MATTER_BLOCK); return FRONTMATTER_START; }
"---" { isInsideFrontMatter = true; yybegin(FRONT_MATTER_BLOCK); return FRONTMATTER_START; }
{CODE_CONTENT} { return content(); }
{NEWLINE} { return NEWLINE; }
"[" { yypushback(yylength()); yybegin(COMMENT_BLOCK); }
Expand All @@ -214,16 +243,16 @@ AND =and
{PATTERN_EXPR} { return PATTERN_EXPR; }
":" { yybegin(FRONT_MATTER_VALUE_BLOCK);return COLON; }
"{" { yybegin(QUERY_STATEMENT_BLOCK); return OPEN_BRACE; }
" " { yybegin(FRONT_MATTER_VAL_OBJECT); return INDENT; }
{NEWLINE} { return NEWLINE; }

// end for block
"---" { yybegin(YYINITIAL); return FRONTMATTER_END; }
{WHITE_SPACE} { return getWhitespaceType(); }
{NEWLINE} { return NEWLINE; }
"---" { isInsideFrontMatter = false; yybegin(YYINITIAL); return FRONTMATTER_END; }
[^] { yypushback(yylength()); yybegin(YYINITIAL); }
}

<FRONT_MATTER_VAL_OBJECT> {
{QUOTE_STRING} { return QUOTE_STRING; }
":" { yybegin(FRONT_MATTER_VALUE_BLOCK); return COLON; }
[^] { yypushback(yylength()); yybegin(FRONT_MATTER_BLOCK); }
}

Expand All @@ -237,7 +266,6 @@ AND =and
"[" { return LBRACKET; }
"]" { return RBRACKET; }
"," { return COMMA; }
" " { return TokenType.WHITE_SPACE; }
"!" { return NOT; }
"&&" { return ANDAND; }
"||" { return OROR; }
Expand All @@ -248,10 +276,12 @@ AND =and
"<=" { return LTE; }
">" { return GT; }
">=" { return GTE; }
" " { return TokenType.WHITE_SPACE; }
{WHITE_SPACE} { return getWhitespaceType(); }
// " " { return TokenType.WHITE_SPACE; }
"$" { return VARIABLE_START; }
"(" { return LPAREN; }
")" { return RPAREN; }

[^] { yypushback(yylength()); yybegin(FRONT_MATTER_BLOCK); }
}

Expand Down Expand Up @@ -378,7 +408,7 @@ AND =and
{NUMBER} { return NUMBER; }
{IDENTIFIER} { return IDENTIFIER; }
{QUOTE_STRING} { return QUOTE_STRING; }
{WHITE_SPACE} { return TokenType.WHITE_SPACE; }
{WHITE_SPACE} { return getWhitespaceType(); }
[^] { yypushback(yylength()); if (isInsideShireTemplate) { yybegin(CODE_BLOCK); } else if (isInsideQueryExpression) { yybegin(QUERY_STATEMENT_BLOCK);} else { yybegin(YYINITIAL); } }
}

Expand Down
4 changes: 4 additions & 0 deletions shirelang/src/test/kotlin/com/phodal/shirelang/ShireParsingTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,5 +79,9 @@ class ShireParsingTest : ParsingTestCase("parser", "shire", ShireParserDefinitio
fun testShirePsiQueryExpression() {
doTest(true)
}

fun testMultipleFMVariable() {
doTest(true)
}
}

22 changes: 22 additions & 0 deletions shirelang/src/test/testData/parser/MultipleFMVariable.shire
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
variables:
"extContext": /build\.gradle\.kts/ { cat | grep("org.springframework.boot:spring-boot-starter-jdbc") | print("This project use Spring Framework")}
"testTemplate": /\(.*\).java/ {
case "$1" {
"Controller" { cat(".shire/templates/ControllerTest.java") }
"Service" { cat(".shire/templates/ServiceTest.java") }
default { cat(".shire/templates/DefaultTest.java") }
}
}
"allController": {
from {
PsiClass clazz /* sample */
}
where {
clazz.getAnAnnotation() == "org.springframework.web.bind.annotation.RequestMapping"
}
select {
clazz.id, clazz.name, "code"
}
}
---
228 changes: 228 additions & 0 deletions shirelang/src/test/testData/parser/MultipleFMVariable.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
ShireFile
ShireFrontMatterHeaderImpl(FRONT_MATTER_HEADER)
PsiElement(ShireTokenType.FRONTMATTER_START)('---')
PsiElement(ShireTokenType.NEWLINE)('\n')
ShireFrontMatterEntriesImpl(FRONT_MATTER_ENTRIES)
ShireFrontMatterEntryImpl(FRONT_MATTER_ENTRY)
ShireFrontMatterKeyImpl(FRONT_MATTER_KEY)
ShireFrontMatterIdImpl(FRONT_MATTER_ID)
PsiElement(ShireTokenType.IDENTIFIER)('variables')
PsiElement(ShireTokenType.COLON)(':')
ShireFrontMatterValueImpl(FRONT_MATTER_VALUE)
PsiElement(ShireTokenType.NEWLINE)('\n')
ShireObjectKeyValueImpl(OBJECT_KEY_VALUE)
PsiElement(ShireTokenType.INDENT)(' ')
ShireKeyValueImpl(KEY_VALUE)
ShireFrontMatterEntryImpl(FRONT_MATTER_ENTRY)
ShireFrontMatterKeyImpl(FRONT_MATTER_KEY)
PsiElement(ShireTokenType.QUOTE_STRING)('"extContext"')
PsiElement(ShireTokenType.COLON)(':')
PsiWhiteSpace(' ')
ShirePatternActionImpl(PATTERN_ACTION)
PatternElement(PATTERN)
PsiElement(ShireTokenType.PATTERN_EXPR)('/build\.gradle\.kts/')
PsiWhiteSpace(' ')
ShireActionBlockImpl(ACTION_BLOCK)
PsiElement(ShireTokenType.{)('{')
PsiWhiteSpace(' ')
ShireActionBodyImpl(ACTION_BODY)
ShireActionExprImpl(ACTION_EXPR)
ShireFuncCallImpl(FUNC_CALL)
ShireFuncNameImpl(FUNC_NAME)
PsiElement(ShireTokenType.IDENTIFIER)('cat')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.|)('|')
PsiWhiteSpace(' ')
ShireActionExprImpl(ACTION_EXPR)
ShireFuncCallImpl(FUNC_CALL)
ShireFuncNameImpl(FUNC_NAME)
PsiElement(ShireTokenType.IDENTIFIER)('grep')
PsiElement(ShireTokenType.()('(')
ShirePipelineArgsImpl(PIPELINE_ARGS)
ShirePipelineArgImpl(PIPELINE_ARG)
PsiElement(ShireTokenType.QUOTE_STRING)('"org.springframework.boot:spring-boot-starter-jdbc"')
PsiElement(ShireTokenType.))(')')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.|)('|')
PsiWhiteSpace(' ')
ShireActionExprImpl(ACTION_EXPR)
ShireFuncCallImpl(FUNC_CALL)
ShireFuncNameImpl(FUNC_NAME)
PsiElement(ShireTokenType.IDENTIFIER)('print')
PsiElement(ShireTokenType.()('(')
ShirePipelineArgsImpl(PIPELINE_ARGS)
ShirePipelineArgImpl(PIPELINE_ARG)
PsiElement(ShireTokenType.QUOTE_STRING)('"This project use Spring Framework"')
PsiElement(ShireTokenType.))(')')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireFrontMatterEntryImpl(FRONT_MATTER_ENTRY)
ShireFrontMatterKeyImpl(FRONT_MATTER_KEY)
PsiElement(ShireTokenType.QUOTE_STRING)('"testTemplate"')
PsiElement(ShireTokenType.COLON)(':')
PsiWhiteSpace(' ')
ShirePatternActionImpl(PATTERN_ACTION)
PatternElement(PATTERN)
PsiElement(ShireTokenType.PATTERN_EXPR)('/\(.*\).java/')
PsiWhiteSpace(' ')
ShireActionBlockImpl(ACTION_BLOCK)
PsiElement(ShireTokenType.{)('{')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireActionBodyImpl(ACTION_BODY)
ShireActionExprImpl(ACTION_EXPR)
ShireCaseBodyImpl(CASE_BODY)
PsiElement(ShireTokenType.case)('case')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.QUOTE_STRING)('"$1"')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.{)('{')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireCasePatternActionImpl(CASE_PATTERN_ACTION)
ShireCaseConditionImpl(CASE_CONDITION)
PsiElement(ShireTokenType.QUOTE_STRING)('"Controller"')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.{)('{')
PsiWhiteSpace(' ')
ShireActionExprImpl(ACTION_EXPR)
ShireFuncCallImpl(FUNC_CALL)
ShireFuncNameImpl(FUNC_NAME)
PsiElement(ShireTokenType.IDENTIFIER)('cat')
PsiElement(ShireTokenType.()('(')
ShirePipelineArgsImpl(PIPELINE_ARGS)
ShirePipelineArgImpl(PIPELINE_ARG)
PsiElement(ShireTokenType.QUOTE_STRING)('".shire/templates/ControllerTest.java"')
PsiElement(ShireTokenType.))(')')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireCasePatternActionImpl(CASE_PATTERN_ACTION)
ShireCaseConditionImpl(CASE_CONDITION)
PsiElement(ShireTokenType.QUOTE_STRING)('"Service"')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.{)('{')
PsiWhiteSpace(' ')
ShireActionExprImpl(ACTION_EXPR)
ShireFuncCallImpl(FUNC_CALL)
ShireFuncNameImpl(FUNC_NAME)
PsiElement(ShireTokenType.IDENTIFIER)('cat')
PsiElement(ShireTokenType.()('(')
ShirePipelineArgsImpl(PIPELINE_ARGS)
ShirePipelineArgImpl(PIPELINE_ARG)
PsiElement(ShireTokenType.QUOTE_STRING)('".shire/templates/ServiceTest.java"')
PsiElement(ShireTokenType.))(')')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireCasePatternActionImpl(CASE_PATTERN_ACTION)
ShireCaseConditionImpl(CASE_CONDITION)
PsiElement(ShireTokenType.default)('default')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.{)('{')
PsiWhiteSpace(' ')
ShireActionExprImpl(ACTION_EXPR)
ShireFuncCallImpl(FUNC_CALL)
ShireFuncNameImpl(FUNC_NAME)
PsiElement(ShireTokenType.IDENTIFIER)('cat')
PsiElement(ShireTokenType.()('(')
ShirePipelineArgsImpl(PIPELINE_ARGS)
ShirePipelineArgImpl(PIPELINE_ARG)
PsiElement(ShireTokenType.QUOTE_STRING)('".shire/templates/DefaultTest.java"')
PsiElement(ShireTokenType.))(')')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireFrontMatterEntryImpl(FRONT_MATTER_ENTRY)
ShireFrontMatterKeyImpl(FRONT_MATTER_KEY)
PsiElement(ShireTokenType.QUOTE_STRING)('"allController"')
PsiElement(ShireTokenType.COLON)(':')
PsiWhiteSpace(' ')
ShireQueryStatementImpl(QUERY_STATEMENT)
PsiElement(ShireTokenType.{)('{')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireFromClauseImpl(FROM_CLAUSE)
PsiElement(ShireTokenType.from)('from')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.{)('{')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShirePsiElementDeclImpl(PSI_ELEMENT_DECL)
ShireVariableDeclImpl(VARIABLE_DECL)
ShirePsiTypeImpl(PSI_TYPE)
PsiElement(ShireTokenType.IDENTIFIER)('PsiClass')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.IDENTIFIER)('clazz')
PsiWhiteSpace(' ')
PsiComment(ShireTokenType.BLOCK_COMMENT)('/* sample */')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireWhereClauseImpl(WHERE_CLAUSE)
PsiElement(ShireTokenType.where)('where')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.{)('{')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireEqComparisonExprImpl(EQ_COMPARISON_EXPR)
ShireCallExprImpl(CALL_EXPR)
ShireRefExprImpl(REF_EXPR)
ShireRefExprImpl(REF_EXPR)
PsiElement(ShireTokenType.IDENTIFIER)('clazz')
PsiElement(ShireTokenType..)('.')
PsiElement(ShireTokenType.IDENTIFIER)('getAnAnnotation')
PsiElement(ShireTokenType.()('(')
PsiElement(ShireTokenType.))(')')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.==)('==')
PsiWhiteSpace(' ')
ShireLiteralExprImpl(LITERAL_EXPR)
PsiElement(ShireTokenType.QUOTE_STRING)('"org.springframework.web.bind.annotation.RequestMapping"')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireSelectClauseImpl(SELECT_CLAUSE)
PsiElement(ShireTokenType.select)('select')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.{)('{')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
ShireRefExprImpl(REF_EXPR)
ShireRefExprImpl(REF_EXPR)
PsiElement(ShireTokenType.IDENTIFIER)('clazz')
PsiElement(ShireTokenType..)('.')
PsiElement(ShireTokenType.IDENTIFIER)('id')
PsiElement(ShireTokenType.,)(',')
PsiWhiteSpace(' ')
ShireRefExprImpl(REF_EXPR)
ShireRefExprImpl(REF_EXPR)
PsiElement(ShireTokenType.IDENTIFIER)('clazz')
PsiElement(ShireTokenType..)('.')
PsiElement(ShireTokenType.IDENTIFIER)('name')
PsiElement(ShireTokenType.,)(',')
PsiWhiteSpace(' ')
ShireLiteralExprImpl(LITERAL_EXPR)
PsiElement(ShireTokenType.QUOTE_STRING)('"code"')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiWhiteSpace(' ')
PsiElement(ShireTokenType.})('}')
PsiElement(ShireTokenType.NEWLINE)('\n')
PsiElement(ShireTokenType.FRONTMATTER_END)('---')

0 comments on commit 9eb9f8d

Please sign in to comment.