Skip to content

Commit

Permalink
Ensure tokens generated by Lexer are continous, fixes gshakhn#55
Browse files Browse the repository at this point in the history
  • Loading branch information
@fhoeben
fhoeben committed Aug 17, 2020
1 parent aadb933 commit 3e07a4b
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 19 deletions.
31 changes: 26 additions & 5 deletions src/main/scala/fitnesse/idea/lexer/FitnesseLexer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class FitnesseLexer extends LexerBase {
symbolList = fetchNextSymbols()

// if TABLE_START, ROW_START -> reduce to TABLE_START, fix offsets
// if COLLAPSIBLE_START, title -> fix offsets
// drop all CELL_START | ROW_START -> advance
// CELL_END, ROW_END -> advance to ROW_END
// ROW_END, TABLE_END -> advance to TABLE_END
Expand All @@ -61,6 +62,12 @@ class FitnesseLexer extends LexerBase {
case table: Table =>
// Fetch new simples to ensure the table contents are properly parsed.
symbolList = LexerSymbol(FitnesseTokenType.TABLE_START, symbol.getStartOffset, symbol.getChildren.get(0).getStartOffset) :: fetchNextSymbols()
case _: Collapsible =>
// Fetch new simples to ensure the collapsible contents are properly parsed.
val title :: tail = fetchNextSymbols()
val titleEnd = title.getChildren.last.getEndOffset
val startAfter = previousEnd(symbol, titleEnd, title.getEndOffset)
symbolList = LexerSymbol(FitnesseTokenType.COLLAPSIBLE_START, symbol.getStartOffset, title.getStartOffset) :: title :: LexerSymbol(FitnesseTokenType.LINE_TERMINATOR, startAfter, startAfter + 1) :: tail
case _ =>
}
case _ =>
Expand Down Expand Up @@ -139,13 +146,20 @@ object FitnesseLexer {
case class LexerSymbol(elementType: IElementType, start: Int, end: Int) extends Symbol(new SymbolType(elementType.toString), "", start, end)

def terminatorFor(symbol: Symbol): Option[Symbol] = {
val startOffset = if (lastChild(symbol).getEndOffset == symbol.getEndOffset) lastChild(symbol).getStartOffset else lastChild(symbol).getEndOffset
val lastNestedChild : Symbol = lastChild(symbol)
val symbolEndOffset = symbol.getEndOffset
val startOffset = if (lastNestedChild.getEndOffset == symbolEndOffset) lastNestedChild.getStartOffset else lastNestedChild.getEndOffset
val previousEndOffset = previousEnd(symbol, startOffset, symbolEndOffset)

symbol.getType match {
case _ : Table => Some(LexerSymbol(FitnesseTokenType.TABLE_END, startOffset, symbol.getEndOffset))
case _ : Collapsible => Some(LexerSymbol(FitnesseTokenType.COLLAPSIBLE_END, lastChild(symbol).getEndOffset, symbol.getEndOffset))
case s if s eq Table.tableRow => Some(LexerSymbol(FitnesseTokenType.ROW_END, startOffset, symbol.getEndOffset))
case s if s eq Table.tableCell => Some(LexerSymbol(FitnesseTokenType.CELL_END, startOffset, symbol.getEndOffset))
case _ : Table => Some(LexerSymbol(FitnesseTokenType.TABLE_END, previousEndOffset, symbolEndOffset))
case _ : Collapsible => {
val childPreviousEndOffset = previousEnd(symbol.getChildren.last, startOffset, symbol.getChildren.last.getEndOffset)
val collEndStart = if (symbolEndOffset - childPreviousEndOffset > 1) childPreviousEndOffset else lastNestedChild.getEndOffset
Some(LexerSymbol(FitnesseTokenType.COLLAPSIBLE_END, collEndStart, symbolEndOffset))
}
case s if s eq Table.tableRow => Some(LexerSymbol(FitnesseTokenType.ROW_END, previousEndOffset, symbolEndOffset))
case s if s eq Table.tableCell => Some(LexerSymbol(FitnesseTokenType.CELL_END, previousEndOffset, symbolEndOffset))
case _ => None
}
}
Expand All @@ -154,6 +168,13 @@ object FitnesseLexer {
val children = symbol.getChildren
if (children.isEmpty || children.exists(_.getStartOffset == -1)) symbol else lastChild(children.last)
}

private def previousEnd(symbol: Symbol, previousMax: Int, endOffset: Int): Int = {
val children = symbol.getChildren
val symbolEnd = symbol.getEndOffset
val currentEnd = if ((symbolEnd < endOffset) && (symbolEnd > previousMax)) symbolEnd else previousMax
if (children.isEmpty || children.exists(_.getStartOffset == -1)) currentEnd else previousEnd(children.last, currentEnd, endOffset)
}
}

class LexerParsingPage extends ParsingPage(new LexerSourcePage) {
Expand Down
73 changes: 63 additions & 10 deletions src/test/scala/fitnesse/idea/lexer/MiscLexerSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,9 @@ class MiscLexerSuite extends LexerSuite {
test("Collapsible section") {
assertResult(
List(
(FitnesseTokenType.COLLAPSIBLE_START, "!* abc\ndef\n*!"),
(FitnesseTokenType.COLLAPSIBLE_START, "!* "),
(FitnesseTokenType.WORD, "abc"),
(FitnesseTokenType.LINE_TERMINATOR, "\n"),
(FitnesseTokenType.WORD, "def"),
(FitnesseTokenType.LINE_TERMINATOR, "\n"),
(FitnesseTokenType.COLLAPSIBLE_END, "*!"),
Expand All @@ -119,19 +120,71 @@ class MiscLexerSuite extends LexerSuite {
}
}

test("Collapsed section") {
assertResult(
List(
(FitnesseTokenType.COLLAPSIBLE_START, 0, 7, "!****< "),
(FitnesseTokenType.WORD, 7, 10, "bbc"),
(FitnesseTokenType.LINE_TERMINATOR, 10, 11, "\n"),
(FitnesseTokenType.WORD, 11, 14, "def"),
(FitnesseTokenType.LINE_TERMINATOR, 14, 15, "\n"),
(FitnesseTokenType.COLLAPSIBLE_END, 15, 17, "*!"),
(FitnesseTokenType.WHITE_SPACE, 17, 18, " "),
(FitnesseTokenType.WORD, 18, 22, "word")
)) {
lexWithOffset("!****< bbc\ndef\n*! word")
}
}

test("Collapsible section multi word title") {
assertResult(
List(
(FitnesseTokenType.COLLAPSIBLE_START, 0, 4, "!*> "),
(FitnesseTokenType.WORD, 4, 9, "Extra"),
(FitnesseTokenType.WHITE_SPACE, 9, 10, " "),
(FitnesseTokenType.WORD, 10, 16, "Import"),
(FitnesseTokenType.LINE_TERMINATOR, 16, 17, "\n"),
(FitnesseTokenType.LINE_TERMINATOR, 17, 18, "\n"),
(FitnesseTokenType.WORD, 18, 20, "Hi"),
(FitnesseTokenType.LINE_TERMINATOR, 20, 21, "\n"),
(FitnesseTokenType.COLLAPSIBLE_END, 21, 24, "*!\n"),
(FitnesseTokenType.LINE_TERMINATOR, 24, 25, "\n")
)) {
lexWithOffset("!*> Extra Import\n\nHi\n*!\n\n")
}
}

test("Collapsible section containing a table") {
assertResult(
List(
(FitnesseTokenType.COLLAPSIBLE_START, "!* title\n|abc|\n|xyz|\n*!"),
(FitnesseTokenType.WORD, "title"),
(FitnesseTokenType.TABLE_START, "|"),
(FitnesseTokenType.WORD, "abc"),
(FitnesseTokenType.ROW_END, "|\n|"),
(FitnesseTokenType.WORD, "xyz"),
(FitnesseTokenType.TABLE_END, "|\n"),
(FitnesseTokenType.COLLAPSIBLE_END, "|\n*!")
(FitnesseTokenType.COLLAPSIBLE_START, 0, 3, "!* "),
(FitnesseTokenType.WORD, 3, 8, "title"),
(FitnesseTokenType.LINE_TERMINATOR, 8, 9, "\n"),
(FitnesseTokenType.TABLE_START, 9, 10, "|"),
(FitnesseTokenType.WORD, 10, 13, "abc"),
(FitnesseTokenType.ROW_END, 13, 16, "|\n|"),
(FitnesseTokenType.WORD, 16, 19, "xyz"),
(FitnesseTokenType.TABLE_END, 19, 21, "|\n"),
(FitnesseTokenType.COLLAPSIBLE_END, 21, 23, "*!")
)) {
lexWithOffset("!* title\n|abc|\n|xyz|\n*!")
}
}

test("Collapsible section containing a table followed by newline") {
assertResult(
List(
(FitnesseTokenType.COLLAPSIBLE_START, 0, 3, "!* "),
(FitnesseTokenType.WORD, 3, 8, "title"),
(FitnesseTokenType.LINE_TERMINATOR, 8, 9, "\n"),
(FitnesseTokenType.TABLE_START, 9, 10, "|"),
(FitnesseTokenType.WORD, 10, 13, "abc"),
(FitnesseTokenType.ROW_END, 13, 16, "|\n|"),
(FitnesseTokenType.WORD, 16, 19, "xyz"),
(FitnesseTokenType.TABLE_END, 19, 21, "|\n"),
(FitnesseTokenType.COLLAPSIBLE_END, 21, 24, "*!\n")
)) {
lex("!* title\n|abc|\n|xyz|\n*!")
lexWithOffset("!* title\n|abc|\n|xyz|\n*!\n")
}
}

Expand Down
49 changes: 48 additions & 1 deletion src/test/scala/fitnesse/idea/lexer/TableLexerSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,4 +315,51 @@ class TableLexerSuite extends LexerSuite {
lexWithOffset("!||\n||")
}
}
}

test("Formatted cells should not make table offsets overlap") {
assertResult(
List(
(FitnesseTokenType.TABLE_START, 0, 1, "|"),
(FitnesseTokenType.BOLD, 1, 8, "'''E'''"),
(FitnesseTokenType.CELL_END, 8, 9, "|"),
(FitnesseTokenType.ITALIC, 9, 14, "''A''"),
(FitnesseTokenType.TABLE_END, 14, 15, "|")
)) {
lexWithOffset("|'''E'''|''A''|")
}
}

test("Formatted cells should not make table offsets overlap with multiple rows") {
assertResult(
List(
(FitnesseTokenType.TABLE_START, 0, 1, "|"),
(FitnesseTokenType.BOLD, 1, 8, "'''E'''"),
(FitnesseTokenType.CELL_END, 8, 9, "|"),
(FitnesseTokenType.ITALIC, 9, 14, "''A''"),
(FitnesseTokenType.ROW_END, 14, 17, "|\n|"),
(FitnesseTokenType.BOLD, 17, 24, "'''E'''"),
(FitnesseTokenType.CELL_END, 24, 25, "|"),
(FitnesseTokenType.ITALIC, 25, 30, "''A''"),
(FitnesseTokenType.TABLE_END, 30, 31, "|")
)) {
lexWithOffset("|'''E'''|''A''|\n|'''E'''|''A''|")
}
}

test("Formatted cells should not make table offsets overlap with multiple rows and text after") {
assertResult(
List(
(FitnesseTokenType.TABLE_START, 0, 1, "|"),
(FitnesseTokenType.BOLD, 1, 8, "'''E'''"),
(FitnesseTokenType.CELL_END, 8, 9, "|"),
(FitnesseTokenType.ITALIC, 9, 14, "''A''"),
(FitnesseTokenType.ROW_END, 14, 17, "|\n|"),
(FitnesseTokenType.BOLD, 17, 24, "'''E'''"),
(FitnesseTokenType.CELL_END, 24, 25, "|"),
(FitnesseTokenType.ITALIC, 25, 30, "''A''"),
(FitnesseTokenType.TABLE_END, 30, 32, "|\n")
)) {
lexWithOffset("|'''E'''|''A''|\n|'''E'''|''A''|\n")
}
}
}
8 changes: 5 additions & 3 deletions src/test/scala/fitnesse/idea/parser/MiscParserSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,16 +49,18 @@ class MiscParserSuite extends ParserSuite {
Node(FitnesseElementType.FILE, List(
Node(FitnesseElementType.COLLAPSIBLE, List(
Leaf(FitnesseTokenType.COLLAPSIBLE_START, "!* "),
Leaf(FitnesseTokenType.WORD, "title\n"),
Leaf(FitnesseTokenType.WORD, "title"),
Leaf(FitnesseTokenType.WHITE_SPACE, "\n"),
Node(TableElementType.DECISION_TABLE, List(
Leaf(FitnesseTokenType.TABLE_START,"|"),
Node(FitnesseElementType.ROW, List(
Node(FitnesseElementType.FIXTURE_CLASS, List(
Leaf(FitnesseTokenType.WORD, "abc")
))
))
)),
Leaf(FitnesseTokenType.TABLE_END,"|\n")
)),
Leaf(FitnesseTokenType.COLLAPSIBLE_END, "|\n*!\n")
Leaf(FitnesseTokenType.COLLAPSIBLE_END, "*!\n")
))
))
) {
Expand Down

0 comments on commit 3e07a4b

Please sign in to comment.