Add dymanic parser testing (#17)

* replace handwriteParser to test folder

* Refactor rsmState, extraction of terminal and nonterminal edges from the state is made faster

* Add hand-write "generated" parsers

* Add state id for generation, refactor parsers code

* simple refactoring

* add simple kotlinpoet example

* add simple generated parsers

* minor refactoring:
* replace getRsm method to rsm property
* replace parser exception to parser directory

* done base parser generator

* done dynamic test system for online gll

* refactor test directory structure

* generate and compile parser class

* refactor tests directory

* refactor tests generator

.gitignore

@@ -79,3 +79,4 @@ bin/
 
 ### Mac OS ###
 .DS_Store
+/src/test/resources/gen/parser/

build.gradle.kts

@@ -1,18 +1,23 @@
 plugins {
-  java
-  application
-  kotlin("jvm") version "1.9.20"
+    java
+    application
+    kotlin("jvm") version "1.9.20"
 }
 
 repositories {
-  mavenCentral()
+    mavenCentral()
 }
 
 dependencies {
-  testImplementation(kotlin("test"))
-  testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
-  implementation("org.jetbrains.kotlinx:kotlinx-cli:0.3.5")
-  implementation(kotlin("reflect"))
+    implementation("org.jetbrains.kotlinx:kotlinx-cli:0.3.5")
+    implementation(kotlin("reflect"))
+    // https://mvnrepository.com/artifact/com.squareup/kotlinpoet
+    implementation("com.squareup:kotlinpoet:1.16.0")
+    implementation(kotlin("reflect"))
+
+    testImplementation(kotlin("test"))
+    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+    testImplementation("com.github.tschuchortdev:kotlin-compile-testing:1.5.0")
 }
 
 tasks.test { useJUnitPlatform() }
@@ -21,18 +26,18 @@ kotlin { jvmToolchain(11) }
 application { mainClass.set("org.srcgll.MainKt") }
 
 configure<SourceSetContainer> {
-  named("main") {
-    java.srcDir("src/main/kotlin")
-  }
+    named("main") {
+        java.srcDir("src/main/kotlin")
+    }
 }
 
 tasks.withType<Jar> {
-  dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources"))
-  duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-  manifest { attributes(mapOf("Main-Class" to application.mainClass)) }
-  val sourcesMain = sourceSets.main.get()
-  val contents =
-      configurations.runtimeClasspath.get().map { if (it.isDirectory) it else zipTree(it) } +
-          sourcesMain.output
-  from(contents)
+    dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources"))
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    manifest { attributes(mapOf("Main-Class" to application.mainClass)) }
+    val sourcesMain = sourceSets.main.get()
+    val contents =
+        configurations.runtimeClasspath.get().map { if (it.isDirectory) it else zipTree(it) } +
+                sourcesMain.output
+    from(contents)
 }

src/main/kotlin/org/srcgll/Example.kt

@@ -192,8 +192,8 @@ fun createStackExampleGraph(startVertex: Int): SimpleGraph {
 }
 
 fun main() {
-    val rsmAnBnStartState = AnBn().buildRsm()
-    val rsmStackStartState = Stack().buildRsm()
+    val rsmAnBnStartState = AnBn().rsm
+    val rsmStackStartState = Stack().rsm
     val startVertex = 0
     val inputGraphAnBn = createAnBnExampleGraph(startVertex)
     val inputGraphStack = createStackExampleGraph(startVertex)

src/main/kotlin/org/srcgll/Main.kt

@@ -46,7 +46,7 @@ fun main(args: Array<String>) {
 
 
     val input = File(pathToInput).readText().replace("\n", "").trim()
-    val grammar = JavaGrammar().buildRsm()
+    val grammar = JavaGrammar().rsm
     val inputGraph = RecoveryLinearInput<Int, LinearInputLabel>()
     val lexer = JavaLexer(StringReader(input))
     val gll = Gll.recoveryGll(grammar, inputGraph)

src/main/kotlin/org/srcgll/Utils.kt

@@ -0,0 +1,47 @@
+package org.srcgll
+
+import java.util.*
+
+fun <ElementType> incrementalDfs(
+    startElementL: ElementType,
+    next: (ElementType) -> Iterable<ElementType>
+): HashSet<ElementType> {
+    val used = HashSet<ElementType>()
+    val queue = LinkedList<ElementType>()
+    queue.add(startElementL)
+    while (queue.isNotEmpty()) {
+        val element = queue.remove()
+        used.add(element)
+        for (nextElement in next(element)) {
+            if (!used.contains(nextElement)) {
+                queue.add(nextElement)
+            }
+        }
+    }
+    return used
+}
+
+fun <CollectionType, ElementType> incrementalDfs(
+    startElementL: ElementType,
+    next: (ElementType) -> Iterable<ElementType>,
+    collection: CollectionType,
+    addToCollection: (ElementType, CollectionType) -> Unit
+): CollectionType {
+    val used = HashSet<ElementType>()
+    val queue = LinkedList<ElementType>()
+    queue.add(startElementL)
+    while (queue.isNotEmpty()) {
+        val element = queue.remove()
+        used.add(element)
+        addToCollection(element, collection)
+        for (nextElement in next(element)) {
+            if (!used.contains(nextElement)) {
+                queue.add(nextElement)
+            }
+        }
+    }
+    return collection
+}
+
+
+

src/main/kotlin/org/srcgll/descriptors/DescriptorsStorage.kt

@@ -1,6 +1,6 @@
 package org.srcgll.descriptors
 
-import org.srcgll.exceptions.ParsingException
+import org.srcgll.parser.ParsingException
 
 
 open class DescriptorsStorage<VertexType>{

src/main/kotlin/org/srcgll/grammar/combinator/Grammar.kt

@@ -2,39 +2,54 @@ package org.srcgll.grammar.combinator
 
 import org.srcgll.grammar.combinator.regexp.Nt
 import org.srcgll.grammar.combinator.regexp.Regexp
-import org.srcgll.rsm.PrintableRsmState
+import org.srcgll.incrementalDfs
 import org.srcgll.rsm.RsmState
+import org.srcgll.rsm.symbol.Terminal
+import java.util.HashSet
 
 
 open class Grammar {
     val nonTerms = ArrayList<Nt>()
 
     private lateinit var startNt: Nt
 
+    private var _rsm: RsmState? = null
+    val rsm: RsmState
+        get() {
+            if (_rsm == null) {
+                _rsm = buildRsm()
+            }
+            return _rsm!!
+        }
+
     fun setStart(expr: Regexp) {
         if (expr is Nt) {
             startNt = expr
         } else throw IllegalArgumentException("Only NT object can be start state for Grammar")
     }
 
 
-    fun buildPrintableRsm(): PrintableRsmState {
-        nonTerms.forEach { it.buildPrintableRsmBox() }
-        val startState = startNt.getNonterminal()?.startState
-        //if nonterminal not initialized -- it will be checked in buildRsmBox()
-        return startState as PrintableRsmState
-    }
-
-
     /**
-     * Builds a new Rsm for grammar
-     *
-     * Each call of Rsm building update links to startState in nonterminals!!!
+     * Builds a Rsm for the grammar
      */
-    fun buildRsm(): RsmState {
+    private fun buildRsm(): RsmState {
         nonTerms.forEach { it.buildRsmBox() }
         val startState = startNt.getNonterminal()?.startState
         //if nonterminal not initialized -- it will be checked in buildRsmBox()
         return startState!!
     }
+
+    /**
+     * Get all terminals used in RSM from current state (recursive)
+     */
+    fun getTerminals(): HashSet<Terminal<*>> {
+        return incrementalDfs(
+            rsm,
+            { state: RsmState -> state.outgoingEdges.values.flatten() +
+            state.nonterminalEdges.keys.map{it.startState}},
+            hashSetOf(),
+            { state, set -> set.addAll(state.terminalEdges.keys) }
+        )
+    }
+
 }

src/main/kotlin/org/srcgll/grammar/combinator/regexp/Nt.kt

@@ -1,7 +1,6 @@
 package org.srcgll.grammar.combinator.regexp
 
 import org.srcgll.grammar.combinator.Grammar
-import org.srcgll.rsm.PrintableRsmState
 import org.srcgll.rsm.RsmState
 import org.srcgll.rsm.symbol.Nonterminal
 import kotlin.reflect.KProperty
@@ -20,12 +19,6 @@ open class Nt : DerivedSymbol {
         nonterm.startState.buildRsmBox(rsmDescription)
     }
 
-    fun buildPrintableRsmBox() {
-        val printableState = PrintableRsmState(nonterm.startState)
-        printableState.buildRsmBox(rsmDescription)
-        nonterm.startState = printableState
-    }
-
     override fun getNonterminal(): Nonterminal? {
         return nonterm
     }

src/main/kotlin/org/srcgll/input/LinearInput.kt

@@ -1,5 +1,7 @@
 package org.srcgll.input
 
+import org.srcgll.rsm.symbol.Terminal
+
 open class LinearInput<VertexType, LabelType : ILabel> : IInputGraph<VertexType, LabelType> {
     override val vertices: MutableMap<VertexType, VertexType> = HashMap()
     override val edges: MutableMap<VertexType, MutableList<Edge<VertexType, LabelType>>> = HashMap()
@@ -47,4 +49,23 @@ open class LinearInput<VertexType, LabelType : ILabel> : IInputGraph<VertexType,
 
     override fun isStart(vertex: VertexType) = startVertices.contains(vertex)
     override fun isFinal(vertex: VertexType) = getEdges(vertex).isEmpty()
+
+
+    companion object {
+
+
+        fun buildFromString(input: String): IInputGraph<Int, LinearInputLabel> {
+            var curVertexId = 0
+            val inputGraph = LinearInput<Int, LinearInputLabel>()
+            inputGraph.addVertex(curVertexId)
+            for (x in input) {
+                inputGraph.addEdge(curVertexId, LinearInputLabel(Terminal(x.toString())), ++curVertexId)
+                inputGraph.addVertex(curVertexId)
+            }
+            inputGraph.addStartVertex(0)
+            return inputGraph
+        }
+
+    }
+
 }

src/main/kotlin/org/srcgll/parser/Gll.kt

@@ -46,12 +46,12 @@ class Gll<VertexType, LabelType : ILabel> private constructor(
     }
 
 
-    override fun parse(curDescriptor: Descriptor<VertexType>) {
-        val state = curDescriptor.rsmState
-        val pos = curDescriptor.inputPosition
-        val curSppfNode = curDescriptor.getCurSppfNode(ctx)
+    override fun parse(descriptor: Descriptor<VertexType>) {
+        val state = descriptor.rsmState
+        val pos = descriptor.inputPosition
+        val curSppfNode = descriptor.getCurSppfNode(ctx)
 
-        ctx.descriptors.addToHandled(curDescriptor)
+        ctx.descriptors.addToHandled(descriptor)
 
         val leftExtent = curSppfNode?.leftExtent
         val rightExtent = curSppfNode?.rightExtent
@@ -62,11 +62,11 @@ class Gll<VertexType, LabelType : ILabel> private constructor(
             this::handleTerminalOrEpsilonEdge,
             this::handleNonterminalEdge,
             ctx,
-            curDescriptor,
+            descriptor,
             curSppfNode
         )
 
-        if (state.isFinal) pop(curDescriptor.gssNode, curSppfNode, pos)
+        if (state.isFinal) pop(descriptor.gssNode, curSppfNode, pos)
     }
 
 }

src/main/kotlin/org/srcgll/parser/IGll.kt

@@ -30,7 +30,7 @@ interface IGll<VertexType, LabelType : ILabel> {
         return Pair(ctx.parseResult, ctx.reachabilityPairs)
     }
 
-    fun parse(curDescriptor: Descriptor<VertexType>)
+    fun parse(descriptor: Descriptor<VertexType>)
 
     /**
      *
@@ -166,6 +166,23 @@ interface IGll<VertexType, LabelType : ILabel> {
         }
     }
 
+    fun handleNonterminalEdge(
+        descriptor: Descriptor<VertexType>,
+        nonterminal: Nonterminal,
+        curSppfNode: SppfNode<VertexType>?
+    ) {
+        val targetStates: HashSet<RsmState> = descriptor.rsmState.nonterminalEdges[nonterminal]!!
+        for (target in targetStates) {
+            val newDescriptor = Descriptor(
+                nonterminal.startState,
+                createGssNode(nonterminal, target, descriptor.gssNode, curSppfNode, descriptor.inputPosition),
+                sppfNode = null,
+                descriptor.inputPosition
+            )
+            ctx.addDescriptor(newDescriptor)
+        }
+    }
+
 
 
     fun handleTerminalOrEpsilonEdge(

src/main/kotlin/org/srcgll/exceptions/ParsingException.kt → src/main/kotlin/org/srcgll/parser/ParsingException.kt

@@ -1,4 +1,4 @@
-package org.srcgll.exceptions
+package org.srcgll.parser
 
 class ParsingException(msg: String = "Parsing exception") : Exception(msg) {
 }