Switch to a new REPL API

Author: ileasile

build.gradle

@@ -155,7 +155,7 @@ configurations {
 dependencies {
     compile project(":jupyter-lib")
     compile "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
-    compile "org.jetbrains.kotlin:kotlin-scripting-jvm-host-embeddable:$kotlinVersion"
+    compile "org.jetbrains.kotlin:kotlin-scripting-ide-services-impl-embeddable:$kotlinVersion"
     compile "org.jetbrains.kotlin:kotlin-scripting-common:$kotlinVersion"
     compile "org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable:$kotlinVersion"
     compile "org.jetbrains.kotlin:kotlin-compiler-embeddable:$kotlinVersion"

jupyter-lib/src/main/kotlin/jupyter/kotlin/receivers/ConstReceiver.kt

@@ -0,0 +1,3 @@
+package jupyter.kotlin.receivers
+
+class ConstReceiver(val value: Int)

src/main/kotlin/org/jetbrains/kotlin/jupyter/receivers.kt jupyter-lib/src/main/kotlin/jupyter/kotlin/receivers/TypeProviderReceiver.kt

@@ -1,13 +1,11 @@
-package org.jetbrains.kotlin.jupyter
-
-class ConstReceiver(val value: Int)
+package jupyter.kotlin.receivers
 
 class TypeProviderReceiver {
 
     fun generateCode(values: List<Int>): List<String> {
         val properties = (0 until values.size)
                 .map { "val value$it : Int get() = list[$it]" }
-                .joinToLines()
+                .joinToString("\n")
 
         val classDeclaration = """
                 class TypedIntList###(val list: List<Int>): List<Int> by list {

resources/notebook-extension/kernel.js

@@ -153,7 +153,7 @@ define(function(){
             return -1;
         }
 
-        function getTokenBounds(buf, cursor) {
+        function getTokenBounds(buf, cursor, editor) {
             if (cursor > buf.length) {
                 throw new Error("Position " + cursor + " does not exist in code snippet <" + buf + ">");
             }
@@ -174,7 +174,9 @@ define(function(){
                 tokenBeforeCursor: buf.substring(start, cursor),
                 after: buf.substring(end, buf.length),
                 start: start,
-                end: end
+                end: end,
+                posStart: editor.posFromIndex(start),
+                posEnd: editor.posFromIndex(end)
             }
         }
 
@@ -211,7 +213,7 @@ define(function(){
             cursor_pos = utils.js_idx_to_char_idx(cursor_pos, text);
 
             var prevBounds = this.tokenBounds;
-            var bounds = getTokenBounds(text, cursor_pos);
+            var bounds = getTokenBounds(text, cursor_pos, this.editor);
             this.tokenBounds = bounds;
             if (prevBounds && this.raw_result) {
                 if (bounds.before === prevBounds.before &&
@@ -223,6 +225,10 @@ define(function(){
                         if (displayName[0] === '`')
                             displayName = displayName.substring(1, displayName.length - 1);
                         return displayName.startsWith(bounds.tokenBeforeCursor)
+                    }).map((completion) => {
+                        completion.from = bounds.posStart;
+                        completion.to = bounds.posEnd;
+                        return completion;
                     });
 
                     if (newResult.length > 0) {

src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt

@@ -23,7 +23,7 @@ val LocalSettingsPath = Paths.get(System.getProperty("user.home"), ".jupyter_kot
 val GitHubApiHost = "api.github.com"
 val GitHubRepoOwner = "kotlin"
 val GitHubRepoName = "kotlin-jupyter"
-val GitHubBranchName = "demo-jan2020"
+val GitHubBranchName = "psi_completion_with_typing"
 val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName"
 
 val LibraryDescriptorExt = "json"

src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt

@@ -13,6 +13,7 @@ import java.io.PrintStream
 import java.lang.reflect.InvocationTargetException
 import java.util.concurrent.atomic.AtomicLong
 import kotlin.concurrent.timer
+import kotlin.script.experimental.api.ScriptDiagnostic
 
 enum class ResponseState {
     Ok, Error, Abort
@@ -356,17 +357,18 @@ fun JupyterConnection.evalWithIO(config: OutputConfig, srcMessage: Message, body
             forkedOut.flush()
             forkedError.flush()
 
-            val additionalInfo = ex.errorResult.location?.let {
-                val errorMessage = ex.errorResult.message
-                jsonObject("lineStart" to it.line, "colStart" to it.column,
-                        "lineEnd" to it.lineEnd, "colEnd" to it.columnEnd,
+            val firstDiagnostic = ex.firstDiagnostics
+            val additionalInfo = firstDiagnostic?.location?.let {
+                val errorMessage = firstDiagnostic.message
+                jsonObject("lineStart" to it.start.line, "colStart" to it.start.col,
+                        "lineEnd" to (it.end?.line ?: -1), "colEnd" to (it.end?.col ?: -1),
                         "message" to errorMessage,
-                        "path" to it.path)
+                        "path" to firstDiagnostic.sourcePath.orEmpty())
             } ?: jsonObject()
 
             ErrorResponseWithMessage(
                     textResult("Error!"),
-                    ex.errorResult.message,
+                    ex.message,
                     ex.javaClass.canonicalName,
                     ex.message ?: "",
                     ex.stackTrace.map { it.toString() },
@@ -376,8 +378,8 @@ fun JupyterConnection.evalWithIO(config: OutputConfig, srcMessage: Message, body
 
             val stdErr = StringBuilder()
             with(stdErr) {
-                val cause = ex.errorResult.cause
-                if (cause == null) appendln(ex.errorResult.message)
+                val cause = ex.cause
+                if (cause == null) appendln(ex.message)
                 else {
                     when (cause) {
                         is InvocationTargetException -> appendln(cause.targetException.toString())

src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt

@@ -3,41 +3,42 @@ package org.jetbrains.kotlin.jupyter
 import jupyter.kotlin.*
 import jupyter.kotlin.KotlinContext
 import jupyter.kotlin.KotlinReceiver
-import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
-import org.jetbrains.kotlin.cli.common.repl.*
 import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult
 import org.jetbrains.kotlin.jupyter.repl.completion.KotlinCompleter
 import org.jetbrains.kotlin.jupyter.repl.completion.ListErrorsResult
+import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl
 import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater
-import org.jetbrains.kotlin.jupyter.repl.reflect.lines
 import org.jetbrains.kotlin.jupyter.repl.spark.ClassWriter
+import org.jetbrains.kotlin.scripting.ide_services.compiler.KJvmReplCompilerImpl
+import org.jetbrains.kotlin.scripting.ide_services.evaluator.KJvmReplEvaluatorImpl
 import java.io.File
 import java.net.URLClassLoader
 import java.util.*
-import java.util.concurrent.locks.ReentrantReadWriteLock
 import kotlin.script.dependencies.ScriptContents
 import kotlin.script.experimental.api.*
 import kotlin.script.experimental.host.withDefaultsFrom
 import kotlin.script.experimental.jvm.*
-import kotlin.script.experimental.jvmhost.repl.JvmReplCompiler
-import kotlin.script.experimental.jvmhost.repl.JvmReplEvaluator
+import kotlin.script.experimental.util.hasErrors
+import kotlin.script.experimental.util.isIncomplete
+import kotlin.script.experimental.util.renderError
 
 data class EvalResult(val resultValue: Any?)
 
-data class CheckResult(val codeLine: LineId, val isComplete: Boolean = true)
+data class CheckResult(val isComplete: Boolean = true)
 
 open class ReplException(message: String, cause: Throwable? = null) : Exception(message, cause)
 
-class ReplEvalRuntimeException(val errorResult: ReplEvalResult.Error.Runtime) : ReplException(errorResult.message, errorResult.cause)
+class ReplEvalRuntimeException(message: String, cause: Throwable? = null) : ReplException(message, cause)
 
-class ReplCompilerException(val errorResult: ReplCompileResult.Error) : ReplException(errorResult.message) {
-    constructor (checkResult: ReplCheckResult.Error) : this(ReplCompileResult.Error(checkResult.message, checkResult.location))
-    constructor (incompleteResult: ReplCompileResult.Incomplete) : this(ReplCompileResult.Error("Incomplete Code", null))
-    constructor (checkResult: ReplEvalResult.Error.CompileTime) : this(ReplCompileResult.Error(checkResult.message, checkResult.location))
-    constructor (incompleteResult: ReplEvalResult.Incomplete) : this(ReplCompileResult.Error("Incomplete Code", null))
-    constructor (historyMismatchResult: ReplEvalResult.HistoryMismatch) : this(ReplCompileResult.Error("History Mismatch", CompilerMessageLocation.create(null, historyMismatchResult.lineNo, 0, null)))
-    constructor (message: String) : this(ReplCompileResult.Error(message, null))
+class ReplCompilerException(val errorResult: ResultWithDiagnostics.Failure? = null, message: String? = null)
+    : ReplException(message ?: errorResult?.renderError() ?: "") {
+
+    val firstDiagnostics = errorResult?.reports?.firstOrNull {
+        it.severity == ScriptDiagnostic.Severity.ERROR || it.severity == ScriptDiagnostic.Severity.FATAL
+    }
+
+    constructor(message: String): this(null, message)
 }
 
 enum class ExecutedCodeLogging {
@@ -241,25 +242,17 @@ class ReplForJupyterImpl(val scriptClasspath: List<File> = emptyList(),
 
     private var executionCounter = 0
 
-    private val compiler: IDELikeReplCompiler by lazy {
-        JvmReplCompiler(compilerConfiguration)
+    private val compiler: KJvmReplCompilerImpl by lazy {
+        KJvmReplCompilerImpl()
     }
 
-    private val evaluator: ReplEvaluator by lazy {
-        JvmReplEvaluator(evaluatorConfiguration)
+    private val evaluator: KJvmReplEvaluatorImpl by lazy {
+        KJvmReplEvaluatorImpl()
     }
 
-    private val stateLock = ReentrantReadWriteLock()
-
-    private val compilerState = compiler.createState(stateLock)
-
-    private val evaluatorState = evaluator.createState(stateLock)
-
-    private val state = AggregatedReplStageState(compilerState, evaluatorState, stateLock)
-
     private val completer = KotlinCompleter()
 
-    private val contextUpdater = ContextUpdater(state, ctx)
+    private val contextUpdater = ContextUpdater(ctx, evaluator)
 
     private val typeProvidersProcessor: TypeProvidersProcessor = TypeProvidersProcessorImpl(contextUpdater)
 
@@ -270,12 +263,12 @@ class ReplForJupyterImpl(val scriptClasspath: List<File> = emptyList(),
     private val scheduledExecutions = LinkedList<Code>()
 
     override fun checkComplete(executionNumber: Long, code: String): CheckResult {
-        val codeLine = ReplCodeLine(executionNumber.toInt(), 0, code)
-        return when (val result = compiler.check(compilerState, codeLine)) {
-            is ReplCheckResult.Error -> throw ReplCompilerException(result)
-            is ReplCheckResult.Ok -> CheckResult(LineId(codeLine), true)
-            is ReplCheckResult.Incomplete -> CheckResult(LineId(codeLine), false)
-            else -> throw IllegalStateException("Unknown check result type ${result}")
+        val codeLine = SourceCodeImpl(executionNumber.toInt(), code)
+        val result = compiler.analyze(codeLine, 0, compilerConfiguration)
+        return when {
+            result.isIncomplete() -> CheckResult(false)
+            result.hasErrors() -> throw ReplException(result.renderError())
+            else -> CheckResult(true)
         }
     }
 
@@ -329,7 +322,7 @@ class ReplForJupyterImpl(val scriptClasspath: List<File> = emptyList(),
         typeRenderers.putAll(p.typeRenderers.map { it.className to it.code })
     }
 
-    private fun lastReplLine() = state.lines[0]
+    private fun lastReplLine() = evaluator.lastEvaluatedSnippet()?.kInstance
 
     override fun eval(code: String, displayHandler: ((Any) -> Unit)?, jupyterId: Int): EvalResult {
         synchronized(this) {
@@ -412,14 +405,16 @@ class ReplForJupyterImpl(val scriptClasspath: List<File> = emptyList(),
 
     private val completionQueue = LockQueue<CompletionResult, CompletionArgs>()
     override suspend fun complete(code: String, cursor: Int, callback: (CompletionResult) -> Unit) = doWithLock(CompletionArgs(code, cursor, callback), completionQueue, CompletionResult.Empty(code, cursor)) {
-        completer.complete(compiler, compilerState, code, executionCounter++, cursor)
+        //val preprocessed = preprocessCode(code)
+        completer.complete(compiler, compilerConfiguration, code, executionCounter++, cursor)
     }
 
     private val listErrorsQueue = LockQueue<ListErrorsResult, ListErrorsArgs>()
     override suspend fun listErrors(code: String, callback: (ListErrorsResult) -> Unit) = doWithLock(ListErrorsArgs(code, callback), listErrorsQueue, ListErrorsResult(code)) {
-        val codeLine = ReplCodeLine(executionCounter++, 0, code)
-        val errorsList = compiler.listErrors(compilerState, codeLine)
-        ListErrorsResult(code, errorsList)
+        //val preprocessed = preprocessCode(code)
+        val codeLine = SourceCodeImpl(executionCounter++, code)
+        val errorsList = compiler.analyze(codeLine, 0, compilerConfiguration)
+        ListErrorsResult(code, errorsList.valueOrThrow())
     }
 
     private fun <T, Args: LockQueueArgs<T>> doWithLock(args: Args, queue: LockQueue<T, Args>, default: T, action: (Args) -> T) {
@@ -467,29 +462,31 @@ class ReplForJupyterImpl(val scriptClasspath: List<File> = emptyList(),
         if (executedCodeLogging == ExecutedCodeLogging.All)
             println(code)
         val id = executionCounter++
-        val codeLine = ReplCodeLine(id, 0, code)
-        when (val compileResult = compiler.compile(compilerState, codeLine)) {
-            is ReplCompileResult.CompiledClasses -> {
-                classWriter?.writeClasses(compileResult)
-                val scriptArgs = ScriptArgsWithTypes(arrayOf(this), arrayOf(KotlinKernelHost::class))
-                val result = evaluator.eval(evaluatorState, compileResult, scriptArgs)
+        val codeLine = SourceCodeImpl(id, code)
+        when (val compileResultWithDiagnostics = compiler.compile(codeLine, compilerConfiguration)) {
+            is ResultWithDiagnostics.Success -> {
+                val compileResult = compileResultWithDiagnostics.value
+                classWriter?.writeClasses(compileResult())
+                val repl = this
+                val currentEvalConfig = ScriptEvaluationConfiguration(evaluatorConfiguration) {
+                    constructorArgs.invoke(repl as KotlinKernelHost)
+                }
+                val result = evaluator.eval(compileResult, currentEvalConfig).valueOrThrow()
                 contextUpdater.update()
-                return when (result) {
-                    is ReplEvalResult.Error.CompileTime -> throw ReplCompilerException(result)
-                    is ReplEvalResult.Error.Runtime -> throw ReplEvalRuntimeException(result)
-                    is ReplEvalResult.Incomplete -> throw ReplCompilerException(result)
-                    is ReplEvalResult.HistoryMismatch -> throw ReplCompilerException(result)
-                    is ReplEvalResult.UnitResult -> {
+
+                val pureResult = result()
+                return when {
+                    pureResult.isErrorResult -> throw ReplEvalRuntimeException(pureResult.error?.message.orEmpty(), pureResult.error)
+                    pureResult.isUnitResult -> {
                         InternalEvalResult(Unit, id)
                     }
-                    is ReplEvalResult.ValueResult -> {
-                        InternalEvalResult(result.value, id)
+                    pureResult.isValueResult -> {
+                        InternalEvalResult(pureResult.result, id)
                     }
                     else -> throw IllegalStateException("Unknown eval result type ${this}")
                 }
             }
-            is ReplCompileResult.Error -> throw ReplCompilerException(compileResult)
-            is ReplCompileResult.Incomplete -> throw ReplCompilerException(compileResult)
+            is ResultWithDiagnostics.Failure -> throw ReplCompilerException(compileResultWithDiagnostics)
         }
     }
 

src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinCompleter.kt

@@ -3,13 +3,9 @@ package org.jetbrains.kotlin.jupyter.repl.completion
 import com.beust.klaxon.JsonObject
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.jupyter.jsonObject
-import org.jetbrains.kotlin.cli.common.repl.IReplStageState
-import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine
-import org.jetbrains.kotlin.cli.common.repl.IDELikeReplCompiler
-import org.jetbrains.kotlin.utils.CompletionVariant
-import org.jetbrains.kotlin.utils.KotlinReplError
 import java.io.PrintWriter
 import java.io.StringWriter
+import kotlin.script.experimental.api.*
 
 enum class CompletionStatus(private val value: String) {
     OK("ok"),
@@ -32,7 +28,7 @@ abstract class CompletionResult(
     open class Success(
             private val matches: List<String>,
             private val bounds: CompletionTokenBounds,
-            private val metadata: List<CompletionVariant>,
+            private val metadata: List<ReplCompletionVariant>,
             private val text: String,
             private val cursor: Int
     ): CompletionResult(CompletionStatus.OK) {
@@ -93,7 +89,7 @@ abstract class CompletionResult(
     }
 }
 
-data class ListErrorsResult(val code: String, val errors: List<KotlinReplError> = emptyList()) {
+data class ListErrorsResult(val code: String, val errors: Iterable<ReplDiagnosticMessage> = emptyList()) {
     fun toJson(): JsonObject {
         return jsonObject("code" to code,
                 "errors" to errors.map {
@@ -115,15 +111,22 @@ data class ListErrorsResult(val code: String, val errors: List<KotlinReplError>
     }
 }
 
+internal class SourceCodeImpl(number: Int, override val text: String) : SourceCode {
+    override val name: String? = "Line_$number"
+    override val locationId: String? = "location_$number"
+}
+
 class KotlinCompleter {
-    fun complete(compiler: IDELikeReplCompiler, compilerState: IReplStageState<*>, code: String, id: Int, cursor: Int): CompletionResult {
+    fun complete(compiler: ReplCompleter, configuration: ScriptCompilationConfiguration, code: String, id: Int, cursor: Int): CompletionResult {
         return try {
-            val codeLine = ReplCodeLine(id, 0, code)
-            val completionList = compiler.complete(compilerState, codeLine, cursor)
+            val codeLine = SourceCodeImpl(id, code)
+            val completionResult = compiler.complete(codeLine, cursor, configuration)
 
-            val bounds = getTokenBounds(code, cursor)
+            completionResult.valueOrNull()?.toList()?.let { completionList ->
+                val bounds = getTokenBounds(code, cursor)
+                CompletionResult.Success(completionList.map { it.text }, bounds, completionList, code, cursor)
+            } ?: CompletionResult.Empty(code, cursor)
 
-            CompletionResult.Success(completionList.map { it.text }, bounds, completionList, code, cursor)
         } catch (e: Exception) {
             val sw = StringWriter()
             e.printStackTrace(PrintWriter(sw))

src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/ContextUpdater.kt

@@ -3,7 +3,8 @@ package org.jetbrains.kotlin.jupyter.repl.reflect
 import jupyter.kotlin.KotlinContext
 import jupyter.kotlin.KotlinFunctionInfo
 import jupyter.kotlin.KotlinVariableInfo
-import org.jetbrains.kotlin.cli.common.repl.AggregatedReplStageState
+import org.jetbrains.kotlin.jupyter.instances
+import org.jetbrains.kotlin.scripting.ide_services.evaluator.KJvmReplEvaluatorImpl
 import org.slf4j.LoggerFactory
 
 import java.lang.reflect.Field
@@ -15,12 +16,12 @@ import kotlin.reflect.jvm.kotlinProperty
  * ContextUpdater updates current user-defined functions and variables
  * to use in completion and KotlinContext.
  */
-class ContextUpdater(private val state: AggregatedReplStageState<*, *>,
-                     val context: KotlinContext) {
+class ContextUpdater(val context: KotlinContext, private val evaluator: KJvmReplEvaluatorImpl) {
 
     fun update() {
         try {
-            val lines = state.lines
+            val lastSnippet = evaluator.lastEvaluatedSnippet
+            val lines = lastSnippet.instances()
             refreshVariables(lines)
             refreshMethods(lines)
         } catch (e: ReflectiveOperationException) {