Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize kernel performance #77

Merged
merged 6 commits into from
Jun 25, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions .run/Install for Java 11.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Install for Java 11" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="-PjvmTarget=11" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value="install" />
</list>
</option>
<option name="vmOptions" value="" />
</ExternalSystemSettings>
<GradleScriptDebugEnabled>true</GradleScriptDebugEnabled>
<method v="2" />
</configuration>
</component>
21 changes: 21 additions & 0 deletions .run/Install for Java 8.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Install for Java 8" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value="install" />
</list>
</option>
<option name="vmOptions" value="" />
</ExternalSystemSettings>
<GradleScriptDebugEnabled>true</GradleScriptDebugEnabled>
<method v="2" />
</configuration>
</component>
21 changes: 21 additions & 0 deletions .run/Test for Java 11.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Test for Java 11 (set Gradle JVM to v.11)" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="-PjvmTarget=11" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value="test" />
</list>
</option>
<option name="vmOptions" value="" />
</ExternalSystemSettings>
<GradleScriptDebugEnabled>true</GradleScriptDebugEnabled>
<method v="2" />
</configuration>
</component>
21 changes: 21 additions & 0 deletions .run/Test for Java 8.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Test for Java 8" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value="test" />
</list>
</option>
<option name="vmOptions" value="" />
</ExternalSystemSettings>
<GradleScriptDebugEnabled>true</GradleScriptDebugEnabled>
<method v="2" />
</configuration>
</component>
12 changes: 5 additions & 7 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -156,12 +156,6 @@ allprojects {
}

repositories {
jcenter()
mavenLocal()
mavenCentral()
// only when using Kotlin EAP releases ...
maven { url = uri("https://dl.bintray.com/kotlin/kotlin-eap") }
maven { url = uri("https://dl.bintray.com/kotlin/kotlin-dev") }
maven { url = uri("https://kotlin.bintray.com/kotlin-dependencies") }
}
}
Expand All @@ -175,7 +169,7 @@ dependencies {

implementation(project(":jupyter-lib"))
implementation(kotlin("reflect"))
implementation(kotlin("scripting-ide-services-embeddable") as String) { isTransitive = false }
implementation(kotlin("scripting-ide-services") as String) { isTransitive = false }
implementation(kotlin("scripting-common"))
implementation(kotlin("scripting-compiler-embeddable"))
implementation(kotlin("compiler-embeddable"))
Expand All @@ -185,6 +179,8 @@ dependencies {
implementation(kotlin("scripting-dependencies-maven"))
implementation(kotlin("main-kts"))

compileOnly(kotlin("scripting-compiler-impl"))

implementation("org.apache.maven:maven-core:3.0.3")
implementation("org.slf4j:slf4j-api:1.7.25")
implementation("khttp:khttp:1.0.0")
Expand Down Expand Up @@ -224,6 +220,8 @@ with(ProjectWithOptionsImpl(project, TaskOptions())) {

inputs.property("version", version)
inputs.property("currentBranch", getCurrentBranch())
inputs.property("jvmTargetForSnippets",
rootProject.findProperty("jvmTargetForSnippets") ?: "1.8")
inputs.file(librariesPropertiesPath)

outputs.dir(outputDir)
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/src/main/kotlin/distribution.kt
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ abstract class PipInstallReq : Exec() {
@get:InputFile
var requirementsFile: Path? = null
set(value) {
commandLine("pip", "install", "-r", value)
commandLine("python", "-m", "pip", "install", "-r", value)
field = value
}
}
7 changes: 5 additions & 2 deletions buildSrc/src/main/kotlin/versionDetection.kt
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,11 @@ fun Project.detectVersion(baseVersion: String, artifactsDir: Path, versionFileNa
val isOnProtectedBranch by extra(isProtectedBranch())
val buildCounterStr = rootProject.findProperty("build.counter") as String? ?: "100500"
val buildNumber = rootProject.findProperty("build.number") as String? ?: ""
val devCounter = rootProject.findProperty("build.devCounter") as String? ?: "1"
val devAddition = if(isOnProtectedBranch) "" else ".dev$devCounter"

val devCounterOrNull = rootProject.findProperty("build.devCounter") as String?
val devCounter = devCounterOrNull ?: "1"
val devAddition = if(isOnProtectedBranch && devCounterOrNull == null) "" else ".dev$devCounter"

val defaultBuildNumber = "$baseVersion.$buildCounterStr$devAddition"
val buildNumberRegex = """\d+(\.\d+){3}(\.dev\d+)?"""

Expand Down
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# kotlinVersion=1.4.255-SNAPSHOT
kotlinVersion=1.4.0-dev-7916
kotlinVersion=1.4.20-dev-1121
kotlinLanguageLevel=1.4
jvmTarget=1.8

Expand Down
36 changes: 5 additions & 31 deletions jupyter-lib/src/main/kotlin/jupyter/kotlin/context.kt
Original file line number Diff line number Diff line change
Expand Up @@ -10,41 +10,15 @@ import kotlin.reflect.KProperty
* It can be accessed inside REPL by name `kc`, e.g. kc.showVars()
*/
class KotlinContext(val vars: HashMap<String, KotlinVariableInfo> = HashMap(),
val functions: HashMap<String, KotlinFunctionInfo> = HashMap()) {

fun getVarsList(): List<KotlinVariableInfo> {
return ArrayList(vars.values)
}

fun getFunctionsList(): List<KotlinFunctionInfo> {
return ArrayList(functions.values)
}
}
val functions: HashMap<String, KotlinFunctionInfo> = HashMap())



/**
* The implicit receiver for lines in Kotlin REPL.
* It is passed to the script as an implicit receiver, identical to:
* with (context) {
* ...
* }
*
* KotlinReceiver can be inherited from and passed to REPL building properties,
* so other variables and functions can be accessed inside REPL.
* By default, it only has KotlinContext.
* Inherited KotlinReceivers should be in separate java file, they can't be inner or nested.
*/
class KotlinReceiver(val kc: KotlinContext)
private fun functionSignature(function: KFunction<*>)
= function.toString().replace("Line_\\d+\\.".toRegex(), "")

fun functionSignature(function: KFunction<*>): String {
return function.toString().replace("Line_\\d+\\.".toRegex(), "")
}

fun shortenType(name: String): String {
return name.replace("(\\b[_a-zA-Z$][_a-zA-Z0-9$]*\\b\\.)+".toRegex(), "")
// kotlin.collections.List<kotlin.Int> -> List<Int>
}
private fun shortenType(name: String)
= name.replace("(\\b[_a-zA-Z$][_a-zA-Z0-9$]*\\b\\.)+".toRegex(), "")

class KotlinFunctionInfo(val function: KFunction<*>, val line: Any) : Comparable<KotlinFunctionInfo> {

Expand Down
7 changes: 7 additions & 0 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ pluginManagement {
jcenter()
mavenLocal()
mavenCentral()
// only when using Kotlin EAP releases ...
maven { url = uri("https://dl.bintray.com/kotlin/kotlin-eap") }
maven { url = uri("https://dl.bintray.com/kotlin/kotlin-dev") }
}
Expand All @@ -27,4 +28,10 @@ pluginManagement {

}

gradle.projectsLoaded {
allprojects {
repositories.addAll(pluginManagement.repositories)
}
}

include("jupyter-lib")
21 changes: 21 additions & 0 deletions src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,27 @@ data class RuntimeKernelProperties(val map: Map<String, String>) {
get() = map["librariesFormatVersion"]?.toIntOrNull() ?: throw RuntimeException("Libraries format version is not specified!")
val currentBranch: String
get() = map["currentBranch"] ?: throw RuntimeException("Current branch is not specified!")
val jvmTargetForSnippets by lazy {
map["jvmTargetForSnippets"] ?: currentJavaVersion
}
}

val currentJavaVersion by lazy {
val defaultVersion = "1.8"
val version: String? = System.getProperty("java.version")

val versionParts = version?.split('.')
if (versionParts.isNullOrEmpty()){
defaultVersion
} else if (versionParts[0] == "1") {
if (versionParts.size > 1) {
"1.${versionParts[1]}"
} else {
defaultVersion
}
} else {
versionParts[0]
}
}

val runtimeProperties by lazy {
Expand Down
14 changes: 8 additions & 6 deletions src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package org.jetbrains.kotlin.jupyter

import jupyter.kotlin.*
import jupyter.kotlin.KotlinContext
import jupyter.kotlin.KotlinReceiver
import kotlinx.coroutines.runBlocking
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult
Expand All @@ -12,6 +11,7 @@ import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl
import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater
import org.jetbrains.kotlin.jupyter.repl.spark.ClassWriter
import org.jetbrains.kotlin.scripting.ide_services.compiler.KJvmReplCompilerWithIdeServices
import org.jetbrains.kotlin.scripting.resolve.skipExtensionsResolutionForImplicitsExceptInnermost
import java.io.File
import java.net.URLClassLoader
import java.util.*
Expand Down Expand Up @@ -166,7 +166,7 @@ class ReplForJupyterImpl(val scriptClasspath: List<File> = emptyList(),

private val ctx = KotlinContext()

private val receivers: List<Any> = listOf<Any>(KotlinReceiver(ctx)) + scriptReceivers
private val receivers: List<Any> = scriptReceivers.asList()

val magics = MagicsProcessor(this, LibrariesProcessor(resolverConfig?.libraries))

Expand Down Expand Up @@ -195,17 +195,19 @@ class ReplForJupyterImpl(val scriptClasspath: List<File> = emptyList(),
hostConfiguration.update { it.withDefaultsFrom(defaultJvmScriptingHostConfiguration) }
baseClass.put(KotlinType(ScriptTemplateWithDisplayHelpers::class))
fileExtension.put("jupyter.kts")
defaultImports(DependsOn::class, Repository::class, ScriptTemplateWithDisplayHelpers::class, KotlinReceiver::class)
defaultImports(DependsOn::class, Repository::class, ScriptTemplateWithDisplayHelpers::class)
jvm {
updateClasspath(scriptClasspath)
}
refineConfiguration {
onAnnotations(DependsOn::class, Repository::class, handler = { configureMavenDepsOnAnnotations(it) })
}

implicitReceivers.invoke(receivers.map { KotlinType(it.javaClass.canonicalName) })
val receiversTypes = receivers.map { KotlinType(it.javaClass.canonicalName) }
implicitReceivers(receiversTypes)
skipExtensionsResolutionForImplicitsExceptInnermost(receiversTypes)

compilerOptions.invoke(listOf("-jvm-target", "1.8"))
compilerOptions(listOf("-jvm-target", runtimeProperties.jvmTargetForSnippets))
}
}

Expand Down Expand Up @@ -429,7 +431,7 @@ class ReplForJupyterImpl(val scriptClasspath: List<File> = emptyList(),
//val preprocessed = preprocessCode(code)
val codeLine = SourceCodeImpl(executionCounter++, code)
val errorsList = runBlocking { compiler.analyze(codeLine, 0.toSourceCodePosition(codeLine), compilerConfiguration) }
ListErrorsResult(code, errorsList.valueOrThrow())
ListErrorsResult(code, errorsList.valueOrThrow()[ReplAnalyzerResult.analysisDiagnostics]!!)
}

private fun <T, Args: LockQueueArgs<T>> doWithLock(args: Args, queue: LockQueue<T, Args>, default: T, action: (Args) -> T) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package org.jetbrains.kotlin.jupyter.repl.reflect
import jupyter.kotlin.KotlinContext
import jupyter.kotlin.KotlinFunctionInfo
import jupyter.kotlin.KotlinVariableInfo
import org.jetbrains.kotlin.jupyter.instances
import org.slf4j.LoggerFactory

import java.lang.reflect.Field
Expand All @@ -20,7 +19,7 @@ import kotlin.script.experimental.util.LinkedSnippet
*/
class ContextUpdater(val context: KotlinContext, private val evaluator: BasicJvmReplEvaluator) {

var lastProcessedSnippet: LinkedSnippet<KJvmEvaluatedSnippet>? = null
private var lastProcessedSnippet: LinkedSnippet<KJvmEvaluatedSnippet>? = null

fun update() {
try {
Expand Down Expand Up @@ -64,18 +63,6 @@ class ContextUpdater(val context: KotlinContext, private val evaluator: BasicJvm
}
}

// For implicit receiver, we want to also get fields in parent classes
@Throws(IllegalAccessException::class)
private fun findReceiverVariables(receiver: Any) {
val fieldsList = ArrayList<Field>()
var cl: Class<*>? = receiver.javaClass
while (cl != null) {
fieldsList.addAll(listOf(*cl.declaredFields))
cl = cl.superclass
}
findVariables(fieldsList.toTypedArray(), receiver)
}

@Throws(IllegalAccessException::class)
private fun findVariables(fields: Array<Field>, o: Any) {
for (field in fields) {
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
* so this class provides writing classes on disk.
*/
class ClassWriter(_outputDir: String = "") {
val outputDir = if(_outputDir == "") {
val outputDir: Path = if(_outputDir == "") {
val tempDir = Files.createTempDirectory("kotlin-jupyter")
tempDir.toFile().deleteOnExit()
tempDir.toAbsolutePath()
Expand Down
Loading