From 655d88405558a9d2267b579708633d7600c7aa46 Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Wed, 25 Dec 2019 15:31:40 +0300 Subject: [PATCH 1/4] Added possibility of using properties from Gradle in source code, fixed implementation version --- build.gradle | 27 ++++++++++++++++++- .../org/jetbrains/kotlin/jupyter/config.kt | 13 ++++++++- .../org/jetbrains/kotlin/jupyter/ikotlin.kt | 13 +++++++-- .../org/jetbrains/kotlin/jupyter/protocol.kt | 7 +++-- .../org/jetbrains/kotlin/jupyter/repl.kt | 3 ++- 5 files changed, 56 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 02da87f1d..2c212a56a 100644 --- a/build.gradle +++ b/build.gradle @@ -2,11 +2,12 @@ import com.beust.klaxon.JsonObject import java.nio.file.Paths import java.util.regex.Pattern +import java.util.stream.Collectors buildscript { ext.shadowJarVersion = "5.2.0" ext.kotlinVersion = '1.3.70-eap-3' - ext.baseVersion = '0.7.40' + ext.baseVersion = '0.7.41' repositories { jcenter() mavenLocal() @@ -81,6 +82,10 @@ allprojects { debugPort = 1044 debuggerConfig = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$debugPort".toString() + mainSourceSetDir = "main" + resourcesDir = "resources" + runtimePropertiesFile = "runtime.properties" + jarsPath = "jars" librariesPath = "libraries" kernelFile = "kernel.json" @@ -112,6 +117,7 @@ allprojects { distribGroup = "distrib" condaGroup = "conda" pyPiGroup = "pip" + buildGroup = "build" condaUserStable = rootProject.findProperty('condaUserStable') ?: '' condaPasswordStable = rootProject.findProperty('condaPasswordStable') ?: '' @@ -161,6 +167,25 @@ jar.manifest.attributes( 'Implementation-Version': version ) +task buildProperties(group: buildGroup) { + def outputDir = file(getSubDir(buildDir.toPath(), resourcesDir, mainSourceSetDir)) + + inputs.property "version", version + outputs.dir outputDir + + doLast { + outputDir.mkdirs() + def propertiesFile = file(getSubDir(outputDir.toPath(), runtimePropertiesFile)) + propertiesFile.text = inputs.properties.entrySet().stream().map { + "${it.key}=${it.value}\n" + }.collect(Collectors.joining()) + } +} + +processResources { + dependsOn buildProperties +} + shadowJar { archiveBaseName.set(packageName) archiveClassifier.set('') diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt index 4c00c8b12..c7bd3f797 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt @@ -40,6 +40,16 @@ enum class JupyterSockets { iopub } +data class RuntimeKernelProperties( + var version: String = "unspecified" +) { + constructor(map: Map?) : this() { + if (map == null) + return + version = map["version"] ?: version + } +} + data class KernelConfig( val ports: Array, val transport: String, @@ -47,7 +57,8 @@ data class KernelConfig( val signatureKey: String, val pollingIntervalMillis: Long = 100, val scriptClasspath: List = emptyList(), - val resolverConfig: ResolverConfig? + val resolverConfig: ResolverConfig?, + val runtimeProperties: RuntimeKernelProperties = RuntimeKernelProperties() ) val protocolVersion = "5.3" diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt index f342e9d13..889a62986 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt @@ -45,6 +45,14 @@ fun printClassPath() { log.info("Current classpath: " + cp.joinToString()) } +fun loadRuntimeProperties(): RuntimeKernelProperties { + val map = object{}.javaClass.classLoader + .getResource("runtime.properties") + ?.readText()?.parseIniConfig() + + return RuntimeKernelProperties(map) +} + fun main(vararg args: String) { try { log.info("Kernel args: "+ args.joinToString { it }) @@ -62,7 +70,8 @@ fun main(vararg args: String) { signatureScheme = sigScheme ?: "hmac1-sha256", signatureKey = if (sigScheme == null || key == null) "" else key, scriptClasspath = scriptClasspath, - resolverConfig = loadResolverConfig(rootPath) + resolverConfig = loadResolverConfig(rootPath), + runtimeProperties = loadRuntimeProperties() )) } catch (e: Exception) { log.error("exception running kernel with args: \"${args.joinToString()}\"", e) @@ -80,7 +89,7 @@ fun kernelServer(config: KernelConfig) { val executionCount = AtomicLong(1) - val repl = ReplForJupyter(config.scriptClasspath, config.resolverConfig) + val repl = ReplForJupyter(config.scriptClasspath, config.resolverConfig, config.runtimeProperties) val mainThread = Thread.currentThread() diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt index 4dcb21dd7..2f611f070 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt @@ -32,13 +32,16 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup "language_info" to jsonObject( "name" to "kotlin", "codemirror_mode" to "text/x-kotlin", - "file_extension" to ".kt" + "file_extension" to ".kt", + "mimetype" to "text/x-kotlin", + "pygments_lexer" to "kotlin", + "version" to KotlinCompilerVersion.VERSION ), // Jupyter lab Console support "banner" to "Kotlin language, version ${KotlinCompilerVersion.VERSION}", "implementation" to "Kotlin", - "implementation_version" to KotlinCompilerVersion.VERSION, + "implementation_version" to repl!!.properties.version, "status" to "ok" ))) "history_request" -> diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt index 1e26b868f..9f5c03653 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt @@ -38,7 +38,8 @@ class ReplCompilerException(val errorResult: ReplCompileResult.Error) : ReplExce } class ReplForJupyter(val scriptClasspath: List = emptyList(), - val config: ResolverConfig? = null) { + val config: ResolverConfig? = null, + val properties: RuntimeKernelProperties = RuntimeKernelProperties()) { private val resolver = JupyterScriptDependenciesResolver(config) From 109d27d8707aaa5263168f2cf51c10f6b50a08ce Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Wed, 25 Dec 2019 16:22:48 +0300 Subject: [PATCH 2/4] Fixed NPE --- src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt index 2f611f070..0e8fa4ae7 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt @@ -22,6 +22,7 @@ data class ResponseWithMessage(val state: ResponseState, val result: MimeTypedRe fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJupyter?, executionCount: AtomicLong) { val msgType = msg.header!!["msg_type"] + val replProperties = repl?.properties ?: RuntimeKernelProperties() when (msgType) { "kernel_info_request" -> sendWrapped(msg, makeReplyMessage(msg, "kernel_info_reply", @@ -41,7 +42,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup // Jupyter lab Console support "banner" to "Kotlin language, version ${KotlinCompilerVersion.VERSION}", "implementation" to "Kotlin", - "implementation_version" to repl!!.properties.version, + "implementation_version" to replProperties.version, "status" to "ok" ))) "history_request" -> From 16f536f5d604f09ba297bc9c333700d8ac56d65c Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Thu, 26 Dec 2019 12:34:02 +0300 Subject: [PATCH 3/4] Refactored build.gradle --- build.gradle | 104 ++++++++++++++++++++++++++++----------------------- 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/build.gradle b/build.gradle index 2c212a56a..82792f8aa 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,6 @@ -import com.beust.klaxon.JsonObject +import groovy.json.JsonOutput +import java.nio.file.Path import java.nio.file.Paths import java.util.regex.Pattern import java.util.stream.Collectors @@ -19,7 +20,6 @@ buildscript { //noinspection DifferentKotlinGradleVersion classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" classpath "com.github.jengelman.gradle.plugins:shadow:$shadowJarVersion" - classpath "com.beust:klaxon:5.2" } } @@ -43,42 +43,54 @@ allprojects { testCompile "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" } - String artifactsPathStr = rootProject.findProperty('artifactsPath') ?: 'artifacts' - String buildCounterStr = rootProject.findProperty('build.counter') ?: '100500' - String buildNumber = rootProject.findProperty('build.number') ?: '' - String installPath = rootProject.findProperty('installPath') - - ext.rootPath = rootDir.toPath() - ext.packageName = 'kotlin-jupyter-kernel' - ext.artifactsDir = rootPath.resolve(artifactsPathStr) - ext.isProtectedBranch = isProtectedBranch() - ext.versionFileName = "VERSION" - - ext.installPathLocal = installPath ? Paths.get(installPath) : - Paths.get(System.properties['user.home'].toString(), ".ipython", "kernels", "kotlin") - ext.distributionPath = rootPath.resolve("distrib") - ext.distribBuildPath = rootPath.resolve("distrib-build") - ext.logosPath = getSubDir(rootPath, "resources", "logos") - - String devAddition = isProtectedBranch ? '' : '.dev1' - String defaultBuildNumber = "$baseVersion.$buildCounterStr$devAddition" - String buildNumberRegex = "[0-9]+(\\.[0-9]+){3}(\\.dev[0-9]+)?" - - if (!Pattern.matches(buildNumberRegex, buildNumber)) { - def versionFile = artifactsDir.resolve(versionFileName).toFile() - if (versionFile.exists()) { - def lines = versionFile.readLines() - assert !lines.empty, "There should be at least one line in VERSION file" - buildNumber = lines.first().trim() + ext { + packageName = "kotlin-jupyter-kernel" + versionFileName = "VERSION" + + String artifactsPathStr = rootProject.findProperty('artifactsPath') ?: 'artifacts' + String installPath = rootProject.findProperty('installPath') + + //noinspection GroovyAssignabilityCheck + rootPath = rootDir.toPath() + //noinspection GroovyAssignabilityCheck + artifactsDir = rootPath.resolve(artifactsPathStr) + + //noinspection GroovyAssignabilityCheck + installPathLocal = installPath ? Paths.get(installPath) : + Paths.get(System.properties['user.home'].toString(), ".ipython", "kernels", "kotlin") + //noinspection GroovyAssignabilityCheck + distributionPath = rootPath.resolve("distrib") + //noinspection GroovyAssignabilityCheck + distribBuildPath = rootPath.resolve("distrib-build") + //noinspection GroovyAssignabilityCheck + logosPath = getSubDir(rootPath, "resources", "logos") + + if (project == rootProject) { + isProtectedBranch = isProtectedBranch() + String buildCounterStr = rootProject.findProperty('build.counter') ?: '100500' + String buildNumber = rootProject.findProperty('build.number') ?: '' + String devAddition = isProtectedBranch ? '' : '.dev1' + String defaultBuildNumber = "$baseVersion.$buildCounterStr$devAddition" + String buildNumberRegex = "[0-9]+(\\.[0-9]+){3}(\\.dev[0-9]+)?" + + if (!Pattern.matches(buildNumberRegex, buildNumber)) { + def versionFile = artifactsDir.resolve(versionFileName).toFile() + if (versionFile.exists()) { + def lines = versionFile.readLines() + assert !lines.empty, "There should be at least one line in VERSION file" + buildNumber = lines.first().trim() + } else { + buildNumber = defaultBuildNumber + } + } + + project.version = buildNumber + println("##teamcity[buildNumber '$version']") } else { - buildNumber = defaultBuildNumber + isProtectedBranch = rootProject.isProtectedBranch + project.version = rootProject.version } - } - project.version = buildNumber - println("##teamcity[buildNumber '$version']") - - ext { debugPort = 1044 debuggerConfig = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$debugPort".toString() @@ -207,14 +219,14 @@ static String makeTaskName(String prefix, Boolean local) { return prefix + (local ? "Local" : "Distrib") } -static void makeDirs(java.nio.file.Path path) { +static void makeDirs(Path path) { File dir = path.toFile() if (!dir.exists()) { dir.mkdirs() } } -static java.nio.file.Path getSubDir(java.nio.file.Path dir, String... subDir) { +static Path getSubDir(Path dir, String... subDir) { def newDir = dir for (s in subDir) { newDir = newDir.resolve(s) @@ -222,9 +234,9 @@ static java.nio.file.Path getSubDir(java.nio.file.Path dir, String... subDir) { return newDir } -static void writeJson(Map json, java.nio.file.Path path) { - def jsonString = new JsonObject(json).toJsonString(true, false) - path.toFile().write(jsonString, 'UTF-8') +static void writeJson(Map json, Path path) { + def str = JsonOutput.prettyPrint(JsonOutput.toJson(json)) + path.toFile().write(str, 'UTF-8') } void createCleanTasks() { @@ -241,7 +253,7 @@ void createCleanTasks() { createCleanTasks() @SuppressWarnings("unused") -void createInstallTasks(Boolean local, java.nio.file.Path specPath, java.nio.file.Path mainInstallPath) { +void createInstallTasks(Boolean local, Path specPath, Path mainInstallPath) { def groupName = local ? localGroup : distribGroup def cleanDirTask = getTasks().getByName(makeTaskName(cleanInstallDirTaskPrefix, local)) def args = [type: Copy, dependsOn: cleanDirTask, group: groupName] @@ -267,7 +279,7 @@ void createInstallTasks(Boolean local, java.nio.file.Path specPath, java.nio.fil } } -String createTaskForSpecs(Boolean debug, Boolean local, String group, Task cleanDir, java.nio.file.Path specPath, java.nio.file.Path mainInstallPath) { +String createTaskForSpecs(Boolean debug, Boolean local, String group, Task cleanDir, Path specPath, Path mainInstallPath) { String taskName = makeTaskName(debug ? "createDebugSpecs" : "createSpecs", local) task([group: group], taskName) { dependsOn cleanDir, shadowJar @@ -308,7 +320,7 @@ void createMainInstallTask(Boolean debug, Boolean local, String group, String sp } } -void makeKernelSpec(java.nio.file.Path installPath, Boolean localInstall) { +void makeKernelSpec(Path installPath, Boolean localInstall) { def argv = localInstall ? Arrays.asList("python", installPath.resolve(runKernelPy).toString(), @@ -329,7 +341,7 @@ void makeKernelSpec(java.nio.file.Path installPath, Boolean localInstall) { } } -void makeJarArgs(java.nio.file.Path installPath, String kernelJarPath, List classPath, String debuggerConfig = "") { +void makeJarArgs(Path installPath, String kernelJarPath, List classPath, String debuggerConfig = "") { writeJson([ "mainJar": kernelJarPath, "classPath": classPath, @@ -346,8 +358,6 @@ task copyRunKernelPy(type: Copy, dependsOn: cleanInstallDirLocal, group: localGr createInstallTasks(true, installPathLocal, installPathLocal) -task uninstall(dependsOn: cleanInstallDirLocal, group: localGroup) { - -} +task uninstall(dependsOn: cleanInstallDirLocal, group: localGroup) apply from: 'distrib.gradle' From 84f4ad338347a4dd396914b657d876a4423ee571 Mon Sep 17 00:00:00 2001 From: Anatoly Nikitin Date: Thu, 26 Dec 2019 17:26:05 +0300 Subject: [PATCH 4/4] Simplify runtimeProperties: make it lazy static field --- .../org/jetbrains/kotlin/jupyter/config.kt | 18 ++++++++---------- .../org/jetbrains/kotlin/jupyter/ikotlin.kt | 13 ++----------- .../org/jetbrains/kotlin/jupyter/protocol.kt | 3 +-- .../org/jetbrains/kotlin/jupyter/repl.kt | 3 +-- 4 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt index c7bd3f797..68476c894 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt @@ -40,14 +40,13 @@ enum class JupyterSockets { iopub } -data class RuntimeKernelProperties( - var version: String = "unspecified" -) { - constructor(map: Map?) : this() { - if (map == null) - return - version = map["version"] ?: version - } +data class RuntimeKernelProperties(val map: Map) { + val version: String + get() = map["version"] ?: "unspecified" +} + +val runtimeProperties by lazy { + RuntimeKernelProperties(ClassLoader.getSystemResource("runtime.properties")?.readText()?.parseIniConfig().orEmpty()) } data class KernelConfig( @@ -57,8 +56,7 @@ data class KernelConfig( val signatureKey: String, val pollingIntervalMillis: Long = 100, val scriptClasspath: List = emptyList(), - val resolverConfig: ResolverConfig?, - val runtimeProperties: RuntimeKernelProperties = RuntimeKernelProperties() + val resolverConfig: ResolverConfig? ) val protocolVersion = "5.3" diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt index 889a62986..f342e9d13 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt @@ -45,14 +45,6 @@ fun printClassPath() { log.info("Current classpath: " + cp.joinToString()) } -fun loadRuntimeProperties(): RuntimeKernelProperties { - val map = object{}.javaClass.classLoader - .getResource("runtime.properties") - ?.readText()?.parseIniConfig() - - return RuntimeKernelProperties(map) -} - fun main(vararg args: String) { try { log.info("Kernel args: "+ args.joinToString { it }) @@ -70,8 +62,7 @@ fun main(vararg args: String) { signatureScheme = sigScheme ?: "hmac1-sha256", signatureKey = if (sigScheme == null || key == null) "" else key, scriptClasspath = scriptClasspath, - resolverConfig = loadResolverConfig(rootPath), - runtimeProperties = loadRuntimeProperties() + resolverConfig = loadResolverConfig(rootPath) )) } catch (e: Exception) { log.error("exception running kernel with args: \"${args.joinToString()}\"", e) @@ -89,7 +80,7 @@ fun kernelServer(config: KernelConfig) { val executionCount = AtomicLong(1) - val repl = ReplForJupyter(config.scriptClasspath, config.resolverConfig, config.runtimeProperties) + val repl = ReplForJupyter(config.scriptClasspath, config.resolverConfig) val mainThread = Thread.currentThread() diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt index 0e8fa4ae7..7345a7b00 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt @@ -22,7 +22,6 @@ data class ResponseWithMessage(val state: ResponseState, val result: MimeTypedRe fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJupyter?, executionCount: AtomicLong) { val msgType = msg.header!!["msg_type"] - val replProperties = repl?.properties ?: RuntimeKernelProperties() when (msgType) { "kernel_info_request" -> sendWrapped(msg, makeReplyMessage(msg, "kernel_info_reply", @@ -42,7 +41,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup // Jupyter lab Console support "banner" to "Kotlin language, version ${KotlinCompilerVersion.VERSION}", "implementation" to "Kotlin", - "implementation_version" to replProperties.version, + "implementation_version" to runtimeProperties.version, "status" to "ok" ))) "history_request" -> diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt index 9f5c03653..1e26b868f 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt @@ -38,8 +38,7 @@ class ReplCompilerException(val errorResult: ReplCompileResult.Error) : ReplExce } class ReplForJupyter(val scriptClasspath: List = emptyList(), - val config: ResolverConfig? = null, - val properties: RuntimeKernelProperties = RuntimeKernelProperties()) { + val config: ResolverConfig? = null) { private val resolver = JupyterScriptDependenciesResolver(config)