diff --git a/.idea/dictionaries/ProjectDictionary.xml b/.idea/dictionaries/ProjectDictionary.xml
index 84c237075..be051276f 100644
--- a/.idea/dictionaries/ProjectDictionary.xml
+++ b/.idea/dictionaries/ProjectDictionary.xml
@@ -8,6 +8,7 @@
clikt
comms
deeplearning
+ displayname
distrib
doyaaaaaken
dsdsda
diff --git a/README.md b/README.md
index ff856155a..5b954f283 100644
--- a/README.md
+++ b/README.md
@@ -100,6 +100,7 @@ The following line magics are supported:
- `%use , ...` - injects code for supported libraries: artifact resolution, default imports, initialization code, type renderers
- `%trackClasspath` - logs any changes of current classpath. Useful for debugging artifact resolution failures
- `%trackExecution` - logs pieces of code that are going to be executed. Useful for debugging of libraries support
+ - `%useLatestDescriptors` - use latest versions of library descriptors available. By default, bundled descriptors are used
- `%output [options]` - output capturing settings.
See detailed info about line magics [here](doc/magics.md).
@@ -128,6 +129,32 @@ Several libraries can be included in single `%use` statement, separated by `,`:
```
%use lets-plot, krangl, mysql(8.0.15)
```
+You can also specify the source of library descriptor. By default, it's downloaded from the latest commit on the
+branch which kernel was built from. If you want to try descriptor from another revision, use the following syntax:
+```
+// Specify tag
+%use lets-plot@0.8.2.5
+// Specify commit sha, with more verbose syntax
+%use lets-plot@ref[24a040fe22335648885b106e2f4ddd63b4d49469]
+// Specify git ref along with library arguments
+%use krangl@dev(0.10)
+```
+Other options are resolving library descriptor from a local file or from remote URL:
+```
+// Load library from file
+%use mylib@file[/home/user/lib.json]
+// Load library from file: kernel will guess it's a file actually
+%use @/home/user/libs/lib.json
+// Or use another approach: specify a directory and file name without
+// extension (it should be JSON in such case) before it
+%use lib@/home/user/libs
+// Load library descriptor from a remote URL
+%use herlib@url[https://site.com/lib.json]
+// If your URL responds with 200(OK), you may skip `url[]` part:
+%use @https://site.com/lib.json
+// You may omit library name for file and URL resolution:
+%use @file[lib.json]
+```
List of supported libraries:
- [klaxon](https://github.com/cbeust/klaxon) - JSON parser for Kotlin
diff --git a/gradle.properties b/gradle.properties
index aa5499f97..7570c7bd6 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,5 +1,5 @@
# kotlinVersion=1.4.255-SNAPSHOT
-kotlinVersion=1.4.20-dev-2342
+kotlinVersion=1.4.20-dev-3647
kotlinLanguageLevel=1.4
jvmTarget=1.8
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 9b4216ef3..7c56670df 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -11,6 +11,12 @@ pluginManagement {
// only when using Kotlin EAP releases ...
maven { url = uri("https://dl.bintray.com/kotlin/kotlin-eap") }
maven { url = uri("https://dl.bintray.com/kotlin/kotlin-dev") }
+
+ // Used for TeamCity build
+ val m2LocalPath = File(".m2/repository")
+ if (m2LocalPath.exists()) {
+ maven(m2LocalPath.toURI())
+ }
}
resolutionStrategy {
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt
index fdf6f64c5..60f0eb553 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt
@@ -1,7 +1,7 @@
package org.jetbrains.kotlin.jupyter
import jupyter.kotlin.KotlinFunctionInfo
-import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater
+import org.jetbrains.kotlin.jupyter.repl.ContextUpdater
interface AnnotationsProcessor {
@@ -57,4 +57,4 @@ class AnnotationsProcessorImpl(private val contextUpdater: ContextUpdater) : Ann
}
return codeToExecute
}
-}
\ No newline at end of file
+}
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt
index 6c57c6981..066639411 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt
@@ -1,10 +1,11 @@
package org.jetbrains.kotlin.jupyter
import jupyter.kotlin.textResult
-import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult
-import org.jetbrains.kotlin.jupyter.repl.completion.KotlinCompleter
-import org.jetbrains.kotlin.jupyter.repl.completion.ListErrorsResult
-import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl
+import org.jetbrains.kotlin.jupyter.libraries.parseLibraryDescriptor
+import org.jetbrains.kotlin.jupyter.repl.CompletionResult
+import org.jetbrains.kotlin.jupyter.repl.KotlinCompleter
+import org.jetbrains.kotlin.jupyter.repl.ListErrorsResult
+import org.jetbrains.kotlin.jupyter.repl.SourceCodeImpl
import kotlin.script.experimental.api.ScriptDiagnostic
import kotlin.script.experimental.api.SourceCode
import kotlin.script.experimental.api.SourceCodeCompletionVariant
@@ -51,12 +52,12 @@ fun doCommandCompletion(code: String, cursor: Int): CompletionResult {
return KotlinCompleter.getResult(code, cursor, completions)
}
-fun runCommand(code: String, repl: ReplForJupyter?): Response {
+fun runCommand(code: String, repl: ReplForJupyter): Response {
val args = code.trim().substring(1).split(" ")
val cmd = getCommand(args[0]) ?: return AbortResponseWithMessage(textResult("Failed!"), "unknown command: $code\nto see available commands, enter :help")
return when (cmd) {
ReplCommands.classpath -> {
- val cp = repl!!.currentClasspath
+ val cp = repl.currentClasspath
OkResponseWithMessage(textResult("Current classpath (${cp.count()} paths):\n${cp.joinToString("\n")}"))
}
ReplCommands.help -> {
@@ -66,9 +67,16 @@ fun runCommand(code: String, repl: ReplForJupyter?): Response {
if (it.argumentsUsage != null) s += "\n Usage: %${it.name} ${it.argumentsUsage}"
s
}
- val libraries = repl?.resolverConfig?.libraries?.awaitBlocking()?.toList()?.joinToStringIndented {
- "${it.first} ${it.second.link ?: ""}"
- }
+ val libraryFiles =
+ repl.homeDir?.resolve(LibrariesDir)?.listFiles { file -> file.isFile && file.name.endsWith(".$LibraryDescriptorExt") } ?: emptyArray()
+ val libraries = libraryFiles.toList().mapNotNull { file ->
+ val libraryName = file.nameWithoutExtension
+ log.info("Parsing descriptor for library '$libraryName'")
+ val descriptor = log.catchAll("Parsing descriptor for library '$libraryName' failed") {
+ parseLibraryDescriptor(file.readText())
+ }
+ if (descriptor != null) "$libraryName ${descriptor.link ?: ""}" else null
+ }.joinToStringIndented()
OkResponseWithMessage(textResult("Commands:\n$commands\n\nMagics\n$magics\n\nSupported libraries:\n$libraries"))
}
}
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index a5dafb967..ec1363e51 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -5,34 +5,45 @@ import com.beust.klaxon.Parser
import jupyter.kotlin.JavaRuntime
import jupyter.kotlin.KotlinKernelVersion
import khttp.responses.Response
-import kotlinx.coroutines.Deferred
-import kotlinx.coroutines.GlobalScope
-import kotlinx.coroutines.async
-import org.apache.commons.io.FileUtils
-import org.json.JSONObject
+import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory
+import org.jetbrains.kotlin.jupyter.libraries.LibraryResolver
import org.slf4j.LoggerFactory
import org.zeromq.SocketType
import java.io.File
-import java.nio.file.Files
import java.nio.file.Paths
import kotlin.script.experimental.dependencies.RepositoryCoordinates
-val LibrariesDir = "libraries"
-val LocalCacheDir = "cache"
-val CachedLibrariesFootprintFile = "libsCommit"
+const val LibrariesDir = "libraries"
+const val LocalCacheDir = "cache"
val LocalSettingsPath = Paths.get(System.getProperty("user.home"), ".jupyter_kotlin").toString()
-val GitHubApiHost = "api.github.com"
-val GitHubRepoOwner = "kotlin"
-val GitHubRepoName = "kotlin-jupyter"
-val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName"
+const val GitHubApiHost = "api.github.com"
+const val GitHubRepoOwner = "kotlin"
+const val GitHubRepoName = "kotlin-jupyter"
+const val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName"
-val LibraryDescriptorExt = "json"
-val LibraryPropertiesFile = ".properties"
+const val LibraryDescriptorExt = "json"
+const val LibraryPropertiesFile = ".properties"
+
+const val protocolVersion = "5.3"
internal val log by lazy { LoggerFactory.getLogger("ikotlin") }
+val defaultRuntimeProperties by lazy {
+ RuntimeKernelProperties(ClassLoader.getSystemResource("runtime.properties")?.readText()?.parseIniConfig().orEmpty())
+}
+
+val defaultRepositories = arrayOf(
+ "https://jcenter.bintray.com/",
+ "https://repo.maven.apache.org/maven2/",
+ "https://jitpack.io",
+).map { RepositoryCoordinates(it) }
+
+val defaultGlobalImports = listOf(
+ "kotlin.math.*",
+)
+
enum class JupyterSockets(val zmqKernelType: SocketType, val zmqClientType: SocketType) {
hb(SocketType.REP, SocketType.REQ),
shell(SocketType.ROUTER, SocketType.REQ),
@@ -57,25 +68,21 @@ data class OutputConfig(
}
}
-data class RuntimeKernelProperties(val map: Map) {
- val version: KotlinKernelVersion? by lazy {
+class RuntimeKernelProperties(val map: Map): ReplRuntimeProperties {
+ override val version: KotlinKernelVersion? by lazy {
map["version"]?.let{ KotlinKernelVersion.from(it) }
}
- val librariesFormatVersion: Int
+ override val librariesFormatVersion: Int
get() = map["librariesFormatVersion"]?.toIntOrNull() ?: throw RuntimeException("Libraries format version is not specified!")
- val currentBranch: String
+ override val currentBranch: String
get() = map["currentBranch"] ?: throw RuntimeException("Current branch is not specified!")
- val currentSha: String
+ override val currentSha: String
get() = map["currentSha"] ?: throw RuntimeException("Current commit SHA is not specified!")
- val jvmTargetForSnippets by lazy {
+ override val jvmTargetForSnippets by lazy {
map["jvmTargetForSnippets"] ?: JavaRuntime.version
}
}
-val runtimeProperties by lazy {
- RuntimeKernelProperties(ClassLoader.getSystemResource("runtime.properties")?.readText()?.parseIniConfig().orEmpty())
-}
-
data class KernelConfig(
val ports: List,
val transport: String,
@@ -83,9 +90,11 @@ data class KernelConfig(
val signatureKey: String,
val pollingIntervalMillis: Long = 100,
val scriptClasspath: List = emptyList(),
- val resolverConfig: ResolverConfig?
+ val homeDir: File?,
+ val resolverConfig: ResolverConfig?,
+ val libraryFactory: LibraryFactory,
) {
- fun toArgs(prefix: String = "", homeDir: File? = null): KernelArgs {
+ fun toArgs(prefix: String = ""): KernelArgs {
val cfgJson = jsonObject(
"transport" to transport,
"signature_scheme" to signatureScheme,
@@ -102,7 +111,7 @@ data class KernelConfig(
}
companion object {
- fun fromArgs(args: KernelArgs): KernelConfig {
+ fun fromArgs(args: KernelArgs, libraryFactory: LibraryFactory): KernelConfig {
val (cfgFile, scriptClasspath, homeDir) = args
val cfgJson = Parser.default().parse(cfgFile.canonicalPath) as JsonObject
fun JsonObject.getInt(field: String): Int = int(field) ?: throw RuntimeException("Cannot find $field in $cfgFile")
@@ -116,17 +125,17 @@ data class KernelConfig(
signatureScheme = sigScheme ?: "hmac1-sha256",
signatureKey = if (sigScheme == null || key == null) "" else key,
scriptClasspath = scriptClasspath,
- resolverConfig = homeDir?.let { loadResolverConfig(it.toString()) }
+ homeDir = homeDir,
+ resolverConfig = homeDir?.let { loadResolverConfig(it.toString(), libraryFactory) },
+ libraryFactory = libraryFactory,
)
}
}
}
-val protocolVersion = "5.3"
-
data class TypeHandler(val className: TypeName, val code: Code)
-data class Variable(val name: String, val value: String)
+data class Variable(val name: String, val value: String, val required: Boolean = false)
open class LibraryDefinition(
val dependencies: List,
@@ -141,6 +150,7 @@ open class LibraryDefinition(
)
class LibraryDescriptor(
+ val originalJson: JsonObject,
dependencies: List,
val variables: List,
initCell: List,
@@ -156,56 +166,7 @@ class LibraryDescriptor(
) : LibraryDefinition(dependencies, initCell, imports, repositories, init, shutdown, renderers, converters, annotations)
data class ResolverConfig(val repositories: List,
- val libraries: Deferred