Add separate repository for library descriptors

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "libraries"]
+	path = libraries
+	url = https://github.com/Kotlin/kotlin-jupyter-libraries.git

docs/README-STUB.md

@@ -52,7 +52,7 @@ Uninstall: `pip uninstall kotlin-jupyter-kernel`
 ### From sources
 
 ```bash
-git clone [[repo_url]]
+git clone --recurse-submodules [[repo_url]]
 cd kotlin-jupyter
 ./gradlew install
 ```
@@ -160,7 +160,7 @@ When a library is included with `%use` keyword, the following functionality is a
  - library initialization code
  - renderers for special types, e.g. charts and data frames
 
-This behavior is defined by `json` library descriptor. Descriptors for all supported libraries can be found in [libraries](../libraries) directory.
+This behavior is defined by `json` library descriptor. Descriptors for all supported libraries can be found in [libraries](https://github.com/Kotlin/kotlin-jupyter-libraries) repository.
 A library descriptor may provide a set of properties with default values that can be overridden when library is included.
 The major use case for library properties is to specify a particular version of library. If descriptor has only one property, it can be 
 defined without naming:
@@ -175,8 +175,7 @@ Several libraries can be included in single `%use` statement, separated by `,`:
 ```
 %use lets-plot, krangl, mysql(8.0.15)
 ```
-You can also specify the source of library descriptor. By default, it's taken from the `libraries` directory
-of kernel installation. If you want to try descriptor from another revision, use the following syntax:
+You can also specify the source of library descriptor. By default, it's taken from the [libraries repository](https://github.com/Kotlin/kotlin-jupyter-libraries). If you want to try descriptor from another revision, use the following syntax:
 ```
 // Specify some git tag from this repository
 %use lets-plot@0.8.2.5

docs/README.md

@@ -52,7 +52,7 @@ Uninstall: `pip uninstall kotlin-jupyter-kernel`
 ### From sources
 
 ```bash
-git clone https://github.com/Kotlin/kotlin-jupyter.git
+git clone --recurse-submodules https://github.com/Kotlin/kotlin-jupyter.git
 cd kotlin-jupyter
 ./gradlew install
 ```
@@ -167,7 +167,7 @@ When a library is included with `%use` keyword, the following functionality is a
  - library initialization code
  - renderers for special types, e.g. charts and data frames
 
-This behavior is defined by `json` library descriptor. Descriptors for all supported libraries can be found in [libraries](../libraries) directory.
+This behavior is defined by `json` library descriptor. Descriptors for all supported libraries can be found in [libraries](https://github.com/Kotlin/kotlin-jupyter-libraries) repository.
 A library descriptor may provide a set of properties with default values that can be overridden when library is included.
 The major use case for library properties is to specify a particular version of library. If descriptor has only one property, it can be 
 defined without naming:
@@ -182,8 +182,7 @@ Several libraries can be included in single `%use` statement, separated by `,`:
 ```
 %use lets-plot, krangl, mysql(8.0.15)
 ```
-You can also specify the source of library descriptor. By default, it's taken from the `libraries` directory
-of kernel installation. If you want to try descriptor from another revision, use the following syntax:
+You can also specify the source of library descriptor. By default, it's taken from the [libraries repository](https://github.com/Kotlin/kotlin-jupyter-libraries). If you want to try descriptor from another revision, use the following syntax:
 ```
 // Specify some git tag from this repository
 %use lets-plot@0.8.2.5

docs/libraries.md

@@ -8,7 +8,7 @@ Generally, there are two ways of adding new library:
 
 To support new `JVM` library and make it available via `%use` magic command you need to create a library descriptor for it.
 
-Check [libraries](../libraries) directory to see examples of library descriptors.
+Check [libraries][libs-repo] repository to see examples of library descriptors.
 
 Library descriptor is a `<libName>.json` file with the following fields:
 - `properties`: a dictionary of properties that are used within library descriptor
@@ -37,7 +37,7 @@ To register new library descriptor:
 1. For private usage - create it anywhere on your computer and reference it using file syntax.
 2. Alternative way for private usage - create descriptor in `.jupyter_kotlin/libraries` folder and reference
    it using "default" syntax
-3. For sharing with community - commit it to [libraries](../libraries) directory and create pull request.
+3. For sharing with community - commit it to [libraries][libs-repo] repository and create pull request.
 
 If you are maintaining some library and want to update your library descriptor, create pull request with your update.
 After your request is accepted, new version of your library will be available to all Kotlin Jupyter users
@@ -155,3 +155,5 @@ resources. This file should contain FQNs of all integration classes in the JSON
 ```
 Classes derived from `LibraryDefinition` should be added to the `definitions` array.
 Classes derived from `LibraryDefinitionProducer` should be added to the `producers` array.
+
+[libs-repo]: https://github.com/Kotlin/kotlin-jupyter-libraries

gradle.properties

@@ -25,6 +25,8 @@ githubRepoName=kotlin-jupyter
 projectRepoUrl=https://github.com/Kotlin/kotlin-jupyter
 docsRepo=git@github.com:ileasile/kotlin-jupyter-docs.git
 pushRepoUrl=git@github.com:Kotlin/kotlin-jupyter.git
+librariesRepoUrl=git@github.com:Kotlin/kotlin-jupyter-libraries.git
+librariesRepoUserAndName=Kotlin/kotlin-jupyter-libraries
 
 # Workaround for https://github.com/Kotlin/dokka/issues/1405
 org.gradle.jvmargs=-XX:MaxMetaspaceSize=1024m -Xmx2048m

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/ResolutionInfoProvider.kt

@@ -42,7 +42,7 @@ class StandardResolutionInfoProvider(override var fallback: LibraryResolutionInf
     }
 
     private fun tryGetAsRef(ref: String): LibraryResolutionInfo? {
-        val response = getHttp("$GitHubApiPrefix/contents/$LibrariesDir?ref=$ref")
+        val response = getHttp("$GitHubApiPrefix/contents/$RemoteLibrariesDir?ref=$ref")
         return if (response.status.successful) LibraryResolutionInfo.getInfoByRef(ref) else null
     }
 

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/constants.kt

@@ -7,8 +7,9 @@ const val LibrariesDir = "libraries"
 const val LocalCacheDir = "cache"
 const val LocalSettingsDir = ".jupyter_kotlin"
 const val GitHubApiHost = "api.github.com"
-const val GitHubRepoOwner = "kotlin"
-const val GitHubRepoName = "kotlin-jupyter"
+const val GitHubRepoOwner = "Kotlin"
+const val GitHubRepoName = "kotlin-jupyter-libraries"
+const val RemoteLibrariesDir = ""
 const val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName"
 const val LibraryDescriptorExt = "json"
 const val LibraryPropertiesFile = ".properties"

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/httpUtil.kt

@@ -10,7 +10,7 @@ import org.jetbrains.kotlinx.jupyter.config.getLogger
 fun getLatestCommitToLibraries(ref: String, sinceTimestamp: String?): Pair<String, String>? {
     val logger = getLogger()
     return logger.catchAll {
-        var url = "$GitHubApiPrefix/commits?path=$LibrariesDir&sha=$ref"
+        var url = "$GitHubApiPrefix/commits?path=$RemoteLibrariesDir&sha=$ref"
         if (sinceTimestamp != null) {
             url += "&since=$sinceTimestamp"
         }
@@ -20,7 +20,7 @@ fun getLatestCommitToLibraries(ref: String, sinceTimestamp: String?): Pair<Strin
             if (sinceTimestamp != null) {
                 getLatestCommitToLibraries(ref, null)
             } else {
-                logger.info("Didn't find any commits to '$LibrariesDir' at $url")
+                logger.info("Didn't find any commits to libraries at $url")
                 null
             }
         } else {

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/libraryResolvers.kt

@@ -112,7 +112,7 @@ object FallbackLibraryResolver : LibraryDescriptorResolver() {
         resolver<LibraryResolutionInfo.ByGitRef> { name ->
             if (name == null) throw ReplLibraryLoadingException(message = "Reference library resolver needs name to be specified")
 
-            val url = "$GitHubApiPrefix/contents/$LibrariesDir/$name.$LibraryDescriptorExt?ref=$sha"
+            val url = "$GitHubApiPrefix/contents/$RemoteLibrariesDir/$name.$LibraryDescriptorExt?ref=$sha"
             getLogger().info("Requesting library descriptor at $url")
             val response = getHttp(url).jsonObject
 

kotlin-jupyter-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/build/versionDetection.kt

@@ -4,9 +4,14 @@ import org.gradle.api.Project
 import org.gradle.kotlin.dsl.extra
 import org.gradle.kotlin.dsl.invoke
 import java.io.ByteArrayOutputStream
+import java.io.File
 import java.nio.file.Path
 
-fun Project.getPropertyByCommand(propName: String, cmdArgs: Array<String>): String {
+fun Project.getPropertyByCommand(
+    propName: String,
+    cmdArgs: Array<String>,
+    workingDir: File? = null,
+): String {
     val prop = project.findProperty(propName) as String?
 
     if (prop != null) {
@@ -17,6 +22,7 @@ fun Project.getPropertyByCommand(propName: String, cmdArgs: Array<String>): Stri
     val result = exec {
         commandLine(*cmdArgs)
         standardOutput = outputStream
+        workingDir?.let { this.workingDir = it }
     }
 
     val output = outputStream.toString()

kotlin-jupyter-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/build/versionUpdating.kt

@@ -31,9 +31,6 @@ private val Project.libParamValue get() = prop<String>("jupyter.lib.param.value"
 private val Project.prGithubUser get() = prop<String>("jupyter.github.user")
 private val Project.prGithubToken get() = prop<String>("jupyter.github.token")
 
-private val Project.githubRepoOwner get() = prop<String>("githubRepoUser")
-private val Project.githubRepoName get() = prop<String>("githubRepoName")
-
 @Serializable
 class NewPrData(
     val title: String,
@@ -89,11 +86,12 @@ fun ProjectWithInstallOptions.prepareKotlinVersionUpdateTasks() {
     val pushChangesTask = tasks.register("pushChanges") {
         dependsOn(updateLibraryParamTask)
 
+        val librariesDir = projectDir.resolve(librariesPath)
         fun execGit(vararg args: String, configure: ExecSpec.() -> Unit = {}): ExecResult {
             return exec {
                 this.executable = "git"
                 this.args = args.asList()
-                this.workingDir = projectDir
+                this.workingDir = librariesDir
 
                 configure()
             }
@@ -106,8 +104,13 @@ fun ProjectWithInstallOptions.prepareKotlinVersionUpdateTasks() {
             execGit("add", ".")
             execGit("commit", "-m", "[AUTO] Update library version")
 
-            val repoUrl = rootProject.property("pushRepoUrl") as String
-            execGit("push", "--force", "-u", repoUrl, getCurrentBranch() + ":refs/heads/" + updateLibBranchName!!) {
+            val repoUrl = rootProject.property("librariesRepoUrl") as String
+            val currentBranch = getPropertyByCommand(
+                "build.libraries.branch",
+                arrayOf("git", "rev-parse", "--abbrev-ref", "HEAD"),
+                librariesDir,
+            )
+            execGit("push", "--force", "-u", repoUrl, "$currentBranch:refs/heads/" + updateLibBranchName!!) {
                 this.standardOutput = object : OutputStream() {
                     override fun write(b: Int) { }
                 }
@@ -123,6 +126,7 @@ fun ProjectWithInstallOptions.prepareKotlinVersionUpdateTasks() {
         doLast {
             val user = rootProject.prGithubUser
             val password = rootProject.prGithubToken
+            val repoUserAndName = rootProject.property("librariesRepoUserAndName") as String
             fun githubRequest(
                 method: Method,
                 request: String,
@@ -141,9 +145,8 @@ fun ProjectWithInstallOptions.prepareKotlinVersionUpdateTasks() {
                 return response
             }
 
-            val fullRepo = "${rootProject.githubRepoOwner}/${rootProject.githubRepoName}"
             val prResponse = githubRequest(
-                Method.POST, "repos/$fullRepo/pulls",
+                Method.POST, "repos/$repoUserAndName/pulls",
                 Json.encodeToJsonElement(
                     NewPrData(
                         title = "Update `${rootProject.libName}` library to `${rootProject.libParamValue}`",
@@ -157,7 +160,7 @@ fun ProjectWithInstallOptions.prepareKotlinVersionUpdateTasks() {
 
             val prNumber = (prResponse.jsonObject["number"] as JsonPrimitive).int
             githubRequest(
-                Method.POST, "repos/$fullRepo/issues/$prNumber/labels",
+                Method.POST, "repos/$repoUserAndName/issues/$prNumber/labels",
                 Json.encodeToJsonElement(
                     SetLabelsData(listOf("no-changelog", "library-descriptors"))
                 )