Remove libraries submodule and add Gradle task to checkout libraries from the remote

Author: ileasile

.gitignore

@@ -14,11 +14,15 @@
 # Folders with build files
 out/
 /build/
+/build-tools/*/.idea/
 /build-tools/*/build/
 /*jupyter*/*/build/
 /api-examples/*/build/
 /teamcity-artifacts/
 
+# Folder with library descriptors
+libraries/
+
 # Gradle caches and internal files
 .gradle/
 

.gitmodules

@@ -11,8 +11,7 @@ repositories {
 }
 
 dependencies {
-    // implementation(projects.commonDependencies)
-    implementation(libs.jupyter.commonDependencies)
+    implementation(projects.commonDependencies)
     api(libs.bundles.allGradlePlugins)
 }
 

build-tools/build-plugin/build.gradle.kts

@@ -52,8 +52,7 @@ class RootSettingsExtension(
     val readmeFile: File = project.rootDir.resolve("docs").resolve("README.md")
     val readmeStubFile: File = project.rootDir.resolve("docs").resolve("README-STUB.md")
 
-    val librariesDir: File = project.rootDir.resolve("libraries")
-    val librariesPropertiesFile: File = librariesDir.resolve(".properties")
+    val librariesDir: File = BUILD_LIBRARIES.localLibrariesDir
 
     val localInstallDir: File = run {
         val installPath = project.typedProperty<String?>("installPath")

build-tools/build-plugin/src/build/RootSettingsExtension.kt

@@ -0,0 +1,38 @@
+package build
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.Project
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.OutputDirectory
+import org.gradle.api.tasks.TaskAction
+import org.gradle.kotlin.dsl.create
+import org.gradle.kotlin.dsl.property
+import java.io.File
+
+abstract class UpdateLibrariesTask : DefaultTask() {
+    @get:Input
+    val latestCommitHash: Property<String> = project.objects.property<String>().apply {
+        set(project.provider { BUILD_LIBRARIES.latestCommitOnDefaultBranch })
+    }
+
+    @get:OutputDirectory
+    val librariesDir: Property<File> = project.objects.property<File>().apply {
+        set(BUILD_LIBRARIES.localLibrariesDir)
+    }
+
+    @TaskAction
+    fun update() {
+        val latestSha = latestCommitHash.get()
+        if (BUILD_LIBRARIES.checkIfRefUpToDate(latestSha)) return
+        BUILD_LIBRARIES.downloadLibraries(latestSha)
+    }
+
+    companion object {
+        fun getOrCreate(project: Project): UpdateLibrariesTask {
+            val task = project.tasks.findByName(UPDATE_LIBRARIES_TASK_NAME)
+            if (task != null) return task as UpdateLibrariesTask
+            return project.tasks.create<UpdateLibrariesTask>(UPDATE_LIBRARIES_TASK_NAME)
+        }
+    }
+}

build-tools/build-plugin/src/build/UpdateLibrariesTask.kt

@@ -41,6 +41,7 @@ import java.io.OutputStream
 
 @Serializable
 class NewPrData(
+    @Suppress("unused")
     val title: String,
     @Suppress("unused")
     val head: String,
@@ -75,6 +76,8 @@ internal class KernelBuildConfigurator(private val project: Project) {
         setupVersionsPlugin()
         setupKtLintForAllProjects()
 
+        setupUpdateLibrariesTask()
+
         println("##teamcity[buildNumber '${opts.pyPackageVersion}']")
         println("##teamcity[setParameter name='mavenVersion' value='${opts.mavenVersion}']")
 
@@ -138,6 +141,10 @@ internal class KernelBuildConfigurator(private val project: Project) {
         }
     }
 
+    private fun setupUpdateLibrariesTask() {
+        UpdateLibrariesTask.getOrCreate(project)
+    }
+
     private fun createCleanTasks() {
         listOf(true, false).forEach { local ->
             val dir = if (local) opts.localInstallDir else opts.distribBuildDir
@@ -156,9 +163,10 @@ internal class KernelBuildConfigurator(private val project: Project) {
         val groupName = if (local) LOCAL_INSTALL_GROUP else DISTRIBUTION_GROUP
         val cleanDirTask = tasks.getByName(makeTaskName(opts.cleanInstallDirTaskPrefix, local))
         val shadowJar = tasks.getByName(SHADOW_JAR_TASK)
+        val updateLibrariesTask = tasks.named(UPDATE_LIBRARIES_TASK_NAME)
 
         tasks.register<Copy>(makeTaskName(opts.copyLibrariesTaskPrefix, local)) {
-            dependsOn(cleanDirTask)
+            dependsOn(cleanDirTask, updateLibrariesTask)
             group = groupName
             from(opts.librariesDir)
             into(mainInstallPath.resolve(opts.librariesDir))
@@ -610,7 +618,7 @@ internal class KernelBuildConfigurator(private val project: Project) {
     }
 
     private fun updateLibraryParam(libName: String, paramName: String, paramValue: String) {
-        val libFile = project.file(opts.librariesDir).resolve("$libName.json")
+        val libFile = project.file(opts.librariesDir).resolve(BUILD_LIBRARIES.descriptorFileName(libName))
         val libText = libFile.readText()
         val paramRegex = Regex("""^([ \t]*"$paramName"[ \t]*:[ \t]*")(.*)("[ \t]*,?)$""", RegexOption.MULTILINE)
         val newText = libText.replace(paramRegex, "$1$paramValue$3")
@@ -625,7 +633,8 @@ internal class KernelBuildConfigurator(private val project: Project) {
             opts.jvmTargetForSnippets?.let {
                 add("jvmTargetForSnippets" to it)
             }
-            val librariesProperties = readProperties(opts.librariesPropertiesFile)
+            BUILD_LIBRARIES.downloadLatestPropertiesFile()
+            val librariesProperties = readProperties(BUILD_LIBRARIES.localPropertiesFile)
             add("librariesFormatVersion" to librariesProperties["formatVersion"].orEmpty())
         }
 
@@ -636,6 +645,8 @@ internal class KernelBuildConfigurator(private val project: Project) {
     }
 
     private fun prepareReadmeTasks() {
+        val updateTask = tasks.named(UPDATE_LIBRARIES_TASK_NAME)
+
         val kotlinVersion = project.defaultVersionCatalog.versions.devKotlin
 
         val readmeFile = opts.readmeFile
@@ -648,6 +659,8 @@ internal class KernelBuildConfigurator(private val project: Project) {
             inputs.dir(librariesDir)
             inputs.property("kotlinVersion", kotlinVersion)
             inputs.property("projectRepoUrl", opts.projectRepoUrl)
+
+            dependsOn(updateTask)
         }
 
         val generateReadme = tasks.register(GENERATE_README_TASK) {

build-tools/build-plugin/src/build/kernelBuild.kt

@@ -36,3 +36,5 @@ const val UNINSTALL_TASK = "uninstall"
 
 const val MAKE_CHANGES_PR_TASK = "makeChangesPR"
 const val PREPARE_PACKAGE_TASK = "preparePackage"
+
+const val UPDATE_LIBRARIES_TASK_NAME = "updateLibraryDescriptors"

build-tools/build-plugin/src/build/taskNames.kt

@@ -7,6 +7,7 @@ import org.gradle.api.plugins.ExtensionContainer
 import org.gradle.kotlin.dsl.exclude
 import org.gradle.kotlin.dsl.extra
 import org.gradle.tooling.BuildException
+import org.jetbrains.kotlinx.jupyter.common.LibraryDescriptorsManager
 import java.io.File
 import java.nio.file.Path
 import java.util.Optional
@@ -15,6 +16,8 @@ import kotlin.reflect.KProperty
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
+val BUILD_LIBRARIES = LibraryDescriptorsManager.getInstance()
+
 fun makeTaskName(prefix: String, local: Boolean) = prefix + (if (local) "Local" else "Distrib")
 
 fun makeDirs(dir: File) {

build-tools/build-plugin/src/build/util.kt

@@ -103,6 +103,8 @@ tasks {
             classpath = files(shadowJar.get()) + classpath
         }
 
+        dependsOn(updateLibraryDescriptors.get())
+
         systemProperties = mutableMapOf(
             "junit.jupiter.displayname.generator.default" to "org.junit.jupiter.api.DisplayNameGenerator\$ReplaceUnderscores",
 

build.gradle.kts

@@ -2,6 +2,7 @@ package org.jetbrains.kotlinx.jupyter.common
 
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.jsonPrimitive
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import java.io.File
@@ -32,17 +33,29 @@ class LibraryDescriptorsManager private constructor(
     val userLibrariesDir = userSettingsDir.resolve(userPath)
     val userCacheDir = userSettingsDir.resolve("cache")
     val localLibrariesDir = File(localPath)
+    val defaultBranch = "master"
+    val latestCommitOnDefaultBranch by lazy {
+        getLatestCommitToLibraries(defaultBranch)!!.first
+    }
+
     fun homeLibrariesDir(homeDir: File? = null) = (homeDir ?: File("")).resolve(homePath)
 
     val localPropertiesFile = localLibrariesDir.resolve(PROPERTIES_FILE)
+    val commitHashFile by lazy {
+        localLibrariesDir.resolve(COMMIT_HASH_FILE).also { file ->
+            if (!file.exists()) {
+                file.createDirsAndWrite()
+            }
+        }
+    }
 
     fun descriptorFileName(name: String) = "$name.$DESCRIPTOR_EXTENSION"
 
     fun isLibraryDescriptor(file: File): Boolean {
         return file.isFile && file.name.endsWith(".$DESCRIPTOR_EXTENSION")
     }
 
-    fun getLatestCommitToLibraries(ref: String, sinceTimestamp: String?): Pair<String, String>? {
+    fun getLatestCommitToLibraries(ref: String, sinceTimestamp: String? = null): Pair<String, String>? {
         return catchAll {
             var url = "$apiPrefix/commits?path=$remotePath&sha=$ref"
             if (sinceTimestamp != null) {
@@ -69,18 +82,69 @@ class LibraryDescriptorsManager private constructor(
     fun downloadLibraryDescriptor(ref: String, name: String): String {
         val url = "$apiPrefix/contents/$remotePath/$name.$DESCRIPTOR_EXTENSION?ref=$ref"
         logger.info("Requesting library descriptor at $url")
-        val response = getHttp(url).jsonObject
-
-        val downloadURL = (response["download_url"] as JsonPrimitive).content
-        val res = getHttp(downloadURL)
-        return res.text
+        return downloadSingleFile(url)
     }
 
     fun checkRefExistence(ref: String): Boolean {
         val response = getHttp("$apiPrefix/contents/$remotePath?ref=$ref")
         return response.status.successful
     }
 
+    fun checkIfRefUpToDate(remoteRef: String): Boolean {
+        if (!commitHashFile.exists()) return false
+        val localRef = commitHashFile.readText()
+        return localRef == remoteRef
+    }
+
+    fun downloadLibraries(ref: String) {
+        localLibrariesDir.mkdirs()
+
+        val url = "$apiPrefix/contents/$remotePath?ref=$ref"
+        logger.info("Requesting library descriptors at $url")
+        val response = getHttp(url).jsonArray
+
+        for (item in response) {
+            item as JsonObject
+            if (item["type"]?.jsonPrimitive?.content != "file") continue
+
+            val fileName = item["name"]!!.jsonPrimitive.content
+            if (!fileName.endsWith(".$DESCRIPTOR_EXTENSION")) continue
+
+            val downloadUrl = item["download_url"]!!.jsonPrimitive.content
+            val descriptorResponse = getHttp(downloadUrl)
+
+            val descriptorText = descriptorResponse.text
+            val file = localLibrariesDir.resolve(fileName)
+            file.writeText(descriptorText)
+        }
+
+        saveLocalRef(ref)
+    }
+
+    fun downloadLatestPropertiesFile() {
+        val ref = latestCommitOnDefaultBranch
+        val url = "$apiPrefix/contents/$remotePath/$PROPERTIES_FILE?ref=$ref"
+        logger.info("Requesting $PROPERTIES_FILE file at $url")
+        val text = downloadSingleFile(url)
+        localPropertiesFile.createDirsAndWrite(text)
+    }
+
+    private fun downloadSingleFile(contentsApiUrl: String): String {
+        val response = getHttp(contentsApiUrl).jsonObject
+        val downloadUrl = response["download_url"]!!.jsonPrimitive.content
+        val res = getHttp(downloadUrl)
+        return res.text
+    }
+
+    private fun saveLocalRef(ref: String) {
+        commitHashFile.createDirsAndWrite(ref)
+    }
+
+    private fun File.createDirsAndWrite(text: String = "") {
+        parentFile.mkdirs()
+        writeText(text)
+    }
+
     private fun <T> catchAll(message: String = "", body: () -> T): T? = try {
         body()
     } catch (e: Throwable) {
@@ -92,6 +156,7 @@ class LibraryDescriptorsManager private constructor(
         private const val GITHUB_API_HOST = "api.github.com"
         private const val DESCRIPTOR_EXTENSION = "json"
         private const val PROPERTIES_FILE = ".properties"
+        private const val COMMIT_HASH_FILE = "commit_sha"
 
         fun getInstance(
             logger: Logger = LoggerFactory.getLogger(LibraryDescriptorsManager::class.java),