feat: java/graalvm 21+ support

- feat: support for jvm21+/gvm jdk 23, latest truffle
- feat: flag support for gvm, native arch, use new `-Os` optimization mode
- feat: initial transitive native image flag support
- fix: support up to graalvm/jdk 23 (latest)
- fix: don't use `gu` tool for modern graalvm versions
- fix: coordinate change for `shadow` plugin (`com.gradleup.shadow`)
- fix: build with `--add-modules=jdk.unsupported` where needed
- fix: use jdk21 to run the tests (needed for `Unsafe.ensureInitialized`)
- fix: truffle svm dependency is required after graalvm `24.0.0`
- fix: warnings for gvm flag usage, renamed truffle svm macro
- chore: bump graalvm → `24.1.0`
- chore: bump shadow → `8.3.5`
- chore: update gradle → `8.12`

Signed-off-by: Sam Gammon <sam@elide.dev>

Author: sgammon

buildSrc/src/main/kotlin/BuildInfo.kt

@@ -80,6 +80,10 @@ open class BuildInfo(project: Project) {
 
   val isReleaseBuild: Boolean by lazy { java.lang.Boolean.getBoolean("releaseBuild") }
 
+  val isNativeArch: Boolean by lazy { java.lang.Boolean.getBoolean("nativeArch") }
+
+  val isEnableOracleGraalvm: Boolean by lazy { java.lang.Boolean.getBoolean("oracleGraalvm") }
+
   val hasMuslToolchain: Boolean by lazy {
     // see "install musl" in .circleci/jobs/BuildNativeJob.pkl
     File(System.getProperty("user.home"), "staticdeps/bin/x86_64-linux-musl-gcc").exists()

buildSrc/src/main/kotlin/InstallGraalVm.kt

@@ -58,10 +58,16 @@ constructor(
         if (os.isMacOsX) distroDir.resolve("Contents/Home/bin") else distroDir.resolve("bin")
 
       println("Installing native-image into $distroDir")
-      execOperations.exec {
-        val executableName = if (os.isWindows) "gu.cmd" else "gu"
-        executable = distroBinDir.resolve(executableName).toString()
-        args("install", "--no-progress", "native-image")
+      val gvmVersionMajor =
+        requireNotNull(graalVm.get().version.split(".").first().toIntOrNull()) {
+          "Invalid GraalVM JDK version: ${graalVm.get().graalVmJdkVersion}"
+        }
+      if (gvmVersionMajor < 24) {
+        execOperations.exec {
+          val executableName = if (os.isWindows) "gu.cmd" else "gu"
+          executable = distroBinDir.resolve(executableName).toString()
+          args("install", "--no-progress", "native-image")
+        }
       }
 
       println("Creating symlink ${graalVm.get().installDir} for $distroDir")

buildSrc/src/main/kotlin/pklFatJar.gradle.kts

@@ -24,7 +24,7 @@ import org.gradle.kotlin.dsl.*
 plugins {
   `java-library`
   `maven-publish`
-  id("com.github.johnrengelman.shadow")
+  id("com.gradleup.shadow")
 }
 
 // make fat Jar available to other subprojects
@@ -40,8 +40,6 @@ val relocations =
   mapOf(
     // pkl-core dependencies
     "org.antlr.v4." to "org.pkl.thirdparty.antlr.v4.",
-    "com.oracle.truffle" to "org.pkl.thirdparty.truffle",
-    "org.graalvm." to "org.pkl.thirdparty.graalvm.",
     "org.organicdesign.fp." to "org.pkl.thirdparty.paguro.",
     "org.snakeyaml.engine." to "org.pkl.thirdparty.snakeyaml.engine.",
     "org.msgpack." to "org.pkl.thirdparty.msgpack.",
@@ -71,7 +69,11 @@ val relocations =
     "com.squareup.kotlinpoet." to "org.pkl.thirdparty.kotlinpoet.",
   )
 
-val nonRelocations = listOf("com/oracle/truffle/")
+val nonRelocations =
+  listOf(
+    "com/oracle/truffle/",
+    "org/graalvm/",
+  )
 
 tasks.shadowJar {
   inputs.property("relocations", relocations)

buildSrc/src/main/kotlin/pklJavaLibrary.gradle.kts

@@ -86,3 +86,27 @@ tasks.compileJava {
   // (see https://graalvm.slack.com/archives/CNQSB2DHD/p1712380902746829)
   options.compilerArgs.add("-Atruffle.dsl.SuppressWarnings=truffle-limit")
 }
+
+//
+// JPMS Support
+//
+
+fun Project.ifJpms(block: Project.() -> Unit) {
+  // if current java is over 17...
+  if (JavaVersion.current().majorVersion.toInt() >= 17) {
+    block()
+  }
+}
+
+val jpmsJavacArgs =
+  listOf(
+    "--add-modules=jdk.unsupported",
+  )
+
+val jpmsJvmArgs = jpmsJavacArgs.plus(listOf())
+
+tasks.withType<JavaCompile>().configureEach {
+  ifJpms { options.compilerArgumentProviders.add(CommandLineArgumentProvider { jpmsJavacArgs }) }
+}
+
+tasks.withType<JavaExec>().configureEach { ifJpms { jvmArgs(jpmsJvmArgs) } }

gradle/libs.versions.toml

@@ -9,14 +9,14 @@ geantyref = "1.+"
 googleJavaFormat = "1.21.0"
 # must not use `+` because used in download URL
 # 23.1.x requires JDK 20+
-graalVm = "23.0.6"
-graalVmJdkVersion = "17.0.12"
+graalVm = "24.1.1"
+graalVmJdkVersion = "23.0.1"
 # slightly hacky but convenient place so we remember to update the checksum
-graalVmSha256-macos-x64 = "3ecac1471f3fa95a56c5b75c65db9e60ac4551f56eda09eb9da95e6049ea77d7"
-graalVmSha256-macos-aarch64 = "4cdfdc6c9395f6773efcd191b6605f1b7c8e1b78ab900ab5cff34720a3feffc5"
-graalVmSha256-linux-x64 = "b6f3dace24cf1960ec790216f4c86f00d4f43df64e4e8b548f6382f04894713f"
-graalVmSha256-linux-aarch64 = "bd991d486b92deb74337b881e0f13a764c9c1e90fc358819080f7321fa5175e8"
-graalVmSha256-windows-x64 = "8b978e56dddc0edc60db99794b56975740d9c52293b31549cfc3f7516fc18b43"
+graalVmSha256-macos-x64 = "539699d8ff4979623bc7bdf8282ac6f76cd2560f47d14ec5438bada24f136f96"
+graalVmSha256-macos-aarch64 = "c00a7a62ce453aa026bff65e5a18c63464f725c01e5a71771856226928ba5b0f"
+graalVmSha256-linux-x64 = "46ec9582ebe114f93470403f2cc123238ac0c7982129c358af7d8e1de52dd663"
+graalVmSha256-linux-aarch64 = "1835a98b87c439c8c654d97956c22d409855952e5560a8127f56c50f3f919d7d"
+graalVmSha256-windows-x64 = "e758646504cfaf23cf218a22691ad70491f3196448a77d03d78e50dff2145533"
 ideaExtPlugin = "1.1.9"
 javaPoet = "0.+"
 javaxInject = "1"
@@ -44,7 +44,7 @@ msgpack = "0.9.8"
 nexusPublishPlugin = "2.0.0"
 nuValidator = "20.+"
 paguro = "3.+"
-shadowPlugin = "8.1.1"
+shadowPlugin = "8.3.5"
 slf4j = "1.+"
 snakeYaml = "2.+"
 spotlessPlugin = "6.25.0"
@@ -88,14 +88,16 @@ nuValidator = { group = "nu.validator", name = "validator", version.ref = "nuVal
 # to be replaced with https://github.com/usethesource/capsule or https://github.com/lacuna/bifurcan
 paguro = { group = "org.organicdesign", name = "Paguro", version.ref = "paguro" }
 pklConfigJavaAll025 = { group = "org.pkl-lang", name = "pkl-config-java-all", version = "0.25.0" }
-shadowPlugin = { group = "com.github.johnrengelman", name = "shadow", version.ref = "shadowPlugin" }
+shadowPlugin = { group = "com.gradleup.shadow", name = "shadow-gradle-plugin", version.ref = "shadowPlugin" }
 slf4jApi = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j" }
 slf4jSimple = { group = "org.slf4j", name = "slf4j-simple", version.ref = "slf4j" }
 snakeYaml = { group = "org.snakeyaml", name = "snakeyaml-engine", version.ref = "snakeYaml" }
 spotlessPlugin = { group = "com.diffplug.spotless", name = "spotless-plugin-gradle", version.ref = "spotlessPlugin"}
 svm = { group = "org.graalvm.nativeimage", name = "svm", version.ref = "graalVm" }
 truffleApi = { group = "org.graalvm.truffle", name = "truffle-api", version.ref = "graalVm" }
 truffleDslProcessor = { group = "org.graalvm.truffle", name = "truffle-dsl-processor", version.ref = "graalVm" }
+truffleSvm = { group = "org.graalvm.nativeimage", name = "truffle-runtime-svm", version.ref = "graalVm" }
+truffleRuntime = { group = "org.graalvm.truffle", name = "truffle-runtime", version.ref = "graalVm" }
 wiremock = { group = "org.wiremock", name = "wiremock", version.ref = "wiremock" }
 
 [plugins] # ordered alphabetically
@@ -104,4 +106,4 @@ ideaExt = { id = "org.jetbrains.gradle.plugin.idea-ext", version.ref = "ideaExtP
 jmh = { id = "me.champeau.jmh", version.ref = "jmhPlugin" }
 kotlinxSerialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" }
 nexusPublish = { id = "io.github.gradle-nexus.publish-plugin", version.ref = "nexusPublishPlugin" }
-shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadowPlugin" }
+shadow = { id = "com.gradleup.shadow", version.ref = "shadowPlugin" }

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=57dafb5c2622c6cc08b993c85b7c06956a2f53536432a30ead46166dbca0f1e9
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-bin.zip
+distributionSha256Sum=7a00d51fb93147819aab76024feece20b6b84e420694101f276be952e08bef03
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -86,8 +86,7 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
-' "$PWD" ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum

pkl-cli/pkl-cli.gradle.kts

@@ -50,12 +50,16 @@ val stagedWindowsAmd64Executable: Configuration by configurations.creating
 
 dependencies {
   compileOnly(libs.svm)
+  compileOnly(libs.truffleSvm)
+  implementation(libs.truffleRuntime)
 
   // CliEvaluator exposes PClass
   api(projects.pklCore)
   // CliEvaluatorOptions exposes CliBaseOptions
   api(projects.pklCommonsCli)
 
+  compileOnly(libs.graalSdk)
+
   implementation(projects.pklCommons)
   implementation(libs.jansi)
   implementation(libs.jlineReader)
@@ -107,12 +111,19 @@ val javaExecutable by
     inJar.set(tasks.shadowJar.flatMap { it.archiveFile })
     outJar.set(layout.buildDirectory.file("executable/jpkl"))
 
+    jvmArgs.addAll("--add-modules=jdk.unsupported")
+
     // uncomment for debugging
     // jvmArgs.addAll("-ea", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")
   }
 
 val testJavaExecutable by
   tasks.registering(Test::class) {
+    javaLauncher =
+      javaToolchains.launcherFor {
+        languageVersion = JavaLanguageVersion.of(21)
+        vendor = JvmVendorSpec.GRAAL_VM
+      }
     testClassesDirs = tasks.test.get().testClassesDirs
     classpath =
       // compiled test classes
@@ -179,7 +190,7 @@ fun Exec.configureExecutable(
   executable = "${graalVm.baseDir}/bin/$nativeImageCommandName"
 
   // JARs to exclude from the class path for the native-image build.
-  val exclusions = listOf(libs.truffleApi, libs.graalSdk).map { it.get().module.name }
+  val exclusions = listOf(libs.graalSdk).map { it.get().module.name }
   // https://www.graalvm.org/22.0/reference-manual/native-image/Options/
   argumentProviders.add(
     CommandLineArgumentProvider {
@@ -190,13 +201,15 @@ fun Exec.configureExecutable(
         // needed for messagepack-java (see https://github.com/msgpack/msgpack-java/issues/600)
         add("--initialize-at-run-time=org.msgpack.core.buffer.DirectBufferAccess")
         add("--no-fallback")
+        // must be emitted before any experimental options are used
+        add("-H:+UnlockExperimentalVMOptions")
         add("-H:IncludeResources=org/pkl/core/stdlib/.*\\.pkl")
         add("-H:IncludeResources=org/jline/utils/.*")
         add("-H:IncludeResourceBundles=org.pkl.core.errorMessages")
         add("-H:IncludeResources=org/pkl/commons/cli/PklCARoots.pem")
-        add("--macro:truffle")
         add("-H:Class=org.pkl.cli.Main")
-        add("-H:Name=${outputFile.get().asFile.name}")
+        add("-o")
+        add(outputFile.get().asFile.name)
         // the actual limit (currently) used by native-image is this number + 1400 (idea is to
         // compensate for Truffle's own nodes)
         add("-H:MaxRuntimeCompileMethods=1800")
@@ -210,8 +223,20 @@ fun Exec.configureExecutable(
         // executable
         if (!buildInfo.isReleaseBuild) {
           add("-Ob")
+        } else {
+          add("-Os")
+        }
+        if (buildInfo.isNativeArch) {
+          add("-march=native")
+        } else {
+          add("-march=compatibility")
+        }
+        if (buildInfo.isEnableOracleGraalvm) {
+          add("--gc=G1")
+          add("--enable-sbom=cyclonedx")
+        } else {
+          add("--gc=serial")
         }
-        add("-march=compatibility")
         // native-image rejects non-existing class path entries -> filter
         add("--class-path")
         val pathInput =

pkl-core/src/main/resources/META-INF/native-image/org.pkl-lang/native-image.properties

@@ -0,0 +1,2 @@
+
+Args = --initialize-at-build-time=org.pkl.core