Remove capsule launcher dependencies (#3395)

This commit replaces the Capsule launcher with "fat" jar approach that contains all core nextflow classes and dependencies. Main changes: * Remove the dependency on deprecated Capsule loader * Do not rely on Maven for Nextflow installation process * The `nextflow-*-one.jar` file includes all Nextflow core modules (not plugins) and dependencies * Use the Gradle ShadowJar plugin to create the `one` jar file * The `nextflow-*-all` is replaced by the `nextflow-*-dist` package that's the same as the `one` package with self-executing ability. * Enable the default Nextflow plugin install/update capability for `dist` package. Signed-off-by: Jorge Aguilera <jorge.aguilera@seqera.io> Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com> Co-authored-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com> Co-authored-by: Ben Sherman <bentshermann@gmail.com>
nextflow-io · Jul 30, 2024 · f15e424 · f15e424
1 parent 96ec4de
commit f15e424
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 124 deletions.
diff --git a/Makefile b/Makefile
@@ -95,10 +95,7 @@ upload:
 # Create self-contained distribution package
 #
 pack:
-	BUILD_PACK=1 ./gradlew packAll
-
-packCore:
-	BUILD_PACK=1 ./gradlew packCore
+	BUILD_PACK=1 ./gradlew pack
 
 #
 # Upload NF launcher to nextflow.io web site

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-24.06.0-edge
+24.07.0-edge
diff --git a/docs/config.md b/docs/config.md
@@ -1894,9 +1894,6 @@ The following environment variables control the configuration of the Nextflow ru
 `NXF_CHARLIECLOUD_CACHEDIR`
 : Directory where remote Charliecloud images are stored. When using a computing cluster it must be a shared folder accessible from all compute nodes.
 
-`NXF_CLASSPATH`
-: Allows the extension of the Java runtime classpath with extra JAR files or class folders.
-
 `NXF_CLOUDCACHE_PATH`
 : :::{versionadded} 23.07.0-edge
   :::

diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle
@@ -1,3 +1,6 @@
+plugins {
+    id "com.github.johnrengelman.shadow" version "8.1.1"
+}
 apply plugin: 'groovy'
 apply plugin: 'application'
 
@@ -72,3 +75,12 @@ application {
 run{
     args( (project.hasProperty("runCmd") ? project.findProperty("runCmd") : "set a cmd to run").split(' ') )
 }
+
+shadowJar {
+    archiveClassifier='one'
+    manifest {
+        attributes 'Main-Class': "$mainClassName"
+    }
+    mergeServiceFiles()
+    mergeGroovyExtensionModules()
+}
diff --git a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy
@@ -318,17 +318,17 @@ class PluginsFacade implements PluginStateListener {
     }
 
     void start( String pluginId ) {
-        if( isSelfContained() && defaultPlugins.hasPlugin(pluginId) ) {
-            log.debug "Plugin 'start' is not required in self-contained mode -- ignoring for plugin: $pluginId"
+        if( isEmbedded() && defaultPlugins.hasPlugin(pluginId) ) {
+            log.debug "Plugin 'start' is not required in embedded mode -- ignoring for plugin: $pluginId"
             return
         }
 
         start(PluginSpec.parse(pluginId, defaultPlugins))
     }
 
     void start(PluginSpec plugin) {
-        if( isSelfContained() && defaultPlugins.hasPlugin(plugin.id) ) {
-            log.debug "Plugin 'start' is not required in self-contained mode -- ignoring for plugin: $plugin.id"
+        if( isEmbedded() && defaultPlugins.hasPlugin(plugin.id) ) {
+            log.debug "Plugin 'start' is not required in embedded mode -- ignoring for plugin: $plugin.id"
             return
         }
 
@@ -346,19 +346,19 @@ class PluginsFacade implements PluginStateListener {
     }
 
     /**
-     * @return {@code true} when running in self-contained mode ie. the nextflow distribution
+     * @return {@code true} when running in embedded mode ie. the nextflow distribution
      * include also plugin libraries. When running is this mode, plugins should not be started
      * and cannot be updated. 
      */
-    protected boolean isSelfContained() {
-        return env.get('NXF_PACK')=='all' || embedded
+    protected boolean isEmbedded() {
+        return embedded
     }
 
     protected List<PluginSpec> pluginsRequirement(Map config) {
         def specs = parseConf(config)
-        if( isSelfContained() && specs ) {
+        if( isEmbedded() && specs ) {
             // custom plugins are not allowed for nextflow self-contained package
-            log.warn "Nextflow self-contained distribution allows only core plugins -- User config plugins will be ignored: ${specs.join(',')}"
+            log.warn "Nextflow embedded mode only core plugins -- User config plugins will be ignored: ${specs.join(',')}"
             return Collections.emptyList()
         }
         if( specs ) {
@@ -441,7 +441,7 @@ class PluginsFacade implements PluginStateListener {
     boolean startIfMissing(String pluginId) {
         if( env.NXF_PLUGINS_DEFAULT == 'false' )
             return false
-        if( isSelfContained() && defaultPlugins.hasPlugin(pluginId) )
+        if( isEmbedded() && defaultPlugins.hasPlugin(pluginId) )
             return false
 
         if( isStarted(pluginId) )

diff --git a/nextflow b/nextflow
@@ -15,7 +15,7 @@
 #  limitations under the License.
 
 [[ "$NXF_DEBUG" == 'x' ]] && set -x
-NXF_VER=${NXF_VER:-'24.06.0-edge'}
+NXF_VER=${NXF_VER:-'24.07.0-edge'}
 NXF_ORG=${NXF_ORG:-'nextflow-io'}
 NXF_HOME=${NXF_HOME:-$HOME/.nextflow}
 NXF_PROT=${NXF_PROT:-'https'}
@@ -26,6 +26,13 @@ NXF_CLI="$0 $@"
 NXF_CLI_OPTS=${NXF_CLI_OPTS:-}
 NXF_REMOTE_DEBUG_PORT=${NXF_REMOTE_DEBUG_PORT:-5005}
 
+NXF_LEGACY_LAUNCHER=1
+NXF_VER_MAJOR=$(echo $NXF_VER| cut -d'.' -f1)
+NXF_VER_MINOR=$(echo $NXF_VER| cut -d'.' -f2)
+if [[ NXF_VER_MAJOR -ge 24 ]] && [[ NXF_VER_MINOR -ge 7 ]]; then
+  unset NXF_LEGACY_LAUNCHER
+fi
+
 export NXF_CLI
 export NXF_ORG
 export NXF_HOME
@@ -249,18 +256,19 @@ NXF_JAR=${NXF_JAR:-nextflow-$NXF_VER-$NXF_PACK.jar}
 NXF_BIN=${NXF_BIN:-$NXF_DIST/$NXF_VER/$NXF_JAR}
 NXF_PATH=$(dirname "$NXF_BIN")
 NXF_URL=${NXF_URL:-$NXF_BASE/v$NXF_VER/$NXF_JAR}
-NXF_GRAB=${NXF_GRAB:-''}
-NXF_CLASSPATH=${NXF_CLASSPATH:-''}
 NXF_HOST=${HOSTNAME:-localhost}
 [[ $NXF_LAUNCHER ]] || NXF_LAUNCHER=${NXF_HOME}/tmp/launcher/nextflow-${NXF_PACK}_${NXF_VER}/${NXF_HOST}
+# both NXF_GRAB and NXF_CLASSPATH are not supported any more as of version 24.04.7-edge
+NXF_GRAB=${NXF_GRAB:-''}
+NXF_CLASSPATH=${NXF_CLASSPATH:-''}
 
 # Determine the path to this file
-if [[ $NXF_PACK = all ]]; then
+if [[ $NXF_PACK = dist ]]; then
     NXF_BIN=$(which "$0" 2>/dev/null)
     [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0"
 fi
 
-# use nextflow custom java home path 
+# use nextflow custom java home path
 if [[ "$NXF_JAVA_HOME" ]]; then
   JAVA_HOME="$NXF_JAVA_HOME"
   unset JAVA_CMD
@@ -346,10 +354,12 @@ if [[ $cmd == console ]]; then bg=1;
 else JAVA_OPTS+=(-Djava.awt.headless=true)
 fi
 
+if [[ $NXF_LEGACY_LAUNCHER ]]; then
+  [[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME")
+  [[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG)
+  [[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true)
+fi
 [[ "$JAVA_VER" =~ ^(21|22) ]] && [[ ! "$NXF_ENABLE_VIRTUAL_THREADS" ]] && NXF_ENABLE_VIRTUAL_THREADS=true
-[[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME")
-[[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG)
-[[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true)
 [[ "$cmd" != "run" && "$cmd" != "node" ]] && JAVA_OPTS+=(-XX:+TieredCompilation -XX:TieredStopAtLevel=1)
 [[ "$NXF_OPTS" ]] && JAVA_OPTS+=($NXF_OPTS)
 [[ "$NXF_CLASSPATH" ]] && export NXF_CLASSPATH
@@ -401,9 +411,14 @@ LAUNCH_FILE="${NXF_LAUNCHER}/classpath-$(env_md5)"
 if [ -s "$LAUNCH_FILE" ] && [ "$LAUNCH_FILE" -nt "$NXF_BIN" ] && [[ "$remote_debug" -ne 1 ]]; then
     declare -a launcher="($(cat "$LAUNCH_FILE"))"
 else
-    # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file
-    cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN"))
-    [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1
+    if [[ $NXF_LEGACY_LAUNCHER ]]; then
+      # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file
+      cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN"))
+      [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1
+    else
+      # otherwise parse the command and get the result classpath in the 'launcher' and save it to a file
+      cli=("\"$JAVA_CMD\"" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")
+    fi
 
     # first string between double quotes is the full path to java, also blank spaces are included
     # remainder string are arguments
@@ -445,6 +460,8 @@ else
       launcher+=("${cmd_tail[@]}")
     fi
 
+    # create the launch file only when using the legacy launcher (capsule)
+    if [[ $NXF_LEGACY_LAUNCHER ]]; then
     # Don't show errors if the LAUNCH_FILE can't be created
     if mkdir -p "${NXF_LAUNCHER}" 2>/dev/null; then
         STR=''
@@ -455,6 +472,7 @@ else
     else
         echo_yellow "Warning: Couldn't create cached classpath folder: $NXF_LAUNCHER -- Maybe NXF_HOME is not writable?"
     fi
+    fi
 
 fi
 

diff --git a/packing.gradle b/packing.gradle
@@ -1,5 +1,4 @@
 configurations {
-    capsule
     defaultCfg.extendsFrom api 
     //provided
     console.extendsFrom defaultCfg
@@ -17,9 +16,6 @@ dependencies {
     defaultCfg "org.apache.ivy:ivy:2.5.2"
     // default cfg = runtime + httpfs + amazon + tower client + wave client
     defaultCfg project(':nf-httpfs')
-    // Capsule manages the fat jar building process
-    capsule 'io.nextflow:capsule:1.1.1'
-    capsule 'io.nextflow:capsule-maven:1.0.3.2'
     console project(':plugins:nf-console')
     google  project(':plugins:nf-google')
     amazon  project(':plugins:nf-amazon')
@@ -92,121 +88,43 @@ protected coordinates( it ) {
 }
 
 /*
- * Default nextflow package. It contains the capsule loader
+ * Compile and pack all packages
  */
-task packOne(type: Jar) {
-    dependsOn configurations.capsule, configurations.defaultCfg
-    archiveFileName = "nextflow-${version}-one.jar"
-
-    from (configurations.capsule.collect { zipTree(it) })
-
-    // main manifest attributes
-    def deps = resolveDeps('defaultCfg')
-
-    manifest.attributes(
-                'Main-Class'        : 'NextflowLoader',
-                'Application-Name'  : 'nextflow',
-                'Application-Class' : mainClassName,
-                'Application-Version': version,
-                'Min-Java-Version'  : '1.8.0',
-                'Caplets'           : 'MavenCapsule',
-                'Dependencies'      : deps
-    )
-
-    // enable snapshot dependencies lookup
-    if( version.endsWith('-SNAPSHOT') ) {
-        manifest.attributes 'Allow-Snapshots': true
-        manifest.attributes 'Repositories': 'local https://oss.sonatype.org/content/repositories/snapshots central seqera'
-    }
-    else {
-        manifest.attributes 'Repositories': 'central seqera'
-    }
-
-    doLast {
-        ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: releaseDir, overwrite: true)
-        ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: nextflowDir, overwrite: true)
-        println "\n+ Nextflow package `ONE` copied to: $releaseDir"
-    }
-}
-
-task packAll(type: Jar) {
-    dependsOn configurations.capsule, configurations.defaultCfg
-    archiveFileName = "nextflow-${version}-all.jar"
-
-    from jar // embed our application jar
-    from (configurations.amazon + configurations.google + configurations.tower + configurations.wave)
-    from (configurations.capsule.collect { zipTree(it) })
-    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-
-    manifest.attributes( 'Main-Class'        : 'NextflowLoader',
-                          'Application-Name'  : 'nextflow-all',
-                          'Application-Class' : mainClassName,
-                          'Application-Version': version,
-                          'Min-Java-Version'  : '1.8.0'
-                          )
-
-    manifest.attributes('Main-Class': 'NextflowLoader', 'amazon')
-    manifest.attributes('Main-Class': 'NextflowLoader', 'google')
-
+task packOne( dependsOn: [clean, compile, ":nextflow:shadowJar"]) {
     doLast {
-        file(releaseDir).mkdir()
-        // cleanup
-        def source = file("$buildDir/libs/nextflow-${version}-all.jar")
-        def target = file("$releaseDir/nextflow-${version}-all"); target.delete()
-        // append the big jar
-        target.withOutputStream {
-            it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all')
-            it << new FileInputStream(source)
-        }
-        // execute permission
-        "chmod +x $target".execute()
-        // done
-        println "+ Nextflow package `ALL` copied to: $releaseDir\n"
+        def source = "modules/nextflow/build/libs/nextflow-${version}-one.jar"
+        ant.copy(file: source, todir: releaseDir, overwrite: true)
+        ant.copy(file: source, todir: nextflowDir, overwrite: true)
+        println "\n+ Nextflow package `ONE` copied to: $releaseDir/nextflow-${version}-one.jar"
     }
 }
 
-task packCore(type: Jar) {
-    dependsOn configurations.capsule, configurations.defaultCfg
-    archiveFileName = "nextflow-${version}-core.jar"
-
-    from jar // embed our application jar
-    from (configurations.defaultCfg)
-    from (configurations.capsule.collect { zipTree(it) })
-    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-
-    manifest.attributes( 'Main-Class'        : 'NextflowLoader',
-            'Application-Name'  : 'nextflow-core',
-            'Application-Class' : mainClassName,
-            'Application-Version': version,
-            'Min-Java-Version'  : '1.8.0'
-    )
+task packDist( dependsOn: [clean, compile, ":nextflow:shadowJar"]) {
 
     doLast {
-        file(releaseDir).mkdir()
+        file(releaseDir).mkdirs()
         // cleanup
-        def source = file("$buildDir/libs/nextflow-${version}-core.jar")
-        def target = file("$releaseDir/nextflow-${version}-core"); target.delete()
+        def source = file("modules/nextflow/build/libs/nextflow-${version}-one.jar")
+        def target = file("$releaseDir/nextflow-${version}-dist"); target.delete()
         // append the big jar
         target.withOutputStream {
-            it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all')
+            it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=dist')
             it << new FileInputStream(source)
         }
         // execute permission
         "chmod +x $target".execute()
         // done
-        println "+ Nextflow package `CORE` copied to: $releaseDir\n"
+        println "+ Nextflow package `ALL` copied to: $target\n"
     }
 }
 
-
 /*
  * Compile and pack all packages
  */
-task pack( dependsOn: [packOne, packAll]) {
+task pack( dependsOn: [packOne, packDist]) {
 
 }
 
-
 task deploy( type: Exec, dependsOn: [clean, compile, pack]) {
 
     def temp = File.createTempFile('upload',null)