build.gradle

/*
 * Copyright (c) 2013-2018, Centre for Genomic Regulation (CRG).
 * Copyright (c) 2013-2018, Paolo Di Tommaso and the respective authors.
 *
 *   This file is part of 'Nextflow'.
 *
 *   Nextflow is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   Nextflow is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with Nextflow.  If not, see <http://www.gnu.org/licenses/>.
 */

buildscript {
    repositories {
        mavenCentral()
    }
    dependencies {
        classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.5.1"
    }
}

plugins {
    id "com.vanniktech.dependency.graph.generator" version "0.2.0"
}

apply plugin: 'groovy'
apply plugin: "jacoco"
apply plugin: 'idea'
apply plugin: 'io.codearte.nexus-staging'   // See https://github.com/Codearte/gradle-nexus-staging-plugin/

// Add ability to test with upcoming versions of Groovy
def groovyVer = System.getenv('CI_GROOVY_VERSION')
if (groovyVer) {
    logger.lifecycle "Overriden Groovy dependency to use $groovyVer"
    allprojects {
        repositories {
            maven {
                name 'JFrog OSS snapshot repo'
                url 'https://oss.jfrog.org/oss-snapshot-local/'
            }
        }
    }

    configurations.all {
        resolutionStrategy.eachDependency { DependencyResolveDetails details ->
            if (details.requested.group == 'org.codehaus.groovy') {
                details.useVersion(groovyVer)
                println ">> Overriding $details.requested with version: $groovyVer"
            }
        }
    }
}


ext {
    mainClassName = 'nextflow.cli.Launcher'
}

allprojects {
    apply plugin: 'java'
    apply plugin: 'idea'

    version = '0.29.0-RC1'
    group = 'io.nextflow'

    sourceCompatibility = 1.8
    targetCompatibility = 1.8

    idea {
        module.inheritOutputDirs = true
    }

    repositories {
        jcenter()
        mavenCentral()
        maven { url 'http://uk.maven.org/maven2' }
        maven { url 'http://dl.bintray.com/paolo-ditommaso/maven'}
        maven { url 'https://repo.eclipse.org/content/groups/releases' }
        maven { url 'https://oss.sonatype.org/content/repositories/snapshots' }
    }

    configurations {
        // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies
        all*.exclude group: 'org.codehaus.groovy', module: 'groovy-all'
    }

    dependencies {
        // test configuration
        testCompile ("cglib:cglib-nodep:2.2")
        testCompile ("org.objenesis:objenesis:1.4")
        testCompile ("org.spockframework:spock-core:1.1-groovy-2.4")
        testCompile 'com.google.jimfs:jimfs:1.0'
        testCompile 'org.subethamail:subethasmtp:3.1.7'
        // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sec:module_replacement
        modules {
            module("commons-logging:commons-logging") { replacedBy("org.slf4j:jcl-over-slf4j") }
        }
    }

    // this is required due to this IDEA bug
    // https://youtrack.jetbrains.com/issue/IDEA-129282
    sourceSets {
        main {
            output.resourcesDir = 'build/classes/main'
        }
    }

    // Disable strict javadoc checks
    // See http://blog.joda.org/2014/02/turning-off-doclint-in-jdk-8-javadoc.html
    if (JavaVersion.current().isJava8Compatible()) {
        tasks.withType(Javadoc) {
            options.addStringOption('Xdoclint:none', '-quiet')
        }
    }

}

configurations {
    capsule
    //provided
    ignite.extendsFrom runtime
    console.extendsFrom runtime
    gdocLibs.extendsFrom runtime
}


dependencies {
    compile(project(':nxf-commons'))
    compile(project(':nxf-httpfs'))
    compile "org.codehaus.groovy:groovy:2.4.15"
    compile "org.codehaus.groovy:groovy-nio:2.4.15"
    compile "org.codehaus.groovy:groovy-json:2.4.15"
    compile "org.codehaus.groovy:groovy-templates:2.4.15"
    compile "org.slf4j:jcl-over-slf4j:1.7.21"
    compile "org.slf4j:jul-to-slf4j:1.7.21"
    compile "org.slf4j:log4j-over-slf4j:1.7.21"
    compile "ch.qos.logback:logback-classic:1.1.10"
    compile "ch.qos.logback:logback-core:1.1.10"
    compile "org.codehaus.gpars:gpars:1.2.1"
    compile("ch.grengine:grengine:1.2.1") { exclude group: 'org.codehaus.groovy', module: 'groovy-all' }
    compile "commons-lang:commons-lang:2.6"
    compile "com.beust:jcommander:1.35"
    compile("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' }
    compile('org.iq80.leveldb:leveldb:0.7')
    compile('org.eclipse.jgit:org.eclipse.jgit:4.8.0.201706111038-r')
    compile('io.nextflow:nxf-s3fs:1.0.3') {
        exclude group: 'org.apache.tika', module: 'tika-core'
        exclude group: 'com.amazonaws', module: 'aws-java-sdk-s3'
    }
    compile ('com.amazonaws:aws-java-sdk-s3:1.11.172')
    compile ('com.amazonaws:aws-java-sdk-ec2:1.11.172')
    compile ('com.amazonaws:aws-java-sdk-batch:1.11.172')
    compile ('javax.mail:mail:1.4.7')
    compile ('org.yaml:snakeyaml:1.18')
    compile ('org.jsoup:jsoup:1.11.2')

    // include Ivy at runtime in order to have Grape @Grab work correctly
    runtime "org.apache.ivy:ivy:2.3.0"

    // library used to compile but not included in the final jar

    // Capsule manages the fat jar building process
    capsule(project(':my-capsule'))

    ignite(project(':nxf-ignite')) {
        exclude module: 'nextflow'
    }

    console(project(':nxf-console')) {
        exclude module: 'nextflow'
    }

    // Documentation required libraries
    gdocLibs 'org.fusesource.jansi:jansi:1.11'
    gdocLibs "org.codehaus.groovy:groovy-groovydoc:2.4.15"
    gdocLibs "org.codehaus.groovy:groovy-ant:2.4.15"

}

// patched as described here
// http://forums.gradle.org/gradle/topics/gradle_task_groovydoc_failing_with_noclassdeffounderror
groovydoc {
    groovyClasspath = project.configurations.gdocLibs
    includes  = ["nextflow/**"]
}

//sourceSets.main.compileClasspath += configurations.provided
sourceSets.main.java.srcDirs = []
sourceSets.main.groovy.srcDirs = ['src/main/java', 'src/main/groovy']
sourceSets.main.resources.srcDirs = ['src/main/resources']

compileGroovy {
    options.compilerArgs = ['-XDignore.symbol.file']
}


/*
 * http://www.gradle.org/docs/current/dsl/org.gradle.plugins.ide.idea.model.IdeaModule.html
 * http://stackoverflow.com/questions/20421681/provided-dependencies-using-gradle-jetgradle-and-intellij-idea-13
 * http://forums.gradle.org/gradle/topics/gradle_2_0_idea_plugin_adding_a_custom_configuration_to_scope_throws_classcastexception
 */
//idea {
//    module {
//        scopes.PROVIDED.plus += [configurations.provided]
//    }
//}

/*
 * Update the build timestamp in the source source file
 */
task buildInfo { doLast {

    def file = new File('./src/main/groovy/nextflow/Const.groovy')
    def buildNum = 0
    def src = file.text
    src.find(/APP_BUILDNUM *= *([0-9]*)/) { buildNum = it[1]?.toInteger()+1 }
    src = src.replaceAll('APP_VER *= *"[0-9a-zA-Z_\\-\\.]+"', "APP_VER = \"${version}\"" as String)
    src = src.replaceAll('APP_TIMESTAMP *= *[0-9]*', "APP_TIMESTAMP = ${System.currentTimeMillis()}" as String)
    if( buildNum ) {
        src = src.replaceAll('APP_BUILDNUM *= *[0-9]*', "APP_BUILDNUM = ${buildNum}" as String)
    }
    else {
        println "WARN: Unable to find current build number"
    }
    file.text = src

    // -- update 'nextflow' wrapper
    file = new File('nextflow')
    src = file.text
    src = src.replaceAll(/NXF_VER\=\$\{NXF_VER:-'.*'\}/, 'NXF_VER=\\${NXF_VER:-\'' + version + '\'}')
    file.text = src

    // -- update sphynx
    def major = version.split(/\./)[0..1].join('.')
    file = new File('docs/conf.py')
    src = file.text
    src = src.replaceAll(/version *= *'[0-9a-zA-Z_\-\.]+'/, "version = '$major'" as String)
    src = src.replaceAll(/release *= *'[0-9a-zA-Z_\-\.]+'/, "release = '$version'" as String)
    file.text = src

    // -- update dockerfile
    file = new File('docker/Dockerfile')
    src = file.text
    src = src.replaceAll(/releases\/v[0-9a-zA-Z_\-\.]+\//, "releases/v$version/" as String)
    file.text = src
}}

String bytesToHex(byte[] bytes) {
    StringBuffer result = new StringBuffer();
    for (byte byt : bytes) result.append(Integer.toString((byt & 0xff) + 0x100, 16).substring(1));
    return result.toString();
}

task makeDigest { doLast {
    byte[] digest
    String  str = file('nextflow').text
    // create sha1
    digest = java.security.MessageDigest.getInstance("SHA1").digest(str.getBytes())
    file('nextflow.sha1').text = new BigInteger(1, digest).toString(16) + '\n'
    // create sha-256
    digest = java.security.MessageDigest.getInstance("SHA-256").digest(str.getBytes())
    file('nextflow.sha256').text = bytesToHex(digest) + '\n'
    // create md5
    digest = java.security.MessageDigest.getInstance("MD5").digest(str.getBytes())
    file('nextflow.md5').text = bytesToHex(digest) + '\n'
}}

/*
 * Compile sources and copies all libs to target directory
 */
task compile {
    dependsOn buildInfo
    dependsOn allprojects.jar
}

/*
 * Creates a self-contained *fat* jar shipping all required dependencies
 *
 * @link https://github.com/puniverse/capsule
 */

ext.ossrhUserName = project.findProperty('ossrhUserName')
ext.ossrhPassword = project.findProperty('ossrhPassword')
ext.ossrhFullName = project.findProperty('ossrhFullName')
ext.ossrhEmail = project.findProperty('ossrhEmail')

ext.homeDir = System.properties['user.home']
ext.nextflowDir = "$homeDir/.nextflow/framework/$version"
ext.releaseDir = "$buildDir/releases"
ext.s3CmdOpts="--acl public-read --storage-class REDUCED_REDUNDANCY --region eu-west-1"
// `signing.keyId` property needs to be defined in the `gradle.properties` file
ext.enableSignArchives = project.findProperty('signing.keyId')

ext.deployable = []
ext.deployable << project(':')
ext.deployable << project(':nxf-commons')
ext.deployable << project(':nxf-ignite')
ext.deployable << project(':nxf-console')
ext.deployable << project(':nxf-httpfs')

protected error(String message) {
    logger.error message
    throw new StopExecutionException()
}

protected resolveDeps( String configName, String... x ) {

    final deps = [] as Set
    final config = configurations.getByName(configName)
    final incoming = config.getIncoming()

    deps << incoming.getResolutionResult().root.moduleVersion.toString()
    incoming.getDependencies().each{ deps << coordinates(it) }

    config.getResolvedConfiguration().getResolvedArtifacts().each{ deps << coordinates(it) }

    if( x )
        deps.addAll(x as List)

    logger.info ">> Dependencies for configuration: $configName"
    deps.sort().each { logger.info " - $it" }

    return deps.collect{ "$it(*:*)" }.join(' ')
}

protected coordinates( it ) {
    if( it instanceof Dependency )
        return "${it.group}:${it.name}:${it.version}".toString()

    if( it instanceof ResolvedArtifact ) {
        def result = it.moduleVersion.id.toString()
        if( it.classifier ) result += ":${it.classifier}"
        return result
    }

    throw new IllegalArgumentException("Not a valid dependency object [${it?.class?.name}]: $it")
}

/*
 * Default nextflow package. It contains the capsule loader
 */
task packOne(type: Jar) {
    dependsOn ':my-capsule:jar', configurations.runtime
    archiveName = "nextflow-${version}-one.jar"

    from (configurations.capsule.collect { zipTree(it) })

    // main manifest attributes
    def deps = resolveDeps('runtime')

    manifest.attributes(
                'Main-Class'        : 'CapsuleLoader',
                'Application-Name'  : 'nextflow',
                'Application-Class' : mainClassName,
                'Application-Version': version,
                'Min-Java-Version'  : '1.7.0',
                'Caplets'           : 'MavenCapsule',
                'Dependencies'      : deps
    )

    // enable snapshot dependencies lookup
    if( version.endsWith('-SNAPSHOT') ) {
        manifest.attributes 'Allow-Snapshots': true
        manifest.attributes 'Repositories': 'local https://oss.sonatype.org/content/repositories/snapshots central'
    }

    /*
     * `console` dependencies
     */
    deps = resolveDeps('console')
    manifest.attributes('Application-Class' : 'nextflow.ui.console.Nextflow', 'console' )
    manifest.attributes('Dependencies': deps, 'console')

    /*
     * `ignite` dependencies
     */
    deps = resolveDeps('ignite')
    manifest.attributes('Dependencies': deps, 'ignite')

    doLast {
        ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: releaseDir, overwrite: true)
        ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: nextflowDir, overwrite: true)
        println "\n+ Nextflow package `ONE` copied to: $releaseDir"
    }
}

task packAll(type: Jar) {
    dependsOn ':my-capsule:jar', configurations.runtime
    archiveName = "nextflow-${version}-all.jar"

    from jar // embed our application jar
    from (configurations.ignite)
    from (configurations.capsule.collect { zipTree(it) })

    manifest.attributes( 'Main-Class'        : 'CapsuleLoader',
                          'Application-Name'  : 'nextflow-all',
                          'Application-Class' : mainClassName,
                          'Application-Version': version,
                          'Min-Java-Version'  : '1.7.0'
                          )

    manifest.attributes('Main-Class': 'CapsuleLoader', 'ignite')

    doLast {
        file(releaseDir).mkdir()
        // cleanup
        def source = file("$buildDir/libs/nextflow-${version}-all.jar")
        def target = file("$releaseDir/nextflow-${version}-all"); target.delete()
        // append the big jar
        target.withOutputStream {
            it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all')
            it << new FileInputStream(source)
        }
        // done
        println "+ Nextflow package `ALL` copied to: $releaseDir\n"
    }
}


/*
 * Compile and pack all packages
 */
task pack( dependsOn: [packOne, packAll]) {

}


task deploy( type: Exec, dependsOn: [clean, pack, deployable.uploadArchives, makeDigest]) {

    def temp = File.createTempFile('upload',null)
    temp.deleteOnExit()
    def files = []

    doFirst {
        def path = new File(releaseDir)
        if( !path.exists() ) error("Releases path does not exists: $path")
        path.eachFile {
            if( it.name.startsWith("nextflow-$version"))
                files << it
        }

        if( !files ) error("Can't find any file to upload -- Check path: $path")
        files << file('nextflow').absoluteFile
        files << file('nextflow.sha1').absoluteFile
        files << file('nextflow.sha256').absoluteFile
        files << file('nextflow.md5').absoluteFile

        println "Uploading artifacts: "
        files.each { println "- $it"}

        def script = []
        script << "export AWS_ACCESS_KEY_ID=${System.env.NXF_AWS_ACCESS}"
        script << "export AWS_SECRET_ACCESS_KEY=${System.env.NXF_AWS_SECRET}"
        script.addAll( files.collect { "aws s3 cp ${it} s3://www.nextflow.io/releases/v${version}/${it.name} ${s3CmdOpts}"})

        temp.text = script.join('\n')
    }

    commandLine 'bash', '-e', temp.absolutePath
}

task installLauncher(type: Copy, dependsOn: ['pack']) {
    from "$releaseDir/nextflow-$version-one.jar"
    into "$homeDir/.nextflow/framework/$version/"
}

/*
 * build, tag and publish a and new docker packaged nextflow release
 */
task dockerImage(type: Exec) {

    def temp = File.createTempFile('upload',null)
    temp.deleteOnExit()
    temp.text =  """\
    cd docker
    make release version=$version
    """.stripIndent()

    commandLine 'bash', '-e', temp.absolutePath
}

/*
 * Build a docker container image with the current snapshot
 * DO NOT CALL IT DIRECTLY! Use the `make dockerPack` command instead
 */
task dockerPack(type: Exec, dependsOn: ['pack']) {

    def source = new File("$releaseDir/nextflow-$version-one.jar")
    def target = new File("$buildDir/docker/.nextflow/framework/$version/")
    def dockerBase = new File("$buildDir/docker")
    if( dockerBase.exists() ) dockerBase.deleteDir()
    dockerBase.mkdirs()
    def dockerFile = new File("$buildDir/docker/Dockerfile")
    dockerFile.text = """
    FROM openjdk:8-jre-alpine
    RUN apk update && apk add bash && apk add coreutils && apk add curl
    COPY .nextflow /.nextflow
    COPY nextflow /usr/local/bin/nextflow
    COPY entry.sh /usr/local/bin/entry.sh
    COPY dist/docker /usr/local/bin/docker
    ENV NXF_HOME=/.nextflow
    ENV NXF_OPTS='-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap'
    RUN touch /.nextflow/dockerized && rm -rf /.nextflow/launch-classpath
    RUN chmod +x /usr/local/bin/nextflow /usr/local/bin/entry.sh
    RUN nextflow info
    ENTRYPOINT ["/usr/local/bin/entry.sh"]
    """

    def temp = File.createTempFile('upload',null)
    temp.deleteOnExit()
    temp.text =  """\
    mkdir -p $target
    cp $source $target
    cp nextflow $buildDir/docker
    cp docker/entry.sh $buildDir/docker
    cd $buildDir/docker  
    mkdir -p dist
    curl -fsSLO https://get.docker.com/builds/Linux/x86_64/docker-17.03.1-ce.tgz && tar --strip-components=1 -xvzf docker-17.03.1-ce.tgz -C dist
    NXF_HOME=\$PWD/.nextflow nextflow info
    docker build -t nextflow/nextflow:$version .
    """.stripIndent()

    commandLine 'bash', '-e', temp.absolutePath
}

/*
 * Tag and upload the release
 */
task release(type: Exec, dependsOn: [pack, dockerImage]) {

    def launcherFile = file('nextflow').absoluteFile
    def launcherSha1 = file('nextflow.sha1').absoluteFile
    def launcherSha256 = file('nextflow.sha256').absoluteFile
    def nextflowAllFile = file("$releaseDir/nextflow-${version}-all")

    def snapshot = version ==~ /^.+(-RC\d+|\-SNAPSHOT)$/
    def isLatest = !snapshot
    def cmd =  """\
    # tag the release
    git push || exit \$?
    (git tag -a v$version -m 'Tagging version $version [release]' -f && git push origin v$version -f) || exit \$?
    sleep 1
    github-release release -u nextflow-io -r nextflow -t v$version --pre-release
    github-release upload -u nextflow-io -r nextflow -t v$version --file ${launcherFile} --name ${launcherFile.name}
    github-release upload -u nextflow-io -r nextflow -t v$version --file ${nextflowAllFile} --name ${nextflowAllFile.name}
    """.stripIndent()

    if( isLatest )
        cmd += """
        # publish the script as the latest
        export AWS_ACCESS_KEY_ID=${System.env.NXF_AWS_ACCESS}
        export AWS_SECRET_ACCESS_KEY=${System.env.NXF_AWS_SECRET}
        aws s3 cp $launcherFile s3://www.nextflow.io/releases/latest/nextflow $s3CmdOpts
        aws s3 cp $launcherSha1 s3://www.nextflow.io/releases/latest/nextflow.sha1 $s3CmdOpts
        aws s3 cp $launcherSha256 s3://www.nextflow.io/releases/latest/nextflow.sha256 $s3CmdOpts
        aws s3 cp $launcherSha256 s3://www.nextflow.io/releases/latest/nextflow.md5 $s3CmdOpts
        """.stripIndent()

    def temp = File.createTempFile('upload',null)
    temp.deleteOnExit()
    temp.text = cmd
    commandLine 'bash', '-e', temp.absolutePath
}


/*
 * Save the runtime classpath
 */
task exportClasspath {
    dependsOn configurations.ignite, configurations.console

    def home = System.getProperty('user.home')
    def all = configurations.ignite + configurations.console
    def libs = all.collect { File file -> /* println file.canonicalPath.replace(home, '$HOME'); */ file.canonicalPath;  }
    new File('.launch.classpath').text = libs.join(':')
}

/*
 * Configure Nextflow staging plugin -- https://github.com/Codearte/gradle-nexus-staging-plugin
 * It adds the tasks
 * - closeRepository
 * - promoteRepository
 */
nexusStaging {
    packageGroup = 'io.nextflow'
    delayBetweenRetriesInMillis = 10_000
}


/*
 * Maven central deployment
 * http://central.sonatype.org/pages/gradle.html
 */
configure(deployable) {
    apply plugin: 'maven'
    apply plugin: 'signing'

    task javadocJar(type: Jar) {
        classifier = 'javadoc'
        from groovydoc
    }

    task sourcesJar(type: Jar) {
        classifier = 'sources'
        from sourceSets.main.allSource
    }

    artifacts {
        archives javadocJar, sourcesJar
    }

    signing {
        sign configurations.archives
    }

    uploadArchives {
        repositories {
            mavenDeployer {
                beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }

                repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") {
                    authentication(userName: ossrhUserName, password: ossrhPassword)
                }

                snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") {
                    authentication(userName: ossrhUserName, password: ossrhPassword)
                }

                pom.project {
                    name 'Nextflow'
                    packaging 'jar'
                    // optionally artifactId can be defined here
                    description 'A DSL modelled around the UNIX pipe concept, that simplifies writing parallel and scalable pipelines in a portable manner'
                    url 'http://www.nextflow.io'

                    scm {
                        connection 'scm:git:https://github.com/nextflow-io/nextflow'
                        developerConnection 'scm:git:git@github.com:nextflow-io/nextflow.git'
                        url 'https://github.com/nextflow-io/nextflow'
                    }

                    licenses {
                        license {
                            name 'GPLv3'
                            url 'http://www.gnu.org/licenses/gpl-3.0.txt'
                        }
                    }

                    developers {
                        developer {
                            id ossrhUserName
                            name ossrhFullName
                            email ossrhEmail
                        }
                    }

                }
            }
        }
    }

    signArchives.onlyIf { enableSignArchives }
}

//test {
//    testLogging {
//        events "passed", "skipped", "failed", "standardOut", "standardError"
//    }
//}


//task allFiles {
//    doLast {
//        configurations.compile.files.sort{ it.name }.each { file ->
//            println file.name
//        }
//    }
//}