build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import groovy.transform.Memoized
import java.util.regex.Matcher
import java.util.regex.Pattern

buildscript {
  repositories {
    gradlePluginPortal()
  }
  dependencies {
    classpath 'com.github.johnrengelman:shadow:8.1.1'
    classpath 'com.palantir.baseline:gradle-baseline-java:4.42.0'
    // com.palantir.baseline:gradle-baseline-java:4.42.0 (the last version supporting Java 8) pulls
    // in an old version of the errorprone, which doesn't work w/ Gradle 8, so bump errorpone as
    // well.
    classpath "net.ltgt.gradle:gradle-errorprone-plugin:3.1.0"

    classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.13.0'
    classpath 'gradle.plugin.org.inferred:gradle-processors:3.7.0'
    classpath 'me.champeau.jmh:jmh-gradle-plugin:0.7.2'
    classpath 'gradle.plugin.io.morethan.jmhreport:gradle-jmh-report:0.9.6'
    classpath "com.github.alisiikh:gradle-scalastyle-plugin:3.5.0"
    classpath 'com.palantir.gradle.revapi:gradle-revapi:1.7.0'
    classpath 'com.gorylenko.gradle-git-properties:gradle-git-properties:2.4.2'
    classpath 'com.palantir.gradle.gitversion:gradle-git-version:3.0.0'
    classpath 'org.openapitools:openapi-generator-gradle-plugin:6.6.0'
  }
}

String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
String sparkVersionsString = System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")
List<String> sparkVersions = sparkVersionsString != null && !sparkVersionsString.isEmpty() ? sparkVersionsString.split(",") : []

try {
  // apply these plugins in a try-catch block so that we can handle cases without .git directory
  apply plugin: 'com.palantir.git-version'
} catch (Exception e) {
  project.logger.error(e.getMessage())
}

if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
  project.ext.jdkVersion = '8'
  project.ext.extraJvmArgs = []
} else if (JavaVersion.current() == JavaVersion.VERSION_11) {
  project.ext.jdkVersion = '11'
  project.ext.extraJvmArgs = []
} else if (JavaVersion.current() == JavaVersion.VERSION_17) {
  project.ext.jdkVersion = '17'
  project.ext.extraJvmArgs = ["-XX:+IgnoreUnrecognizedVMOptions",
                              "--add-opens", "java.base/java.io=ALL-UNNAMED",
                              "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED",
                              "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED",
                              "--add-opens", "java.base/java.lang=ALL-UNNAMED",
                              "--add-opens", "java.base/java.math=ALL-UNNAMED",
                              "--add-opens", "java.base/java.net=ALL-UNNAMED",
                              "--add-opens", "java.base/java.nio=ALL-UNNAMED",
                              "--add-opens", "java.base/java.text=ALL-UNNAMED",
                              "--add-opens", "java.base/java.time=ALL-UNNAMED",
                              "--add-opens", "java.base/java.util.concurrent.atomic=ALL-UNNAMED",
                              "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED",
                              "--add-opens", "java.base/java.util.regex=ALL-UNNAMED",
                              "--add-opens", "java.base/java.util=ALL-UNNAMED",
                              "--add-opens", "java.base/jdk.internal.ref=ALL-UNNAMED",
                              "--add-opens", "java.base/jdk.internal.reflect=ALL-UNNAMED",
                              "--add-opens", "java.sql/java.sql=ALL-UNNAMED",
                              "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED",
                              "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED",
                              "--add-opens", "java.base/sun.nio.cs=ALL-UNNAMED",
                              "--add-opens", "java.base/sun.security.action=ALL-UNNAMED",
                              "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED"]
} else {
  throw new GradleException("This build must be run with JDK 8 or 11 or 17 but was executed with JDK " + JavaVersion.current())
}

tasks.withType(AbstractArchiveTask).configureEach {
  preserveFileTimestamps = false
  reproducibleFileOrder = true
}

apply plugin: 'com.gorylenko.gradle-git-properties'
// git properties file for the root project for adding to the source tarball
gitProperties {
  gitPropertiesName = 'iceberg-build.properties'
  gitPropertiesResourceDir = file("${rootDir}/build")
  extProperty = 'gitProps'
  failOnNoGitDirectory = true
  keys = ['git.branch', 'git.build.version', 'git.closest.tag.name','git.commit.id.abbrev', 'git.commit.id',
          'git.commit.message.short', 'git.commit.time', 'git.tags']
}
generateGitProperties.outputs.upToDateWhen { false }

if (file("${rootDir}/iceberg-build.properties").exists()) {
  tasks.register('buildInfo', Exec) {
    project.logger.info('Using build info from iceberg-build.properties')
    commandLine 'cp', "${rootDir}/iceberg-build.properties", 'build/iceberg-build.properties'
  }
} else {
  tasks.register('buildInfo') {
    project.logger.info('Generating iceberg-build.properties from git')
    dependsOn generateGitProperties
  }
}

def projectVersion = getProjectVersion()
final REVAPI_PROJECTS = ["iceberg-api", "iceberg-core", "iceberg-parquet", "iceberg-orc", "iceberg-common", "iceberg-data"]

allprojects {
  group = "org.apache.iceberg"
  version = projectVersion
  repositories {
    mavenCentral()
    mavenLocal()
  }
}

subprojects {
  if (it.name == 'iceberg-bom') {
    // the BOM does not build anything, the code below expects "source code"
    return
  }

  apply plugin: 'java-library'

  if (project.name in REVAPI_PROJECTS) {
    apply plugin: 'com.palantir.revapi'
    revapi {
      oldGroup = project.group
      oldName = project.name
      oldVersion = "1.5.0"
    }

    tasks.register('showDeprecationRulesOnRevApiFailure') {
      doLast {
        throw new RuntimeException("==================================================================================" +
                "\nAPI/ABI breaks detected.\n" +
                "Adding RevAPI breaks should only be done after going through a deprecation cycle." +
                "\nPlease make sure to follow the deprecation rules defined in\n" +
                "https://github.com/apache/iceberg/blob/main/CONTRIBUTING.md#semantic-versioning.\n" +
                "==================================================================================")
      }
      onlyIf {
        tasks.revapi.state.failure != null
      }
    }

    tasks.configureEach { rootTask ->
      if (rootTask.name == 'revapi') {
        rootTask.finalizedBy showDeprecationRulesOnRevApiFailure
      }
    }
  }

  configurations {
    testImplementation.extendsFrom compileOnly

    compileClasspath {
      // do not exclude Guava so the bundle project can reference classes.
      if (project.name != 'iceberg-bundled-guava') {
        exclude group: 'com.google.guava', module: 'guava'
      }
      // contains a copy of Guava
      exclude group: 'org.apache.spark', module: 'spark-network-common_2.12'
    }

    all {
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.mortbay.jetty'
      exclude group: 'com.sun.jersey'
      exclude group: 'com.sun.jersey.contribs'
      exclude group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm'
    }

    testArtifacts
  }

  compileJava {
    options.encoding = "UTF-8"
  }

  compileTestJava {
    options.encoding = "UTF-8"
  }

  javadoc {
    options.encoding = 'UTF-8'
  }

  sourceCompatibility = '1.8'
  targetCompatibility = '1.8'

  dependencies {
    implementation libs.slf4j.api

    testImplementation libs.junit.vintage.engine
    testImplementation libs.junit.jupiter
    testImplementation libs.junit.jupiter.engine
    testImplementation libs.slf4j.simple
    testImplementation libs.mockito.core
    testImplementation libs.mockito.inline
    testImplementation libs.assertj.core
  }

  test {
    def logDir = "${rootDir}/build/testlogs"
    def logFile = "${logDir}/${project.name}.log"
    mkdir("${logDir}")
    delete("${logFile}")
    def buildLog = new File(logFile)
    addTestOutputListener(new TestOutputListener() {
      def lastDescriptor
      @Override
      void onOutput(TestDescriptor testDescriptor, TestOutputEvent testOutputEvent) {
        if (lastDescriptor != testDescriptor) {
          buildLog << "--------\n- Test log for: "<< testDescriptor << "\n--------\n"
          lastDescriptor = testDescriptor
        }
        buildLog << testOutputEvent.destination << " " << testOutputEvent.message
      }
    })

    maxHeapSize = "1500m"

    jvmArgs += project.property('extraJvmArgs')

    testLogging {
      events "failed"
      exceptionFormat "full"
    }
  }

  plugins.withType(ScalaPlugin.class) {
    tasks.withType(ScalaCompile.class) {
      scalaCompileOptions.keepAliveMode.set(KeepAliveMode.DAEMON)
    }
  }
}

project(':iceberg-bundled-guava') {
  apply plugin: 'com.github.johnrengelman.shadow'

  tasks.jar.dependsOn tasks.shadowJar

  dependencies {
    compileOnly(libs.guava.guava) {
      exclude group: 'com.google.code.findbugs'
      // may be LGPL - use ALv2 findbugs-annotations instead
      exclude group: 'com.google.errorprone'
      exclude group: 'com.google.j2objc'
    }
  }

  shadowJar {
    archiveClassifier.set(null)
    configurations = [project.configurations.compileClasspath]
    zip64 true

    // include the LICENSE and NOTICE files for the shaded Jar
    from(projectDir) {
      include 'LICENSE'
      include 'NOTICE'
    }

    dependencies {
      exclude(dependency('org.slf4j:slf4j-api'))
      exclude(dependency('org.checkerframework:checker-qual'))
    }

    relocate 'com.google.common', 'org.apache.iceberg.relocated.com.google.common'

    minimize()
  }

  jar {
    archiveClassifier.set('empty')
  }
}

project(':iceberg-api') {
  test {
    useJUnitPlatform()
  }

  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    compileOnly libs.errorprone.annotations
    compileOnly libs.findbugs.jsr305
    testImplementation libs.avro.avro
    testImplementation libs.esotericsoftware.kryo
    testImplementation libs.awaitility
  }

  tasks.processTestResources.dependsOn rootProject.tasks.buildInfo

  // Workaround to prevent:
  // Task ':iceberg-api:processTestResources' uses this output of task
  // ':spotlessInternalRegisterDependencies' without declaring an explicit or implicit dependency.
  // This can lead to incorrect results being produced, depending on what order the tasks are executed.
  rootProject.tasks.configureEach { rootTask ->
    if (rootTask.name == 'spotlessInternalRegisterDependencies') {
      tasks.processTestResources.dependsOn rootTask
    }
  }

  sourceSets {
    test {
      resources {
        srcDir "${rootDir}/build"
      }
    }
  }
}

project(':iceberg-common') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
  }
}

project(':iceberg-core') {
  test {
    useJUnitPlatform()
  }
  dependencies {
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    annotationProcessor libs.immutables.value
    compileOnly libs.immutables.value

    implementation(libs.avro.avro) {
      exclude group: 'org.tukaani' // xz compression is not supported
    }

    implementation libs.aircompressor
    implementation libs.httpcomponents.httpclient5
    implementation platform(libs.jackson.bom)
    implementation "com.fasterxml.jackson.core:jackson-core"
    implementation "com.fasterxml.jackson.core:jackson-databind"
    implementation libs.caffeine
    implementation libs.roaringbitmap
    compileOnly(libs.hadoop2.client) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }

    testImplementation libs.jetty.servlet
    testImplementation libs.jetty.server
    testImplementation libs.mockserver.netty
    testImplementation libs.mockserver.client.java
    testImplementation libs.sqlite.jdbc
    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation libs.esotericsoftware.kryo
    testImplementation libs.guava.testlib
    testImplementation libs.awaitility
  }
}

project(':iceberg-data') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    compileOnly project(':iceberg-parquet')
    compileOnly project(':iceberg-orc')
    compileOnly(libs.hadoop2.common) {
      exclude group: 'commons-beanutils'
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }

    implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
      exclude group: 'org.apache.hadoop'
      exclude group: 'commons-lang'
      // These artifacts are shaded and included in the orc-core fat jar
      exclude group: 'com.google.protobuf', module: 'protobuf-java'
      exclude group: 'org.apache.hive', module: 'hive-storage-api'
    }

    implementation(libs.parquet.avro) {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    compileOnly libs.avro.avro

    testImplementation(libs.hadoop2.client) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
  }

  test {
    useJUnitPlatform()
    // Only for TestSplitScan as of Gradle 5.0+
    maxHeapSize '1500m'
  }
}

project(':iceberg-aliyun') {
  test {
    useJUnitPlatform()
  }
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    implementation project(':iceberg-common')

    compileOnly libs.aliyun.sdk.oss
    compileOnly libs.jaxb.api
    compileOnly libs.activation
    compileOnly libs.jaxb.runtime
    compileOnly(libs.hadoop2.common) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'javax.servlet', module: 'servlet-api'
      exclude group: 'com.google.code.gson', module: 'gson'
    }

    testImplementation platform(libs.jackson.bom)
    testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml"
    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation libs.spring.web
    testImplementation(libs.spring.boot.starter.jetty) {
      exclude module: 'logback-classic'
      exclude group: 'org.eclipse.jetty.websocket', module: 'javax-websocket-server-impl'
      exclude group: 'org.eclipse.jetty.websocket', module: 'websocket-server'
    }
    testImplementation(libs.spring.boot.starter.web) {
      exclude module: 'logback-classic'
      exclude module: 'spring-boot-starter-logging'
    }
  }
}

project(':iceberg-aws') {
  test {
    useJUnitPlatform()
  }
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')
    annotationProcessor libs.immutables.value
    compileOnly libs.immutables.value
    implementation libs.caffeine
    implementation platform(libs.jackson.bom)
    implementation "com.fasterxml.jackson.core:jackson-databind"
    implementation "com.fasterxml.jackson.core:jackson-core"

    compileOnly(platform(libs.awssdk.bom))
    compileOnly(libs.awssdk.s3accessgrants)
    compileOnly("software.amazon.awssdk:url-connection-client")
    compileOnly("software.amazon.awssdk:apache-client")
    compileOnly("software.amazon.awssdk:auth")
    compileOnly("software.amazon.awssdk:s3")
    compileOnly("software.amazon.awssdk:kms")
    compileOnly("software.amazon.awssdk:glue")
    compileOnly("software.amazon.awssdk:sts")
    compileOnly("software.amazon.awssdk:dynamodb")
    compileOnly("software.amazon.awssdk:lakeformation")

    compileOnly(libs.hadoop2.common) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'javax.servlet', module: 'servlet-api'
      exclude group: 'com.google.code.gson', module: 'gson'
    }

    compileOnly libs.httpcomponents.httpclient5

    testImplementation(platform(libs.awssdk.bom))
    testImplementation("software.amazon.awssdk:iam")
    testImplementation("software.amazon.awssdk:s3control")
    testImplementation("software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin")
    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation(libs.s3mock.junit5) {
      exclude module: "spring-boot-starter-logging"
      exclude module: "logback-classic"
      exclude group: 'junit'
    }
    testImplementation libs.esotericsoftware.kryo
    testImplementation libs.sqlite.jdbc
    testImplementation libs.testcontainers
    testImplementation libs.httpcomponents.httpclient5
    testImplementation libs.mockserver.netty
    testImplementation libs.mockserver.client.java
    testImplementation libs.jaxb.api
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
    testImplementation libs.awaitility
  }

  sourceSets {
    integration {
      java.srcDir "$projectDir/src/integration/java"
      resources.srcDir "$projectDir/src/integration/resources"
      compileClasspath += main.output + test.output
      runtimeClasspath += main.output + test.output
    }
  }

  configurations {
    integrationImplementation.extendsFrom testImplementation
    integrationRuntime.extendsFrom testRuntimeOnly
  }

  task integrationTest(type: Test) {
    useJUnitPlatform()
    testClassesDirs = sourceSets.integration.output.classesDirs
    classpath = sourceSets.integration.runtimeClasspath
    jvmArgs += project.property('extraJvmArgs')
  }

  def s3SignerSpec = "$projectDir/src/main/resources/s3-signer-open-api.yaml"
  tasks.register('validateS3SignerSpec', org.openapitools.generator.gradle.plugin.tasks.ValidateTask) {
    inputSpec.set(s3SignerSpec)
    recommend.set(true)
  }
  check.dependsOn('validateS3SignerSpec')
}

project(':iceberg-azure') {
  test {
    useJUnitPlatform()
  }

  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')

    compileOnly platform(libs.azuresdk.bom)
    compileOnly "com.azure:azure-storage-file-datalake"
    compileOnly "com.azure:azure-identity"

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation libs.esotericsoftware.kryo
    testImplementation libs.testcontainers
  }
}

project(':iceberg-delta-lake') {
  // use integration test since we can take advantages of spark 3.3 to read datafiles of delta lake table
  // and create some tests involving sql query.
  test {
    useJUnitPlatform()
  }
  configurations {
    integrationImplementation.extendsFrom testImplementation
    integrationRuntime.extendsFrom testRuntimeOnly
  }

  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')
    implementation project(':iceberg-parquet')
    implementation platform(libs.jackson.bom)
    implementation "com.fasterxml.jackson.core:jackson-databind"
    annotationProcessor libs.immutables.value
    compileOnly libs.immutables.value

    compileOnly "io.delta:delta-standalone_${scalaVersion}:${libs.versions.delta.standalone.get()}"

    compileOnly(libs.hadoop2.common) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'javax.servlet', module: 'servlet-api'
      exclude group: 'com.google.code.gson', module: 'gson'
    }

    // The newest version of delta-core uses Spark 3.5.*. Since its only for test, we do
    // not need to include older version of delta-core
    if (sparkVersions.contains("3.5")) {
      integrationImplementation "io.delta:delta-spark_${scalaVersion}:${libs.versions.delta.spark.get()}"
      integrationImplementation project(path: ":iceberg-spark:iceberg-spark-3.5_${scalaVersion}")
      integrationImplementation(libs.hadoop2.minicluster) {
        exclude group: 'org.apache.avro', module: 'avro'
        // to make sure netty libs only come from project(':iceberg-arrow')
        exclude group: 'io.netty', module: 'netty-buffer'
        exclude group: 'io.netty', module: 'netty-common'
      }
      integrationImplementation project(path: ':iceberg-hive-metastore')
      integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
      integrationImplementation("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark.hive35.get()}") {
        exclude group: 'org.apache.avro', module: 'avro'
        exclude group: 'org.apache.arrow'
        exclude group: 'org.apache.parquet'
        // to make sure netty libs only come from project(':iceberg-arrow')
        exclude group: 'io.netty', module: 'netty-buffer'
        exclude group: 'io.netty', module: 'netty-common'
        exclude group: 'org.roaringbitmap'
      }
    }
  }

  // The newest version of delta-core uses Spark 3.5.*. The integration test should only be built
  // if iceberg-spark-3.5 is available
  if (sparkVersions.contains("3.5")) {
    sourceSets {
      integration {
        java.srcDir "$projectDir/src/integration/java"
        resources.srcDir "$projectDir/src/integration/resources"
        compileClasspath += main.output + test.output
        runtimeClasspath += main.output + test.output
      }
    }

    task integrationTest(type: Test) {
      useJUnitPlatform()
      testClassesDirs = sourceSets.integration.output.classesDirs
      classpath = sourceSets.integration.runtimeClasspath
      jvmArgs += project.property('extraJvmArgs')
    }
    check.dependsOn integrationTest
  }
}

project(':iceberg-gcp') {
  test {
    useJUnitPlatform()
  }

  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')

    compileOnly platform(libs.google.libraries.bom)
    compileOnly "com.google.cloud:google-cloud-storage"

    testImplementation "com.google.cloud:google-cloud-nio"

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')

    testImplementation(libs.hadoop2.common) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'javax.servlet', module: 'servlet-api'
      exclude group: 'com.google.code.gson', module: 'gson'
    }
    testImplementation libs.esotericsoftware.kryo
  }
}

project(':iceberg-hive-metastore') {
  test {
    useJUnitPlatform()
  }

  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    implementation project(':iceberg-core')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    annotationProcessor libs.immutables.value
    compileOnly libs.immutables.value

    implementation libs.caffeine

    compileOnly libs.avro.avro

    compileOnly(libs.hive2.metastore) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
      exclude group: 'org.apache.hbase'
      exclude group: 'org.apache.logging.log4j'
      exclude group: 'co.cask.tephra'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
      exclude group: 'com.tdunning', module: 'json'
      exclude group: 'javax.transaction', module: 'transaction-api'
      exclude group: 'com.zaxxer', module: 'HikariCP'
    }

    // By default, hive-exec is a fat/uber jar and it exports a guava library
    // that's really old. We use the core classifier to be able to override our guava
    // version. Luckily, hive-exec seems to work okay so far with this version of guava
    // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
      exclude group: 'org.apache.hive', module: 'hive-llap-tez'
      exclude group: 'org.apache.logging.log4j'
      exclude group: 'com.google.protobuf', module: 'protobuf-java'
      exclude group: 'org.apache.calcite'
      exclude group: 'org.apache.calcite.avatica'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
    }

    testImplementation(libs.hive2.metastore) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
      exclude group: 'org.apache.hbase'
      exclude group: 'org.apache.logging.log4j'
      exclude group: 'co.cask.tephra'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
      exclude group: 'com.tdunning', module: 'json'
      exclude group: 'javax.transaction', module: 'transaction-api'
      exclude group: 'com.zaxxer', module: 'HikariCP'
    }

    compileOnly(libs.hadoop2.client) {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
    testImplementation libs.awaitility
  }
}

project(':iceberg-orc') {
  test {
    useJUnitPlatform()
  }
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')
    implementation(libs.avro.avro) {
      exclude group: 'org.tukaani' // xz compression is not supported
    }

    implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
      exclude group: 'org.apache.hadoop'
      exclude group: 'commons-lang'
      // These artifacts are shaded and included in the orc-core fat jar
      exclude group: 'com.google.protobuf', module: 'protobuf-java'
      exclude group: 'org.apache.hive', module: 'hive-storage-api'
    }

    compileOnly(libs.hadoop2.common) {
      exclude group: 'commons-beanutils'
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
    compileOnly(libs.hadoop2.client) {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
    testImplementation project(':iceberg-common')
    testImplementation libs.orc.tools
  }
}

project(':iceberg-parquet') {
  test {
    useJUnitPlatform()
  }
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    implementation project(':iceberg-common')

    implementation(libs.parquet.avro) {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    compileOnly libs.avro.avro
    compileOnly(libs.hadoop2.client) {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
  }
}

project(':iceberg-arrow') {
  test {
    useJUnitPlatform()
  }
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    implementation project(':iceberg-parquet')

    implementation(libs.arrow.vector) {
      exclude group: 'io.netty', module: 'netty-buffer'
      exclude group: 'io.netty', module: 'netty-common'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
    }
    implementation(libs.arrow.memory.netty) {
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
      exclude group: 'io.netty', module: 'netty-common'
      exclude group: 'io.netty', module: 'netty-buffer'
    }

    runtimeOnly libs.netty.buffer

    implementation(libs.parquet.avro) {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
    // To run ArrowReaderTest test cases, :netty-common is needed.
    // We import :netty-common through :arrow-memory-netty
    // so that the same version as used by the :arrow-memory-netty module is picked.
    testImplementation libs.arrow.memory.netty
    testImplementation libs.hadoop2.common
    testImplementation libs.hadoop2.mapreduce.client.core
  }
}

project(':iceberg-pig') {
  test {
    useJUnitPlatform()
  }

  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')
    implementation project(':iceberg-parquet')

    implementation(libs.parquet.avro) {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    compileOnly(libs.pig) {
      exclude group: "junit", module: "junit"
    }
    compileOnly(libs.hadoop2.mapreduce.client.core)
    compileOnly(libs.hadoop2.client) {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testImplementation(libs.hadoop2.minicluster) {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

project(':iceberg-nessie') {
  if (JavaVersion.current().isJava11Compatible()) {
    test {
      useJUnitPlatform()
    }
    compileTestJava {
      sourceCompatibility = "11"
      targetCompatibility = "11"
    }
  } else {
    // Do not test Nessie against Java 8, because in-JVM testing requires Nessie server components,
    // which require Java 11+.
    test {
      enabled = false
    }
    compileTestJava {
      enabled = false
    }
  }

  dependencies {
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    implementation(libs.nessie.client) {
      exclude group: 'com.fasterxml.jackson'
    }
    implementation platform(libs.jackson.bom)
    implementation "com.fasterxml.jackson.core:jackson-databind"
    implementation "com.fasterxml.jackson.core:jackson-core"

    compileOnly libs.hadoop2.common
    // Only there to prevent "warning: unknown enum constant SchemaType.OBJECT" compile messages
    compileOnly libs.microprofile.openapi.api

    if (JavaVersion.current().isJava11Compatible()) {
      testImplementation libs.nessie.jaxrs.testextension
      testImplementation libs.nessie.versioned.storage.inmemory.tests
      testImplementation libs.nessie.versioned.storage.testextension
      // Need to "pull in" el-api explicitly :(
      testImplementation libs.jakarta.el.api

      testImplementation libs.avro.avro

      testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
      testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')

      // Only there to prevent "warning: unknown enum constant SchemaType.OBJECT" compile messages
      testCompileOnly libs.microprofile.openapi.api
    }
  }
}

project(':iceberg-dell') {
  test {
    useJUnitPlatform()
  }
  dependencies {
    implementation project(':iceberg-core')
    implementation project(':iceberg-common')
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    compileOnly libs.object.client.bundle

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation libs.jaxb.api
    testImplementation libs.activation
    testImplementation libs.jaxb.runtime
  }
}

project(':iceberg-snowflake') {
  test {
    useJUnitPlatform()
  }

  dependencies {
    implementation project(':iceberg-core')
    implementation project(':iceberg-common')
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    implementation platform(libs.jackson.bom)
    implementation "com.fasterxml.jackson.core:jackson-databind"
    implementation "com.fasterxml.jackson.core:jackson-core"

    runtimeOnly libs.snowflake.jdbc

    testImplementation libs.mockito.junit.jupiter
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
  }
}

project(':iceberg-open-api') {
  def restCatalogSpec = "$projectDir/rest-catalog-open-api.yaml"
  tasks.register('validateRESTCatalogSpec', org.openapitools.generator.gradle.plugin.tasks.ValidateTask) {
    inputSpec.set(restCatalogSpec)
    recommend.set(true)
  }
  check.dependsOn('validateRESTCatalogSpec')
}

@Memoized
boolean versionFileExists() {
  return file('version.txt').exists()
}

@Memoized
String getVersionFromFile() {
  return file('version.txt').text.trim()
}

String getProjectVersion() {
  if (versionFileExists()) {
    return getVersionFromFile()
  }

  try {
    // we're fetching the version from the latest tag (prefixed with 'apache-iceberg-'),
    // which can look like this: '0.13.0-2-g805400f0.dirty' but we're only interested in the MAJOR.MINOR.PATCH part
    String version = gitVersion(prefix: 'apache-iceberg-')
    Pattern pattern = Pattern.compile("^([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?\$")
    Matcher matcher = pattern.matcher(version)
    if (matcher.matches()) {
      // bump the MINOR version and always set the PATCH version to 0
      return matcher.group(1) + "." + (Integer.valueOf(matcher.group(2)) + 1) + ".0-SNAPSHOT"
    }
    return version
  } catch (Exception e) {
    throw new Exception("Neither version.txt nor git version exists: " + e.getMessage(), e)
  }
}

String getJavadocVersion() {
  if (versionFileExists()) {
    return getVersionFromFile()
  }

  try {
    // use the branch name in place of version in Javadoc
    return versionDetails().branchName
  } catch (NullPointerException e) {
    throw new Exception("Neither version.txt nor git version exists")
  }
}

apply from: 'jmh.gradle'
apply from: 'baseline.gradle'
apply from: 'deploy.gradle'
apply from: 'tasks.gradle'

project(':iceberg-bom') {
  apply plugin: 'java-platform'

  dependencies {
    constraints {
      // The Iceberg-Build builds for only one Scala version at a time, so the BOM would also
      // only contain artifacts for that single Scala version. The following code ensures that
      // the BOM references the artifacts for all Scala versions.
      def sparkScalaPattern = ~"(.*)-([0-9][.][0-9]+)_([0-9][.][0-9]+)"
      def sparkScalaVersions = [
        "3.3": ["2.12", "2.13"],
        "3.4": ["2.12", "2.13"],
        "3.5": ["2.12", "2.13"],
      ]
      rootProject.allprojects.forEach {
        // Do not include ':iceberg-spark', the bom itself and the root project.
        if (it.name != 'iceberg-bom' && it != rootProject && it.childProjects.isEmpty()) {
          if (it.name.startsWith("iceberg-spark-")) {
            def sparkScalaMatcher = sparkScalaPattern.matcher(it.name)
            if (!sparkScalaMatcher.find()) {
              throw new GradleException("Expected a Spark/Scala version combination in Gradle project name ${it.name}")
            }
            def prjName = sparkScalaMatcher.group(1)
            def sparkVer = sparkScalaMatcher.group(2)
            for (def scalaVer in sparkScalaVersions[sparkVer]) {
              add("api", "${it.group}:$prjName-${sparkVer}_$scalaVer:${it.version}")
            }
          } else {
            add("api", project(it.path))
          }
        }
      }
    }
  }

  // Needed to get the "faked" Scala artifacts into the bom
  javaPlatform { allowDependencies() }
}