Remove Hive 2

Author: manuzhang

gradle.properties

@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
 jmhIncludeRegex=.*
 systemProp.defaultFlinkVersions=1.20
 systemProp.knownFlinkVersions=1.18,1.19,1.20
-systemProp.defaultHiveVersions=2
-systemProp.knownHiveVersions=2,4
+systemProp.defaultHiveVersions=4
+systemProp.knownHiveVersions=4
 systemProp.defaultSparkVersions=3.5
 systemProp.knownSparkVersions=3.3,3.4,3.5
 systemProp.defaultKafkaVersions=3

gradle/libs.versions.toml

@@ -49,7 +49,6 @@ guava = "33.3.1-jre"
 hadoop2 = "2.7.3"
 hadoop3 = "3.4.1"
 httpcomponents-httpclient5 = "5.4.1"
-hive2 = { strictly = "2.3.9"} # see rich version usage explanation above
 hive4 = "4.0.1"
 immutables-value = "2.10.1"
 jackson-bom = "2.18.2"
@@ -135,14 +134,9 @@ hadoop2-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-c
 hadoop2-minicluster = { module = "org.apache.hadoop:hadoop-minicluster", version.ref = "hadoop2" }
 hadoop3-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop3" }
 hadoop3-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop3" }
-hive2-exec = { module = "org.apache.hive:hive-exec", version.ref = "hive2" }
-hive2-metastore = { module = "org.apache.hive:hive-metastore", version.ref = "hive2" }
-hive2-serde = { module = "org.apache.hive:hive-serde", version.ref = "hive2" }
-hive2-service = { module = "org.apache.hive:hive-service", version.ref = "hive2" }
 hive4-exec = { module = "org.apache.hive:hive-exec", version.ref = "hive4" }
 hive4-metastore = { module = "org.apache.hive:hive-metastore", version.ref = "hive4" }
 hive4-standalone-metastore-server = { module = "org.apache.hive:hive-standalone-metastore-server", version.ref = "hive4" }
-hive4-serde = { module = "org.apache.hive:hive-serde", version.ref = "hive4" }
 hive4-service = { module = "org.apache.hive:hive-service", version.ref = "hive4" }
 httpcomponents-httpclient5 = { module = "org.apache.httpcomponents.client5:httpclient5", version.ref = "httpcomponents-httpclient5" }
 immutables-value = { module = "org.immutables:value", version.ref = "immutables-value" }

mr/build.gradle

@@ -33,48 +33,43 @@ project(':iceberg-mr') {
     implementation project(':iceberg-common')
     implementation project(':iceberg-core')
     api project(':iceberg-data')
-    implementation project(':iceberg-hive-metastore')
     implementation project(':iceberg-orc')
     implementation project(':iceberg-parquet')
 
     compileOnly(libs.hadoop2.client) {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    compileOnly("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
-      exclude group: 'com.google.code.findbugs', module: 'jsr305'
-      exclude group: 'com.google.guava'
+    implementation libs.caffeine
+    implementation libs.parquet.column
+    implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
+      exclude group: 'org.apache.hadoop'
+      exclude group: 'commons-lang'
+      // These artifacts are shaded and included in the orc-core fat jar
       exclude group: 'com.google.protobuf', module: 'protobuf-java'
-      exclude group: 'org.apache.avro', module: 'avro'
-      exclude group: 'org.apache.calcite.avatica'
-      exclude group: 'org.apache.hive', module: 'hive-llap-tez'
-      exclude group: 'org.apache.logging.log4j'
-      exclude group: 'org.pentaho' // missing dependency
-      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: 'org.apache.hive', module: 'hive-storage-api'
     }
-    compileOnly libs.hive2.metastore
-    compileOnly libs.hive2.serde
-
-    implementation libs.caffeine
 
     testImplementation libs.calcite.core
     testImplementation libs.calcite.druid
 
     testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
     testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
     testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
-    testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+    testImplementation project(path: ':iceberg-hive-metastore')
 
     testImplementation libs.avro.avro
     testImplementation libs.calcite.core
     testImplementation libs.kryo.shaded
     testImplementation platform(libs.jackson.bom)
     testImplementation libs.jackson.annotations
-    testImplementation(libs.hive2.service) {
+    testImplementation(libs.hive4.service) {
       exclude group: 'org.apache.hive', module: 'hive-exec'
+      exclude group: 'org.pac4j', module: 'pac4j-saml-opensamlv3'
     }
     testImplementation libs.tez08.dag
     testImplementation libs.tez08.mapreduce
+    testImplementation libs.junit.vintage.engine
   }
 
   test {

mr/src/main/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedSupport.java

@@ -76,11 +76,6 @@ private InputFormatConfig() {}
 
   public static final String CATALOG_CONFIG_PREFIX = "iceberg.catalog.";
 
-  public enum InMemoryDataModel {
-    HIVE,
-    GENERIC // Default data model is of Iceberg Generics
-  }
-
   public static class ConfigBuilder {
     private final Configuration conf;
 
@@ -163,11 +158,6 @@ public ConfigBuilder preferLocality() {
       return this;
     }
 
-    public ConfigBuilder useHiveRows() {
-      conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.HIVE.name());
-      return this;
-    }
-
     /**
      * Compute platforms pass down filters to data sources. If the data source cannot apply some
      * filters, or only partially applies the filter, it will return the residual filter back. If