Upgrade to Hadoop 3.4.0

Author: zhangbutao

common/src/java/org/apache/hadoop/hive/common/JvmMetrics.java

@@ -20,7 +20,6 @@
 
 import static org.apache.hadoop.hive.common.JvmMetricsInfo.*;
 
-import org.apache.hadoop.log.metrics.EventCounter;
 import org.apache.hadoop.metrics2.MetricsCollector;
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -96,7 +95,6 @@ public void getMetrics(MetricsCollector collector, boolean all) {
     getMemoryUsage(rb);
     getGcUsage(rb);
     getThreadUsage(rb);
-    getEventCounters(rb);
   }
 
   private void getMemoryUsage(MetricsRecordBuilder rb) {
@@ -201,11 +199,4 @@ public ThreadCountResult(int threadsNew, int threadsRunnable, int threadsBlocked
       this.threadsTerminated = threadsTerminated;
     }
   }
-
-  private void getEventCounters(MetricsRecordBuilder rb) {
-    rb.addCounter(LogFatal, EventCounter.getFatal())
-        .addCounter(LogError, EventCounter.getError())
-        .addCounter(LogWarn, EventCounter.getWarn())
-        .addCounter(LogInfo, EventCounter.getInfo());
-  }
 }

itests/hcatalog-unit/pom.xml

@@ -357,23 +357,4 @@
       </plugin>
     </plugins>
   </build>
-  <profiles>
-    <profile>
-      <!-- See HIVE-24473 -->
-      <id>gahbase</id>
-      <activation>
-        <property>
-          <name>hbase.version</name>
-        </property>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.hbase</groupId>
-          <artifactId>hbase-zookeeper</artifactId>
-          <classifier>tests</classifier>
-          <scope>test</scope>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
 </project>

itests/util/pom.xml

@@ -258,22 +258,4 @@
       </exclusions>
     </dependency>
   </dependencies>
-  <profiles>
-    <profile>
-      <!-- See HIVE-24473 -->
-      <id>gahbase</id>
-      <activation>
-        <property>
-          <name>hbase.version</name>
-        </property>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.hbase</groupId>
-          <artifactId>hbase-zookeeper</artifactId>
-          <classifier>tests</classifier>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
 </project>

itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java

@@ -106,6 +106,8 @@ private void setUpFixtures(HiveConf conf) throws Exception {
         "org.apache.hadoop.hbase.shaded.");
 
     Configuration hbaseConf = HBaseConfiguration.create(conf);
+    // A workaround for HBASE-28908
+    hbaseConf.set("hbase.wal.provider", "filesystem");
     util = new HBaseTestingUtility(hbaseConf);
 
     util.startMiniDFSCluster(1);

pom.xml

@@ -140,10 +140,10 @@
     <guava.version>22.0</guava.version>
     <groovy.version>2.4.21</groovy.version>
     <h2database.version>2.2.220</h2database.version>
-    <hadoop.version>3.3.6</hadoop.version>
+    <hadoop.version>3.4.0</hadoop.version>
     <hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>
     <hamcrest.version>1.3</hamcrest.version>
-    <hbase.version>2.5.6-hadoop3</hbase.version>
+    <hbase.version>2.5.10-hadoop3</hbase.version>
     <hppc.version>0.7.2</hppc.version>
     <!-- required for logging test to avoid including hbase which pulls disruptor transitively -->
     <disruptor.version>3.3.7</disruptor.version>

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskListExtTblLocs.java

@@ -341,7 +341,7 @@ private void createOutputList(Set<String> locations, String outputDir, String db
    * Table-name followed by "*" indicates that all partitions are inside table location.
    * Otherwise, we record the number of partitions covered by table location.
    */
-  private JSONArray listOutputEntities(HashSet<String> locations) {
+  private JSONArray listOutputEntities(HashSet<String> locations) throws JSONException {
     List<String> listEntities = new ArrayList<>();
     for(String loc : locations) {
       DataLocation data = inputLocations.get(loc);

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java

@@ -49,7 +49,6 @@
 import java.util.Date;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
@@ -58,8 +57,6 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
 import static com.github.tomakehurst.wiremock.client.WireMock.ok;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 
 /*
 Tests JWT auth in HiveMetastore server.
@@ -94,9 +91,7 @@ public class TestRemoteHiveMetastoreWithHttpJwt {
   public static void makeEnvModifiable() throws Exception {
     envMap = new HashMap<>();
     Class<?> envClass = Class.forName("java.lang.ProcessEnvironment");
-    Field theEnvironmentField = envClass.getDeclaredField("theEnvironment");
     Field theUnmodifiableEnvironmentField = envClass.getDeclaredField("theUnmodifiableEnvironment");
-    removeStaticFinalAndSetValue(theEnvironmentField, envMap);
     removeStaticFinalAndSetValue(theUnmodifiableEnvironmentField, envMap);
   }
 

standalone-metastore/pom.xml

@@ -79,7 +79,7 @@
     </dropwizard-metrics-hadoop-metrics2-reporter.version>
     <dropwizard.version>3.1.0</dropwizard.version>
     <guava.version>22.0</guava.version>
-    <hadoop.version>3.3.6</hadoop.version>
+    <hadoop.version>3.4.0</hadoop.version>
     <hikaricp.version>4.0.3</hikaricp.version>
     <jackson.version>2.16.1</jackson.version>
     <jexl.version>3.3</jexl.version>

storage-api/pom.xml

@@ -30,7 +30,7 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <commons-logging.version>1.1.3</commons-logging.version>
     <guava.version>22.0</guava.version>
-    <hadoop.version>3.3.6</hadoop.version>
+    <hadoop.version>3.4.0</hadoop.version>
     <junit.version>4.13.2</junit.version>
     <junit.jupiter.version>5.6.3</junit.jupiter.version>
     <junit.vintage.version>5.6.3</junit.vintage.version>