HBASE-24446 Use EnvironmentEdgeManager to compute clock skew in Master (

#1885) Signed-off-by: Viraj Jasani <vjasani@apache.org>
apache · Jun 12, 2020 · 1dac9f6 · 1dac9f6
1 parent 6b7a93d
commit 1dac9f6
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 1 deletion.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -56,6 +56,7 @@
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FutureUtils;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -360,7 +361,7 @@ void findDeadServersAndProcess(Set<ServerName> deadServersFromPE,
    */
   private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
       throws ClockOutOfSyncException {
-    long skew = Math.abs(System.currentTimeMillis() - serverCurrentTime);
+    long skew = Math.abs(EnvironmentEdgeManager.currentTime() - serverCurrentTime);
     if (skew > maxSkew) {
       String message = "Server " + serverName + " has been " +
         "rejected; Reported time is too far out of sync with master.  " +

diff --git a/...ver/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java b/...ver/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
@@ -37,8 +37,10 @@
 import org.apache.hadoop.hbase.master.LoadBalancer;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.log4j.Appender;
 import org.apache.log4j.Layout;
@@ -259,6 +261,37 @@ public void run() {
     waitForClusterOnline(master);
   }
 
+  /**
+   * Tests region sever reportForDuty with manual environment edge
+   */
+  @Test
+  public void testReportForDutyWithEnvironmentEdge() throws Exception {
+    // Start a master and wait for it to become the active/primary master.
+    // Use a random unique port
+    cluster.getConfiguration().setInt(HConstants.MASTER_PORT, HBaseTestingUtility.randomFreePort());
+    // Set the dispatch and retry delay to 0 since we want the rpc request to be sent immediately
+    cluster.getConfiguration().setInt("hbase.procedure.remote.dispatcher.delay.msec", 0);
+    cluster.getConfiguration().setLong("hbase.regionserver.rpc.retry.interval", 0);
+
+    // master has a rs. defaultMinToStart = 2
+    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(testUtil.getConfiguration());
+    cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
+      tablesOnMaster ? 2 : 1);
+    cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART,
+      tablesOnMaster ? 2 : 1);
+
+    // Inject manual environment edge for clock skew computation between RS and master
+    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
+    EnvironmentEdgeManager.injectEdge(edge);
+    master = cluster.addMaster();
+    rs = cluster.addRegionServer();
+    LOG.debug("Starting master: " + master.getMaster().getServerName());
+    master.start();
+    rs.start();
+
+    waitForClusterOnline(master);
+  }
+
   private void waitForClusterOnline(MasterThread master) throws InterruptedException {
     while (true) {
       if (master.getMaster().isInitialized()) {