YARN-11626. Optimize ResourceManager's operations on Zookeeper metadata

Author: wuxiaobao

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml

@@ -67,6 +67,12 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-inline</artifactId>
+      <version>2.8.9</version>
+      <scope>test</scope>
+    </dependency>
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java

@@ -956,7 +956,7 @@ private void handleApplicationAttemptStateOp(
           zkAcl, fencingNodePath);
       break;
     case REMOVE:
-      zkManager.safeDelete(path, zkAcl, fencingNodePath);
+      safeDeleteAndCheckNode(path, zkAcl, fencingNodePath);
       break;
     default:
       break;
@@ -1035,10 +1035,10 @@ private void removeApp(String removeAppId, boolean safeRemove,
         for (ApplicationAttemptId attemptId : attempts) {
           String attemptRemovePath =
               getNodePath(appIdRemovePath, attemptId.toString());
-          zkManager.safeDelete(attemptRemovePath, zkAcl, fencingNodePath);
+          safeDeleteAndCheckNode(attemptRemovePath, zkAcl, fencingNodePath);
         }
       }
-      zkManager.safeDelete(appIdRemovePath, zkAcl, fencingNodePath);
+      safeDeleteAndCheckNode(appIdRemovePath, zkAcl, fencingNodePath);
     } else {
       CuratorFramework curatorFramework = zkManager.getCurator();
       curatorFramework.delete().deletingChildrenIfNeeded().
@@ -1099,7 +1099,7 @@ protected synchronized void removeRMDelegationTokenState(
     LOG.debug("Removing RMDelegationToken_{}",
         rmDTIdentifier.getSequenceNumber());
 
-    zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath);
+    safeDeleteAndCheckNode(nodeRemovePath, zkAcl, fencingNodePath);
 
     // Check if we should remove the parent app node as well.
     checkRemoveParentZnode(nodeRemovePath, splitIndex);
@@ -1160,7 +1160,7 @@ protected synchronized void removeRMDTMasterKeyState(
 
     LOG.debug("Removing RMDelegationKey_{}", delegationKey.getKeyId());
 
-    zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath);
+    safeDeleteAndCheckNode(nodeRemovePath, zkAcl, fencingNodePath);
   }
 
   @Override
@@ -1200,12 +1200,12 @@ protected synchronized void removeReservationState(String planName,
     LOG.debug("Removing reservationallocation {} for plan {}",
         reservationIdName, planName);
 
-    zkManager.safeDelete(reservationPath, zkAcl, fencingNodePath);
+    safeDeleteAndCheckNode(reservationPath, zkAcl, fencingNodePath);
 
     List<String> reservationNodes = getChildren(planNodePath);
 
     if (reservationNodes.isEmpty()) {
-      zkManager.safeDelete(planNodePath, zkAcl, fencingNodePath);
+      safeDeleteAndCheckNode(planNodePath, zkAcl, fencingNodePath);
     }
   }
 
@@ -1441,6 +1441,29 @@ void delete(final String path) throws Exception {
     zkManager.delete(path);
   }
 
+  /**
+   * Deletes the path more safe.
+   * When NNE is encountered, if the node does not exist,
+   * it will ignore this exception to avoid triggering
+   * a greater impact of ResourceManager failover on the cluster.
+   * @param path Path to be deleted.
+   * @param fencingACL fencingACL.
+   * @param fencingPath fencingNodePath.
+   * @throws Exception if any problem occurs while performing deletion.
+   */
+  public void safeDeleteAndCheckNode(String path, List<ACL> fencingACL,
+      String fencingPath) throws Exception {
+    try{
+      zkManager.safeDelete(path, fencingACL, fencingPath);
+    } catch (KeeperException.NoNodeException nne) {
+      if(!exists(path)){
+        LOG.info("Node " + path + " doesn't exist to delete");
+      } else {
+        throw new KeeperException.NodeExistsException("Node " + path + " should not exist");
+      }
+    }
+  }
+
   /**
    * Helper class that periodically attempts creating a znode to ensure that
    * this RM continues to be the Active.