YARN-11626. Optimize ResourceManager's operations on Zookeeper metadata

Author: wuxiaobao

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml

@@ -67,6 +67,12 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-inline</artifactId>
+      <version>2.8.9</version>
+      <scope>test</scope>
+    </dependency>
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java

@@ -956,7 +956,15 @@ private void handleApplicationAttemptStateOp(
           zkAcl, fencingNodePath);
       break;
     case REMOVE:
-      zkManager.safeDelete(path, zkAcl, fencingNodePath);
+      try{
+        zkManager.safeDelete(path, zkAcl, fencingNodePath);
+      } catch (KeeperException.NoNodeException nne) {
+        if(!exists(path)){
+            LOG.info("Node " + path + " doesn't exist to delete");
+        } else {
+            throw nne;
+        }
+      }
       break;
     default:
       break;
@@ -1031,14 +1039,25 @@ private void removeApp(String removeAppId, boolean safeRemove,
       LOG.debug("Removing info for app: {} at: {} and its attempts.",
           removeAppId, appIdRemovePath);
 
-      if (attempts != null) {
-        for (ApplicationAttemptId attemptId : attempts) {
-          String attemptRemovePath =
-              getNodePath(appIdRemovePath, attemptId.toString());
-          zkManager.safeDelete(attemptRemovePath, zkAcl, fencingNodePath);
+      String path = appIdRemovePath;
+      try {
+        if (attempts != null) {
+          for (ApplicationAttemptId attemptId : attempts) {
+            String attemptRemovePath =
+                getNodePath(appIdRemovePath, attemptId.toString());
+            path = attemptRemovePath;
+            zkManager.safeDelete(attemptRemovePath, zkAcl, fencingNodePath);
+          }
+          path = appIdRemovePath;
+        }
+        zkManager.safeDelete(appIdRemovePath, zkAcl, fencingNodePath);
+      } catch (KeeperException.NoNodeException nne){
+        if(!exists(path)){
+          LOG.info("Node " + path + " doesn't exist to delete");
+        } else {
+          throw nne;
         }
       }
-      zkManager.safeDelete(appIdRemovePath, zkAcl, fencingNodePath);
     } else {
       CuratorFramework curatorFramework = zkManager.getCurator();
       curatorFramework.delete().deletingChildrenIfNeeded().
@@ -1098,8 +1117,15 @@ protected synchronized void removeRMDelegationTokenState(
 
     LOG.debug("Removing RMDelegationToken_{}",
         rmDTIdentifier.getSequenceNumber());
-
-    zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath);
+    try {
+      zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath);
+    } catch (KeeperException.NoNodeException nne){
+      if(!exists(nodeRemovePath)){
+        LOG.info("Node " + nodeRemovePath + " doesn't exist to delete");
+      } else {
+        throw nne;
+      }
+    }
 
     // Check if we should remove the parent app node as well.
     checkRemoveParentZnode(nodeRemovePath, splitIndex);
@@ -1159,8 +1185,15 @@ protected synchronized void removeRMDTMasterKeyState(
             + delegationKey.getKeyId());
 
     LOG.debug("Removing RMDelegationKey_{}", delegationKey.getKeyId());
-
-    zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath);
+    try {
+      zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath);
+    } catch (KeeperException.NoNodeException nne){
+      if(!exists(nodeRemovePath)){
+        LOG.info("Node " + nodeRemovePath + " doesn't exist to delete");
+      } else {
+        throw nne;
+      }
+    }
   }
 
   @Override
@@ -1200,12 +1233,22 @@ protected synchronized void removeReservationState(String planName,
     LOG.debug("Removing reservationallocation {} for plan {}",
         reservationIdName, planName);
 
-    zkManager.safeDelete(reservationPath, zkAcl, fencingNodePath);
+    String path = reservationPath;
+    try{
+      zkManager.safeDelete(reservationPath, zkAcl, fencingNodePath);
 
-    List<String> reservationNodes = getChildren(planNodePath);
+      List<String> reservationNodes = getChildren(planNodePath);
 
-    if (reservationNodes.isEmpty()) {
-      zkManager.safeDelete(planNodePath, zkAcl, fencingNodePath);
+      if (reservationNodes.isEmpty()) {
+        path = planNodePath;
+        zkManager.safeDelete(planNodePath, zkAcl, fencingNodePath);
+      }
+    } catch (KeeperException.NoNodeException nne){
+      if(!exists(path)){
+        LOG.info("Node " + path + " doesn't exist to delete");
+      } else {
+        throw nne;
+      }
     }
   }