HDDS-1906. TestScmSafeMode#testSCMSafeModeRestrictedOp is failing. (#1260)

Author: nandakumar131

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

@@ -226,6 +226,11 @@ void restartHddsDatanode(DatanodeDetails dn, boolean waitForDatanode)
    */
   void startHddsDatanodes();
 
+  /**
+   * Shuts down all the DataNodes.
+   */
+  void shutdownHddsDatanodes();
+
   /**
    * Builder class for MiniOzoneCluster.
    */

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

@@ -375,6 +375,17 @@ public void startHddsDatanodes() {
     });
   }
 
+  @Override
+  public void shutdownHddsDatanodes() {
+    hddsDatanodes.forEach((datanode) -> {
+      try {
+        shutdownHddsDatanode(datanode.getDatanodeDetails());
+      } catch (IOException e) {
+        LOG.error("Exception while trying to shutdown datanodes:", e);
+      }
+    });
+  }
+
   private CertificateClient getCAClient() {
     return this.caClient;
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java

@@ -58,6 +58,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -91,6 +93,8 @@ public class TestScmSafeMode {
   @Before
   public void init() throws Exception {
     conf = new OzoneConfiguration();
+    conf.set(OZONE_SCM_STALENODE_INTERVAL, "10s");
+    conf.set(OZONE_SCM_DEADNODE_INTERVAL, "25s");
     builder = MiniOzoneCluster.newBuilder(conf)
         .setHbInterval(1000)
         .setHbProcessorInterval(500)
@@ -328,6 +332,8 @@ public void testSCMSafeModeRestrictedOp() throws Exception {
     }, 50, 1000 * 30);
     assertTrue(clientProtocolServer.getSafeModeStatus());
 
+    cluster.shutdownHddsDatanodes();
+    Thread.sleep(30000);
     LambdaTestUtils.intercept(SCMException.class,
         "Open container " + containers.get(0).getContainerID() + " "
             + "doesn't have enough replicas to service this operation in Safe"