HDDS-11779. Add DN metrics to show deletion progress

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java

@@ -46,6 +46,9 @@ public final class BlockDeletingServiceMetrics {
   @Metric(about = "The number of failed delete blocks.")
   private MutableCounterLong failureCount;
 
+  @Metric(about = "The number of delete block transaction processed.")
+  private MutableCounterLong processedTransactionCount;
+
   @Metric(about = "The number of out of order delete block transaction.")
   private MutableCounterLong outOfOrderDeleteBlockTransactionCount;
 
@@ -55,8 +58,7 @@ public final class BlockDeletingServiceMetrics {
   @Metric(about = "The total number of DeleteBlockTransaction received")
   private MutableCounterLong receivedTransactionCount;
 
-  @Metric(about = "The total number of DeleteBlockTransaction" +
-      " that is a retry Transaction")
+  @Metric(about = "The total number of DeleteBlockTransaction that is a retry Transaction")
   private MutableCounterLong receivedRetryTransactionCount;
 
   @Metric(about = "The total number of Container received to be processed")
@@ -74,10 +76,20 @@ public final class BlockDeletingServiceMetrics {
   @Metric(about = "The total number of Container chosen to be deleted.")
   private MutableGaugeLong totalContainerChosenCount;
 
-  @Metric(about = "The total number of transactions which failed due" +
-      " to container lock wait timeout.")
+  @Metric(about = "The total number of transactions which failed due to container lock wait timeout.")
   private MutableGaugeLong totalLockTimeoutTransactionCount;
 
+  @Metric(about = "The total number of deletion commands received.")
+  private MutableGaugeLong totalCommandsReceived;
+
+  @Metric(about = "The total number of deletion commands that were discarded " +
+      "due to the queue being full.")
+  private MutableGaugeLong totalCommandsDiscarded;
+
+  @Metric(about = "The total number of deletion transactions that were discarded " +
+      "due to the transaction being a duplicate.")
+  private MutableGaugeLong totalTransactionsDiscarded;
+
   private BlockDeletingServiceMetrics() {
   }
 
@@ -112,6 +124,10 @@ public void incrFailureCount() {
     this.failureCount.incr();
   }
 
+  public void incrProcessedTransactionCount(long count) {
+    processedTransactionCount.incr(count);
+  }
+
   public void incrReceivedTransactionCount(long count) {
     receivedTransactionCount.incr(count);
   }
@@ -148,6 +164,18 @@ public void incrTotalLockTimeoutTransactionCount() {
     totalLockTimeoutTransactionCount.incr();
   }
 
+  public void incrTotalCommandsReceived(long delta) {
+    this.totalCommandsReceived.incr(delta);
+  }
+
+  public void incrTotalCommandsDiscarded(long delta) {
+    this.totalCommandsDiscarded.incr(delta);
+  }
+
+  public void incrTotalTransactionsDiscarded(long delta) {
+    this.totalTransactionsDiscarded.incr(delta);
+  }
+
   public long getSuccessCount() {
     return successCount.value();
   }
@@ -184,6 +212,26 @@ public long getTotalLockTimeoutTransactionCount() {
     return totalLockTimeoutTransactionCount.value();
   }
 
+  public long getProcessedTransactionCount() {
+    return processedTransactionCount.value();
+  }
+
+  public long getReceivedTransactionCount() {
+    return receivedTransactionCount.value();
+  }
+
+  public long getTotalCommandsReceived() {
+    return totalCommandsReceived.value();
+  }
+
+  public long getTotalCommandsDiscarded() {
+    return totalCommandsDiscarded.value();
+  }
+
+  public long getTotalTransactionsDiscarded() {
+    return totalTransactionsDiscarded.value();
+  }
+
   @Override
   public String toString() {
     StringBuffer buffer = new StringBuffer();
@@ -202,14 +250,22 @@ public String toString() {
             + receivedTransactionCount.value()).append("\t")
         .append("receivedRetryTransactionCount = "
             + receivedRetryTransactionCount.value()).append("\t")
+        .append("processedTransactionCount = "
+            + processedTransactionCount.value()).append("\t")
         .append("receivedContainerCount = "
             + receivedContainerCount.value()).append("\t")
         .append("receivedBlockCount = "
             + receivedBlockCount.value()).append("\t")
         .append("markedBlockCount = "
             + markedBlockCount.value()).append("\t")
         .append("totalLockTimeoutTransactionCount = "
-            + totalLockTimeoutTransactionCount.value()).append("\t");
+            + totalLockTimeoutTransactionCount.value()).append("\t")
+        .append("totalCommandsReceived = "
+            + totalCommandsReceived.value()).append("\t")
+        .append("totalCommandsDiscarded = "
+            + totalCommandsDiscarded.value()).append("\t")
+        .append("totalTransactionsDiscarded = "
+            + totalTransactionsDiscarded.value()).append("\t");
     return buffer.toString();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java

@@ -144,6 +144,7 @@ public void handle(SCMCommand command, OzoneContainer container,
         container, context, connectionManager);
     try {
       deleteCommandQueues.add(cmd);
+      blockDeleteMetrics.incrTotalCommandsReceived(1);
     } catch (IllegalStateException e) {
       String dnId = context.getParent().getDatanodeDetails().getUuidString();
       Consumer<CommandStatus> updateFailure = (cmdStatus) -> {
@@ -157,6 +158,7 @@ public void handle(SCMCommand command, OzoneContainer container,
       };
       updateCommandStatus(cmd.getContext(), cmd.getCmd(), updateFailure, LOG);
       LOG.warn("Command is discarded because of the command queue is full");
+      blockDeleteMetrics.incrTotalCommandsDiscarded(1);
     }
   }
 
@@ -462,6 +464,7 @@ public List<Future<DeleteBlockTransactionExecutionResult>> submitTasks(
       Future<DeleteBlockTransactionExecutionResult> future =
           executor.submit(new ProcessTransactionTask(tx));
       futures.add(future);
+      blockDeleteMetrics.incrProcessedTransactionCount(1);
     }
     return futures;
   }
@@ -650,6 +653,7 @@ public static boolean isDuplicateTransaction(long containerId, KeyValueContainer
           containerData.getDeleteTransactionId()));
     } else if (delTX.getTxID() == containerData.getDeleteTransactionId()) {
       duplicate = true;
+      metrics.incrTotalTransactionsDiscarded(1);
       LOG.info(String.format("Delete blocks with txID %d for containerId: %d"
               + " is retried.", delTX.getTxID(), containerId));
     } else {

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java

@@ -331,6 +331,9 @@ public void testDeleteBlockCommandHandleWhenDeleteCommandQueuesFull()
         assertEquals(cmdStatus.getProtoBufMessage().getBlockDeletionAck().getResultsCount(), 0);
       }
     }
+    blockDeleteMetrics = handler.getBlockDeleteMetrics();
+    assertEquals(5, blockDeleteMetrics.getTotalCommandsReceived());
+    assertEquals(2, blockDeleteMetrics.getTotalCommandsDiscarded());
   }
 
   @ContainerTestVersionInfo.ContainerTest
@@ -367,6 +370,7 @@ public void testDuplicateDeleteBlocksCommand(
     assertTrue(results3.get(0).getSuccess());
     assertEquals(0,
         blockDeleteMetrics.getTotalLockTimeoutTransactionCount());
+    assertEquals(1, blockDeleteMetrics.getTotalTransactionsDiscarded());
     // Duplicate cmd content will not be persisted.
     assertEquals(2,
         ((KeyValueContainerData) container.getContainerData()).getNumPendingDeletionBlocks());