HDDS-9426. Calculate Exclusive size for deep cleaned snapshot's delet…

…ed directories (apache#6099)

(cherry picked from commit 79d3c87)

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -3532,6 +3532,25 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.snapshot.directory.service.timeout</name>
+    <value>300s</value>
+    <tag>OZONE, PERFORMANCE, OM</tag>
+    <description>
+      Timeout value for SnapshotDirectoryCleaningService.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.snapshot.directory.service.interval</name>
+    <value>24h</value>
+    <tag>OZONE, PERFORMANCE, OM</tag>
+    <description>
+      The time interval between successive SnapshotDirectoryCleaningService
+      thread run.
+    </description>
+  </property>
+
   <property>
     <name>ozone.scm.event.ContainerReport.thread.pool.size</name>
     <value>10</value>

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -390,6 +390,18 @@ private OMConfigKeys() {
   public static final String OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT
       = "60s";
 
+  /**
+   * Configuration properties for Snapshot Directory Service.
+   */
+  public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL =
+      "ozone.snapshot.directory.service.interval";
+  public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL_DEFAULT
+      = "24h";
+  public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT =
+      "ozone.snapshot.directory.service.timeout";
+  public static final String
+      OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT_DEFAULT = "300s";
+
   public static final String OZONE_PATH_DELETING_LIMIT_PER_TASK =
       "ozone.path.deleting.limit.per.task";
   // default is 6000 taking account of 32MB buffer size, and assuming

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java

@@ -123,6 +123,7 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) {
   private long referencedReplicatedSize;
   private long exclusiveSize;
   private long exclusiveReplicatedSize;
+  private boolean deepCleanedDeletedDir;
 
   /**
    * Private constructor, constructed via builder.
@@ -162,7 +163,8 @@ private SnapshotInfo(UUID snapshotId,
                        long referencedSize,
                        long referencedReplicatedSize,
                        long exclusiveSize,
-                       long exclusiveReplicatedSize) {
+                       long exclusiveReplicatedSize,
+                       boolean deepCleanedDeletedDir) {
     this.snapshotId = snapshotId;
     this.name = name;
     this.volumeName = volumeName;
@@ -181,6 +183,7 @@ private SnapshotInfo(UUID snapshotId,
     this.referencedReplicatedSize = referencedReplicatedSize;
     this.exclusiveSize = exclusiveSize;
     this.exclusiveReplicatedSize = exclusiveReplicatedSize;
+    this.deepCleanedDeletedDir = deepCleanedDeletedDir;
   }
 
   public void setName(String name) {
@@ -285,7 +288,7 @@ public void setSstFiltered(boolean sstFiltered) {
   }
 
   public SnapshotInfo.Builder toBuilder() {
-    return new SnapshotInfo.Builder()
+    return new Builder()
         .setSnapshotId(snapshotId)
         .setName(name)
         .setVolumeName(volumeName)
@@ -302,7 +305,8 @@ public SnapshotInfo.Builder toBuilder() {
         .setReferencedSize(referencedSize)
         .setReferencedReplicatedSize(referencedReplicatedSize)
         .setExclusiveSize(exclusiveSize)
-        .setExclusiveReplicatedSize(exclusiveReplicatedSize);
+        .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+        .setDeepCleanedDeletedDir(deepCleanedDeletedDir);
   }
 
   /**
@@ -327,6 +331,7 @@ public static class Builder {
     private long referencedReplicatedSize;
     private long exclusiveSize;
     private long exclusiveReplicatedSize;
+    private boolean deepCleanedDeletedDir;
 
     public Builder() {
       // default values
@@ -423,6 +428,11 @@ public Builder setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
       return this;
     }
 
+    public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) {
+      this.deepCleanedDeletedDir = deepCleanedDeletedDir;
+      return this;
+    }
+
     public SnapshotInfo build() {
       Preconditions.checkNotNull(name);
       return new SnapshotInfo(
@@ -443,7 +453,8 @@ public SnapshotInfo build() {
           referencedSize,
           referencedReplicatedSize,
           exclusiveSize,
-          exclusiveReplicatedSize
+          exclusiveReplicatedSize,
+          deepCleanedDeletedDir
       );
     }
   }
@@ -465,7 +476,8 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() {
             .setReferencedSize(referencedSize)
             .setReferencedReplicatedSize(referencedReplicatedSize)
             .setExclusiveSize(exclusiveSize)
-            .setExclusiveReplicatedSize(exclusiveReplicatedSize);
+            .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+            .setDeepCleanedDeletedDir(deepCleanedDeletedDir);
 
     if (pathPreviousSnapshotId != null) {
       sib.setPathPreviousSnapshotID(toProtobuf(pathPreviousSnapshotId));
@@ -538,6 +550,11 @@ public static SnapshotInfo getFromProtobuf(
           snapshotInfoProto.getExclusiveReplicatedSize());
     }
 
+    if (snapshotInfoProto.hasDeepCleanedDeletedDir()) {
+      osib.setDeepCleanedDeletedDir(
+          snapshotInfoProto.getDeepCleanedDeletedDir());
+    }
+
     osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath())
         .setCheckpointDir(snapshotInfoProto.getCheckpointDir())
         .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber());
@@ -622,6 +639,14 @@ public long getExclusiveReplicatedSize() {
     return exclusiveReplicatedSize;
   }
 
+  public boolean getDeepCleanedDeletedDir() {
+    return deepCleanedDeletedDir;
+  }
+
+  public void setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) {
+    this.deepCleanedDeletedDir = deepCleanedDeletedDir;
+  }
+
   /**
    * Generate default name of snapshot, (used if user doesn't provide one).
    */
@@ -655,7 +680,8 @@ public static SnapshotInfo newInstance(String volumeName,
         .setSnapshotPath(volumeName + OM_KEY_PREFIX + bucketName)
         .setVolumeName(volumeName)
         .setBucketName(bucketName)
-        .setDeepClean(true);
+        .setDeepClean(false)
+        .setDeepCleanedDeletedDir(false);
 
     if (snapshotId != null) {
       builder.setCheckpointDir(getCheckpointDirName(snapshotId));
@@ -688,7 +714,8 @@ public boolean equals(Object o) {
         referencedSize == that.referencedSize &&
         referencedReplicatedSize == that.referencedReplicatedSize &&
         exclusiveSize == that.exclusiveSize &&
-        exclusiveReplicatedSize == that.exclusiveReplicatedSize;
+        exclusiveReplicatedSize == that.exclusiveReplicatedSize &&
+        deepCleanedDeletedDir == that.deepCleanedDeletedDir;
   }
 
   @Override
@@ -699,7 +726,7 @@ public int hashCode() {
         globalPreviousSnapshotId, snapshotPath, checkpointDir,
         deepClean, sstFiltered,
         referencedSize, referencedReplicatedSize,
-        exclusiveSize, exclusiveReplicatedSize);
+        exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir);
   }
 
   /**
@@ -726,6 +753,7 @@ public SnapshotInfo copyObject() {
         .setReferencedReplicatedSize(referencedReplicatedSize)
         .setExclusiveSize(exclusiveSize)
         .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+        .setDeepCleanedDeletedDir(deepCleanedDeletedDir)
         .build();
   }
 
@@ -750,6 +778,7 @@ public String toString() {
         ", referencedReplicatedSize: '" + referencedReplicatedSize + '\'' +
         ", exclusiveSize: '" + exclusiveSize + '\'' +
         ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' +
+        ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' +
         '}';
   }
 }

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java

@@ -66,12 +66,13 @@ private SnapshotInfo createSnapshotInfo() {
         .setSnapshotPath(SNAPSHOT_PATH)
         .setCheckpointDir(CHECKPOINT_DIR)
         .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER)
-        .setDeepClean(true)
+        .setDeepClean(false)
         .setSstFiltered(false)
         .setReferencedSize(2000L)
         .setReferencedReplicatedSize(6000L)
         .setExclusiveSize(1000L)
         .setExclusiveReplicatedSize(3000L)
+        .setDeepCleanedDeletedDir(false)
         .build();
   }
 
@@ -89,12 +90,13 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() {
         .setSnapshotPath(SNAPSHOT_PATH)
         .setCheckpointDir(CHECKPOINT_DIR)
         .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER)
-        .setDeepClean(true)
+        .setDeepClean(false)
         .setSstFiltered(false)
         .setReferencedSize(2000L)
         .setReferencedReplicatedSize(6000L)
         .setExclusiveSize(1000L)
         .setExclusiveReplicatedSize(3000L)
+        .setDeepCleanedDeletedDir(false)
         .build();
   }
 
@@ -140,6 +142,9 @@ public void testSnapshotInfoToProto() {
     Assertions.assertEquals(
         snapshotInfoEntryExpected.getExclusiveReplicatedSize(),
         snapshotInfoEntryActual.getExclusiveReplicatedSize());
+    Assertions.assertEquals(
+        snapshotInfoEntryExpected.getDeepCleanedDeletedDir(),
+        snapshotInfoEntryActual.getDeepCleanedDeletedDir());
 
     Assertions.assertEquals(snapshotInfoEntryExpected, snapshotInfoEntryActual);
   }
@@ -176,6 +181,8 @@ public void testSnapshotInfoProtoToSnapshotInfo() {
         snapshotInfoActual.getExclusiveSize());
     Assertions.assertEquals(snapshotInfoExpected.getExclusiveReplicatedSize(),
         snapshotInfoActual.getExclusiveReplicatedSize());
+    Assertions.assertEquals(snapshotInfoExpected.getDeepCleanedDeletedDir(),
+        snapshotInfoActual.getDeepCleanedDeletedDir());
 
     Assertions.assertEquals(snapshotInfoExpected, snapshotInfoActual);
   }