HBASE-22648 : Introducing Snapshot TTL

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

@@ -1372,6 +1372,53 @@ default void snapshot(String snapshotName, TableName tableName, SnapshotType typ
     snapshot(new SnapshotDescription(snapshotName, tableName, type));
   }
 
+  /**
+   * Create typed snapshot of the table. Snapshots are considered unique based on <b>the name of the
+   * snapshot</b>. Snapshots are taken sequentially even when requested concurrently, across
+   * all tables. Attempts to take a snapshot with the same name (even a different type or with
+   * different parameters) will fail with a {@link SnapshotCreationException} indicating the
+   * duplicate naming. Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+   * Snapshot can live with ttl seconds.
+   *
+   * @param snapshotName  name to give the snapshot on the filesystem. Must be unique from all other
+   *                      snapshots stored on the cluster
+   * @param tableName     name of the table to snapshot
+   * @param type          type of snapshot to take
+   * @param snapshotProps snapshot additional properties e.g. TTL
+   * @throws IOException               we fail to reach the master
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException  if the snapshot request is formatted incorrectly
+   */
+  default void snapshot(String snapshotName, TableName tableName, SnapshotType type,
+                        Map<String, Object> snapshotProps) throws IOException,
+      SnapshotCreationException, IllegalArgumentException {
+    snapshot(new SnapshotDescription(snapshotName, tableName, type, snapshotProps));
+  }
+
+  /**
+   * Create typed snapshot of the table. Snapshots are considered unique based on <b>the name of the
+   * snapshot</b>. Snapshots are taken sequentially even when requested concurrently, across
+   * all tables. Attempts to take a snapshot with the same name (even a different type or with
+   * different parameters) will fail with a {@link SnapshotCreationException} indicating the
+   * duplicate naming. Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+   * Snapshot can live with ttl seconds.
+   *
+   * @param snapshotName  name to give the snapshot on the filesystem. Must be unique from all other
+   *                      snapshots stored on the cluster
+   * @param tableName     name of the table to snapshot
+   * @param snapshotProps snapshot additional properties e.g. TTL
+   * @throws IOException               we fail to reach the master
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException  if the snapshot request is formatted incorrectly
+   */
+  default void snapshot(String snapshotName, TableName tableName,
+                        Map<String, Object> snapshotProps) throws IOException,
+      SnapshotCreationException, IllegalArgumentException {
+    snapshot(new SnapshotDescription(snapshotName, tableName, SnapshotType.FLUSH, snapshotProps));
+  }
+
   /**
    * Take a snapshot and wait for the server to complete that snapshot (blocking). Snapshots are
    * considered unique based on <b>the name of the snapshot</b>. Snapshots are taken sequentially

hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java

@@ -17,9 +17,13 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import java.util.Map;
+
 import org.apache.hadoop.hbase.TableName;
 import org.apache.yetus.audience.InterfaceAudience;
 
+import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils;
+
 /**
  * The POJO equivalent of HBaseProtos.SnapshotDescription
  */
@@ -30,6 +34,7 @@ public class SnapshotDescription {
   private final SnapshotType snapShotType;
   private final String owner;
   private final long creationTime;
+  private final long ttl;
   private final int version;
 
   public SnapshotDescription(String name) {
@@ -48,7 +53,7 @@ public SnapshotDescription(String name, String table) {
   }
 
   public SnapshotDescription(String name, TableName table) {
-    this(name, table, SnapshotType.DISABLED, null);
+    this(name, table, SnapshotType.DISABLED, null, -1, -1, null);
   }
 
   /**
@@ -63,46 +68,75 @@ public SnapshotDescription(String name, String table, SnapshotType type) {
   }
 
   public SnapshotDescription(String name, TableName table, SnapshotType type) {
-    this(name, table, type, null);
+    this(name, table, type, null, -1, -1, null);
   }
 
   /**
-   * @deprecated since 2.0.0 and will be removed in 3.0.0. Use the version with the TableName
-   *   instance instead.
    * @see #SnapshotDescription(String, TableName, SnapshotType, String)
    * @see <a href="https://issues.apache.org/jira/browse/HBASE-16892">HBASE-16892</a>
+   * @deprecated since 2.0.0 and will be removed in 3.0.0. Use the version with the TableName
+   *   instance instead.
    */
   @Deprecated
   public SnapshotDescription(String name, String table, SnapshotType type, String owner) {
     this(name, TableName.valueOf(table), type, owner);
   }
 
   public SnapshotDescription(String name, TableName table, SnapshotType type, String owner) {
-    this(name, table, type, owner, -1, -1);
+    this(name, table, type, owner, -1, -1, null);
   }
 
   /**
+   * @see #SnapshotDescription(String, TableName, SnapshotType, String, long, int, Map)
+   * @see <a href="https://issues.apache.org/jira/browse/HBASE-16892">HBASE-16892</a>
    * @deprecated since 2.0.0 and will be removed in 3.0.0. Use the version with the TableName
    *   instance instead.
-   * @see #SnapshotDescription(String, TableName, SnapshotType, String, long, int)
-   * @see <a href="https://issues.apache.org/jira/browse/HBASE-16892">HBASE-16892</a>
    */
   @Deprecated
   public SnapshotDescription(String name, String table, SnapshotType type, String owner,
       long creationTime, int version) {
-    this(name, TableName.valueOf(table), type, owner, creationTime, version);
+    this(name, TableName.valueOf(table), type, owner, creationTime, version, null);
   }
 
+  /**
+   * SnapshotDescription Parameterized Constructor
+   *
+   * @param name          Name of the snapshot
+   * @param table         TableName associated with the snapshot
+   * @param type          Type of the snapshot - enum SnapshotType
+   * @param owner         Snapshot Owner
+   * @param creationTime  Creation time for Snapshot
+   * @param version       Snapshot Version
+   * @param snapshotProps Additional properties for snapshot e.g. TTL
+   */
   public SnapshotDescription(String name, TableName table, SnapshotType type, String owner,
-      long creationTime, int version) {
+                             long creationTime, int version, Map<String, Object> snapshotProps) {
     this.name = name;
     this.table = table;
     this.snapShotType = type;
     this.owner = owner;
     this.creationTime = creationTime;
+    this.ttl = getTtlFromSnapshotProps(snapshotProps);
     this.version = version;
   }
 
+  private long getTtlFromSnapshotProps(Map<String, Object> snapshotProps) {
+    return MapUtils.getLongValue(snapshotProps, "TTL", -1);
+  }
+
+  /**
+   * SnapshotDescription Parameterized Constructor
+   *
+   * @param snapshotName  Name of the snapshot
+   * @param tableName     TableName associated with the snapshot
+   * @param type          Type of the snapshot - enum SnapshotType
+   * @param snapshotProps Additional properties for snapshot e.g. TTL
+   */
+  public SnapshotDescription(String snapshotName, TableName tableName, SnapshotType type,
+                             Map<String, Object> snapshotProps) {
+    this(snapshotName, tableName, type, null, -1, -1, snapshotProps);
+  }
+
   public String getName() {
     return this.name;
   }
@@ -139,15 +173,27 @@ public long getCreationTime() {
     return this.creationTime;
   }
 
+  // get snapshot ttl in sec
+  public long getTtl() {
+    return ttl;
+  }
+
   public int getVersion() {
     return this.version;
   }
 
   @Override
   public String toString() {
-    return "SnapshotDescription: name = " + ((name != null) ? name : null) + "/table = "
-        + ((table != null) ? table : null) + " /owner = " + ((owner != null) ? owner : null)
-        + (creationTime != -1 ? ("/creationtime = " + creationTime) : "")
-        + (version != -1 ? ("/version = " + version) : "");
+    return new StringBuilder("SnapshotDescription: ")
+            .append("name = ")
+            .append(name)
+            .append("/table = ")
+            .append(table)
+            .append(" /owner = ")
+            .append(owner)
+            .append(creationTime != -1 ? ("/creationtime = " + creationTime) : "")
+            .append(ttl != -1 ? ("/ttl = " + ttl) : "")
+            .append(version != -1 ? ("/version = " + version) : "")
+            .toString();
   }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java

@@ -29,6 +29,7 @@
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -2932,12 +2933,15 @@ public static SnapshotType createSnapshotType(SnapshotProtos.SnapshotDescription
     if (snapshotDesc.getCreationTime() != -1L) {
       builder.setCreationTime(snapshotDesc.getCreationTime());
     }
+    if (snapshotDesc.getTtl() != -1L &&
+        snapshotDesc.getTtl() < TimeUnit.MILLISECONDS.toSeconds(Long.MAX_VALUE)) {
+      builder.setTtl(snapshotDesc.getTtl());
+    }
     if (snapshotDesc.getVersion() != -1) {
       builder.setVersion(snapshotDesc.getVersion());
     }
     builder.setType(ProtobufUtil.createProtosSnapShotDescType(snapshotDesc.getType()));
-    SnapshotProtos.SnapshotDescription snapshot = builder.build();
-    return snapshot;
+    return builder.build();
   }
 
   /**
@@ -2949,10 +2953,12 @@ public static SnapshotType createSnapshotType(SnapshotProtos.SnapshotDescription
    */
   public static SnapshotDescription
       createSnapshotDesc(SnapshotProtos.SnapshotDescription snapshotDesc) {
+    final Map<String, Object> snapshotProps = new HashMap<>();
+    snapshotProps.put("TTL", snapshotDesc.getTtl());
     return new SnapshotDescription(snapshotDesc.getName(),
-        snapshotDesc.hasTable() ? TableName.valueOf(snapshotDesc.getTable()) : null,
-        createSnapshotType(snapshotDesc.getType()), snapshotDesc.getOwner(),
-        snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
+            snapshotDesc.hasTable() ? TableName.valueOf(snapshotDesc.getTable()) : null,
+            createSnapshotType(snapshotDesc.getType()), snapshotDesc.getOwner(),
+            snapshotDesc.getCreationTime(), snapshotDesc.getVersion(), snapshotProps);
   }
 
   public static RegionLoadStats createRegionLoadStats(ClientProtos.RegionLoadStats stats) {

hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ClientSnapshotDescriptionUtils.java

@@ -61,8 +61,15 @@ public static String toString(SnapshotProtos.SnapshotDescription ssd) {
     if (ssd == null) {
       return null;
     }
-    return "{ ss=" + ssd.getName() +
-           " table=" + (ssd.hasTable()?TableName.valueOf(ssd.getTable()):"") +
-           " type=" + ssd.getType() + " }";
+    return new StringBuilder("{ ss=")
+            .append(ssd.getName())
+            .append(" table=")
+            .append(ssd.hasTable() ? TableName.valueOf(ssd.getTable()) : "")
+            .append(" type=")
+            .append(ssd.getType())
+            .append(" ttl=")
+            .append(ssd.getTtl())
+            .append(" }")
+            .toString();
   }
 }

hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java

@@ -1445,6 +1445,15 @@ public enum OperationStatusCode {
       "hbase.util.default.lossycounting.errorrate";
   public static final String NOT_IMPLEMENTED = "Not implemented";
 
+  // Default TTL - FOREVER
+  public static final long DEFAULT_SNAPSHOT_TTL = 0;
+
+  // User defined Default TTL config key
+  public static final String DEFAULT_SNAPSHOT_TTL_CONFIG_KEY = "hbase.master.snapshot.ttl";
+
+  public static final String SNAPSHOT_CLEANER_DISABLE = "hbase.master.cleaner.snapshot.disable";
+
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }

hbase-common/src/main/resources/hbase-default.xml

@@ -1864,4 +1864,23 @@ possible configurations would overwhelm and obscure the important.
     <description>Default is 5 minutes. Make it 30 seconds for tests. See
     HBASE-19794 for some context.</description>
   </property>
+  <property>
+    <name>hbase.master.cleaner.snapshot.interval</name>
+    <value>1800000</value>
+    <description>
+      Snapshot Cleanup chore interval in milliseconds.
+      The cleanup thread keeps running at this interval
+      to find all snapshots that are expired based on TTL
+      and delete them.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.snapshot.ttl</name>
+    <value>0</value>
+    <description>
+      Default Snapshot TTL to be considered when the user does not specify TTL while
+      creating snapshot. Default value 0 indicates FOREVERE - snapshot should not be
+      automatically deleted until it is manually deleted
+    </description>
+  </property>
 </configuration>

hbase-protocol-shaded/src/main/protobuf/Snapshot.proto

@@ -44,6 +44,7 @@ message SnapshotDescription {
   optional int32 version = 5;
   optional string owner = 6;
   optional UsersAndPermissions users_and_permissions = 7;
+  optional int64 ttl = 8 [default = 0];
 }
 
 message SnapshotFileInfo {

hbase-protocol/src/main/protobuf/HBase.proto

@@ -184,6 +184,7 @@ message SnapshotDescription {
   optional Type type = 4 [default = FLUSH];
   optional int32 version = 5;
   optional string owner = 6;
+  optional int64 ttl = 7 [default = 0];
 }
 
 /**

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -114,6 +114,7 @@
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
 import org.apache.hadoop.hbase.master.cleaner.ReplicationBarrierCleaner;
+import org.apache.hadoop.hbase.master.cleaner.SnapshotCleanerChore;
 import org.apache.hadoop.hbase.master.locking.LockManager;
 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
@@ -382,6 +383,7 @@ public void run() {
   private RegionNormalizerChore normalizerChore;
   private ClusterStatusChore clusterStatusChore;
   private ClusterStatusPublisher clusterStatusPublisherChore = null;
+  private SnapshotCleanerChore snapshotCleanerChore = null;
 
   CatalogJanitor catalogJanitorChore;
   private LogCleaner logCleaner;
@@ -1457,6 +1459,16 @@ this, conf, getMasterWalManager().getFileSystem(),
       replicationPeerManager);
     getChoreService().scheduleChore(replicationBarrierCleaner);
 
+    final boolean isSnapshotChoreDisabled = conf.getBoolean(HConstants.SNAPSHOT_CLEANER_DISABLE,
+        false);
+    if (isSnapshotChoreDisabled) {
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Snapshot Cleaner Chore is disabled. Not starting up the chore..");
+      }
+    } else {
+      this.snapshotCleanerChore = new SnapshotCleanerChore(this, conf, getSnapshotManager());
+      getChoreService().scheduleChore(this.snapshotCleanerChore);
+    }
     serviceStarted = true;
     if (LOG.isTraceEnabled()) {
       LOG.trace("Started service threads");
@@ -1574,6 +1586,7 @@ private void stopChores() {
       choreService.cancelChore(this.logCleaner);
       choreService.cancelChore(this.hfileCleaner);
       choreService.cancelChore(this.replicationBarrierCleaner);
+      choreService.cancelChore(this.snapshotCleanerChore);
     }
   }