HBASE-22648 Snapshot TTL (#371)

Signed-off-by: Reid Chan <reidchan@apache.org> Signed-off-by: Andrew Purtell <apurtell@apache.org> Co-authored-by: Andrew Purtell <apurtell@apache.org> Conflicts: hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java hbase-common/src/main/resources/hbase-default.xml hbase-protocol-shaded/src/main/protobuf/Snapshot.proto hbase-protocol/src/main/protobuf/HBase.proto hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotTemporaryDirectory.java hbase-shell/src/main/ruby/hbase/admin.rb src/main/asciidoc/_chapters/hbase-default.adoc
apache · Jul 23, 2019 · f7bdf76 · f7bdf76
1 parent d38078a
commit f7bdf76
Show file tree

Hide file tree

Showing 17 changed files with 653 additions and 42 deletions.
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -1183,17 +1183,38 @@ void snapshot(final String snapshotName,
       IllegalArgumentException;
 
   /**
-   * Take a snapshot and wait for the server to complete that snapshot (blocking). Only a single
-   * snapshot should be taken at a time for an instance of HBase, or results may be undefined (you
-   * can tell multiple HBase clusters to snapshot at the same time, but only one at a time for a
-   * single cluster). Snapshots are considered unique based on <b>the name of the snapshot</b>.
-   * Attempts to take a snapshot with the same name (even a different type or with different
-   * parameters) will fail with a {@link SnapshotCreationException} indicating the duplicate naming.
-   * Snapshot names follow the same naming constraints as tables in HBase. See {@link
-   * org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}. You should probably
-   * use {@link #snapshot(String, org.apache.hadoop.hbase.TableName)} or
-   * {@link #snapshot(byte[], org.apache.hadoop.hbase.TableName)} unless you are sure about the type
-   * of snapshot that you want to take.
+   * Create typed snapshot of the table. Snapshots are considered unique based on <b>the name of the
+   * snapshot</b>. Snapshots are taken sequentially even when requested concurrently, across
+   * all tables. Attempts to take a snapshot with the same name (even a different type or with
+   * different parameters) will fail with a {@link SnapshotCreationException} indicating the
+   * duplicate naming. Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+   * Snapshot can live with ttl seconds.
+   *
+   * @param snapshotName  name to give the snapshot on the filesystem. Must be unique from all
+   *                      other snapshots stored on the cluster
+   * @param tableName     name of the table to snapshot
+   * @param type          type of snapshot to take
+   * @param snapshotProps snapshot additional properties e.g. TTL
+   * @throws IOException               we fail to reach the master
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException  if the snapshot request is formatted incorrectly
+   */
+  void snapshot(String snapshotName, TableName tableName, SnapshotDescription.Type type,
+      Map<String, Object> snapshotProps)
+      throws IOException, SnapshotCreationException, IllegalArgumentException;
+
+  /**
+   * Take a snapshot and wait for the server to complete that snapshot (blocking). Snapshots are
+   * considered unique based on <b>the name of the snapshot</b>. Snapshots are taken sequentially
+   * even when requested concurrently, across all tables. Attempts to take a snapshot with the
+   * same name (even a different type or with different parameters) will fail with a
+   * {@link SnapshotCreationException} indicating the duplicate naming. Snapshot names follow the
+   * same naming constraints as tables in HBase. See
+   * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}. You should
+   * probably use {@link #snapshot(String, org.apache.hadoop.hbase.TableName)} or
+   * {@link #snapshot(byte[], org.apache.hadoop.hbase.TableName)} unless you are sure about the
+   * type of snapshot that you want to take.
    *
    * @param snapshot snapshot to take
    * @throws IOException or we lose contact with the master.

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -38,7 +38,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
-
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -3611,8 +3611,8 @@ public void snapshot(final String snapshotName,
    * Create snapshot for the given table of given flush type.
    * <p>
    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
-   * snapshot with the same name (even a different type or with different parameters) will fail with
-   * a {@link SnapshotCreationException} indicating the duplicate naming.
+   * snapshot with the same name (even a different type or with different parameters) will fail
+   * with a {@link SnapshotCreationException} indicating the duplicate naming.
    * <p>
    * Snapshot names follow the same naming constraints as tables in HBase.
    * @param snapshotName name of the snapshot to be created
@@ -3627,6 +3627,30 @@ public void snapshot(final byte[] snapshotName, final byte[] tableName,
       IOException, SnapshotCreationException, IllegalArgumentException {
       snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType);
   }
+
+  /**
+   * Create snapshot for the given table of given flush type.
+   * <p>
+   * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
+   * snapshot with the same name (even a different type or with different parameters) will fail
+   * with a {@link SnapshotCreationException} indicating the duplicate naming.
+   * <p>
+   * Snapshot names follow the same naming constraints as tables in HBase.
+   * @param snapshotName name of the snapshot to be created
+   * @param tableName name of the table for which snapshot is created
+   * @param flushType if the snapshot should be taken without flush memstore first
+   * @param snapshotProps snapshot parameters
+   * @throws IOException if a remote or network exception occurs
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
+   */
+   public void snapshot(final byte[] snapshotName, final byte[] tableName,
+       final SnapshotDescription.Type flushType, Map<String,Object> snapshotProps)
+       throws IOException, SnapshotCreationException, IllegalArgumentException {
+     snapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName), flushType,
+       snapshotProps);
+   }
+
   /**
    public void snapshot(final String snapshotName,
     * Create a timestamp consistent snapshot for the given table.
@@ -3671,34 +3695,46 @@ public void snapshot(final byte[] snapshotName,
    *          snapshots stored on the cluster
    * @param tableName name of the table to snapshot
    * @param type type of snapshot to take
+   * @param snapshotProps snapshot parameters
    * @throws IOException we fail to reach the master
    * @throws SnapshotCreationException if snapshot creation failed
    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
    */
   @Override
-  public void snapshot(final String snapshotName,
-                       final TableName tableName,
-                      SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
-      IllegalArgumentException {
+  public void snapshot(final String snapshotName, final TableName tableName,
+      SnapshotDescription.Type type, Map<String,Object> snapshotProps)
+      throws IOException, SnapshotCreationException, IllegalArgumentException {
     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
     builder.setTable(tableName.getNameAsString());
     builder.setName(snapshotName);
     builder.setType(type);
+    builder.setTtl(getTtlFromSnapshotProps(snapshotProps));
     snapshot(builder.build());
   }
 
+  private long getTtlFromSnapshotProps(Map<String, Object> snapshotProps) {
+    return MapUtils.getLongValue(snapshotProps, "TTL", -1);
+  }
+
+  public void snapshot(final String snapshotName,
+      final TableName tableName,
+     SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
+  IllegalArgumentException {
+    snapshot(snapshotName, tableName, type, null);
+  }
+
   public void snapshot(final String snapshotName,
                        final String tableName,
                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
       IllegalArgumentException {
-    snapshot(snapshotName, TableName.valueOf(tableName), type);
+    snapshot(snapshotName, TableName.valueOf(tableName), type, null);
   }
 
   public void snapshot(final String snapshotName,
                        final byte[] tableName,
                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
       IllegalArgumentException {
-    snapshot(snapshotName, TableName.valueOf(tableName), type);
+    snapshot(snapshotName, TableName.valueOf(tableName), type, null);
   }
 
   /**

diff --git a/...client/src/main/java/org/apache/hadoop/hbase/snapshot/ClientSnapshotDescriptionUtils.java b/...client/src/main/java/org/apache/hadoop/hbase/snapshot/ClientSnapshotDescriptionUtils.java
@@ -61,8 +61,15 @@ public static String toString(SnapshotDescription ssd) {
     if (ssd == null) {
       return null;
     }
-    return "{ ss=" + ssd.getName() +
-           " table=" + (ssd.hasTable()?TableName.valueOf(ssd.getTable()):"") +
-           " type=" + ssd.getType() + " }";
+    return new StringBuilder("{ ss=")
+            .append(ssd.getName())
+            .append(" table=")
+            .append(ssd.hasTable() ? TableName.valueOf(ssd.getTable()) : "")
+            .append(" type=")
+            .append(ssd.getType())
+            .append(" ttl=")
+            .append(ssd.getTtl())
+            .append(" }")
+            .toString();
   }
 }
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1331,6 +1331,15 @@ public static enum Modify {
       + System.getProperty("user.name") + "/hbase-staging";
   public static final String DEFAULT_LOSSY_COUNTING_ERROR_RATE =
       "hbase.util.default.lossycounting.errorrate";
+
+  // Default TTL - FOREVER
+  public static final long DEFAULT_SNAPSHOT_TTL = 0;
+
+  // User defined Default TTL config key
+  public static final String DEFAULT_SNAPSHOT_TTL_CONFIG_KEY = "hbase.master.snapshot.ttl";
+
+  public static final String SNAPSHOT_CLEANER_DISABLE = "hbase.master.cleaner.snapshot.disable";
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }

diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
@@ -1624,4 +1624,23 @@ possible configurations would overwhelm and obscure the important.
       Number of rows in a batch operation above which a warning will be logged.
     </description>
   </property>
+  <property>
+    <name>hbase.master.cleaner.snapshot.interval</name>
+    <value>1800000</value>
+    <description>
+      Snapshot Cleanup chore interval in milliseconds.
+      The cleanup thread keeps running at this interval
+      to find all snapshots that are expired based on TTL
+      and delete them.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.snapshot.ttl</name>
+    <value>0</value>
+    <description>
+      Default Snapshot TTL to be considered when the user does not specify TTL while
+      creating snapshot. Default value 0 indicates FOREVER - snapshot should not be
+      automatically deleted until it is manually deleted
+    </description>
+  </property>
 </configuration>