HBASE-26587 Introduce a new Admin API to change SFT implementation (#…

…4030)

Signed-off-by: Wellington Ramos Chevreuil <wchevreuil@apache.org>
Reviewed-by: Josh Elser <elserj@apache.org>

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

@@ -69,7 +69,6 @@
 import org.apache.yetus.audience.InterfaceAudience;
 
 import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
-import org.apache.yetus.audience.InterfaceStability;
 
 /**
  * The administrative API for HBase. Obtain an instance from {@link Connection#getAdmin()} and
@@ -500,6 +499,31 @@ default void modifyColumnFamily(TableName tableName, ColumnFamilyDescriptor colu
   Future<Void> modifyColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamily)
       throws IOException;
 
+  /**
+   * Change the store file tracker of the given table's given family.
+   * @param tableName the table you want to change
+   * @param family the family you want to change
+   * @param dstSFT the destination store file tracker
+   * @throws IOException if a remote or network exception occurs
+   */
+  default void modifyColumnFamilyStoreFileTracker(TableName tableName, byte[] family, String dstSFT)
+    throws IOException {
+    get(modifyColumnFamilyStoreFileTrackerAsync(tableName, family, dstSFT), getSyncWaitTimeout(),
+      TimeUnit.MILLISECONDS);
+  }
+
+  /**
+   * Change the store file tracker of the given table's given family.
+   * @param tableName the table you want to change
+   * @param family the family you want to change
+   * @param dstSFT the destination store file tracker
+   * @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the
+   *         operation to complete
+   * @throws IOException if a remote or network exception occurs
+   */
+  Future<Void> modifyColumnFamilyStoreFileTrackerAsync(TableName tableName, byte[] family,
+    String dstSFT) throws IOException;
+
   /**
    * Get all the online regions on a region server.
    *
@@ -1058,6 +1082,28 @@ default void modifyTable(TableDescriptor td) throws IOException {
    */
   Future<Void> modifyTableAsync(TableDescriptor td) throws IOException;
 
+  /**
+   * Change the store file tracker of the given table.
+   * @param tableName the table you want to change
+   * @param dstSFT the destination store file tracker
+   * @throws IOException if a remote or network exception occurs
+   */
+  default void modifyTableStoreFileTracker(TableName tableName, String dstSFT) throws IOException {
+    get(modifyTableStoreFileTrackerAsync(tableName, dstSFT), getSyncWaitTimeout(),
+      TimeUnit.MILLISECONDS);
+  }
+
+  /**
+   * Change the store file tracker of the given table.
+   * @param tableName the table you want to change
+   * @param dstSFT the destination store file tracker
+   * @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the
+   *         operation to complete
+   * @throws IOException if a remote or network exception occurs
+   */
+  Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT)
+    throws IOException;
+
   /**
    * Shuts down the HBase cluster.
    * <p/>

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java

@@ -230,10 +230,16 @@ public Future<Void> deleteColumnFamilyAsync(TableName tableName, byte[] columnFa
 
   @Override
   public Future<Void> modifyColumnFamilyAsync(TableName tableName,
-      ColumnFamilyDescriptor columnFamily) throws IOException {
+    ColumnFamilyDescriptor columnFamily) throws IOException {
     return admin.modifyColumnFamily(tableName, columnFamily);
   }
 
+  @Override
+  public Future<Void> modifyColumnFamilyStoreFileTrackerAsync(TableName tableName, byte[] family,
+    String dstSFT) throws IOException {
+    return admin.modifyColumnFamilyStoreFileTracker(tableName, family, dstSFT);
+  }
+
   @Override
   public List<RegionInfo> getRegions(ServerName serverName) throws IOException {
     return get(admin.getRegions(serverName));
@@ -475,6 +481,12 @@ public Future<Void> modifyTableAsync(TableDescriptor td) throws IOException {
     return admin.modifyTable(td);
   }
 
+  @Override
+  public Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT)
+    throws IOException {
+    return admin.modifyTableStoreFileTracker(tableName, dstSFT);
+  }
+
   @Override
   public void shutdown() throws IOException {
     get(admin.shutdown());

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java

@@ -183,6 +183,13 @@ CompletableFuture<Void> createTable(TableDescriptor desc, byte[] startKey, byte[
    */
   CompletableFuture<Void> modifyTable(TableDescriptor desc);
 
+  /**
+   * Change the store file tracker of the given table.
+   * @param tableName the table you want to change
+   * @param dstSFT the destination store file tracker
+   */
+  CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT);
+
   /**
    * Deletes a table.
    * @param tableName name of table to delete
@@ -252,6 +259,15 @@ CompletableFuture<Void> addColumnFamily(TableName tableName,
   CompletableFuture<Void> modifyColumnFamily(TableName tableName,
       ColumnFamilyDescriptor columnFamily);
 
+  /**
+   * Change the store file tracker of the given table's given family.
+   * @param tableName the table you want to change
+   * @param family the family you want to change
+   * @param dstSFT the destination store file tracker
+   */
+  CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName, byte[] family,
+    String dstSFT);
+
   /**
    * Create a new namespace.
    * @param descriptor descriptor which describes the new namespace

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java

@@ -146,6 +146,11 @@ public CompletableFuture<Void> modifyTable(TableDescriptor desc) {
     return wrap(rawAdmin.modifyTable(desc));
   }
 
+  @Override
+  public CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT) {
+    return wrap(rawAdmin.modifyTableStoreFileTracker(tableName, dstSFT));
+  }
+
   @Override
   public CompletableFuture<Void> deleteTable(TableName tableName) {
     return wrap(rawAdmin.deleteTable(tableName));
@@ -198,6 +203,12 @@ public CompletableFuture<Void> modifyColumnFamily(TableName tableName,
     return wrap(rawAdmin.modifyColumnFamily(tableName, columnFamily));
   }
 
+  @Override
+  public CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName,
+    byte[] family, String dstSFT) {
+    return wrap(rawAdmin.modifyColumnFamilyStoreFileTracker(tableName, family, dstSFT));
+  }
+
   @Override
   public CompletableFuture<Void> createNamespace(NamespaceDescriptor descriptor) {
     return wrap(rawAdmin.createNamespace(descriptor));

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java

@@ -21,6 +21,7 @@
 import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
 import static org.apache.hadoop.hbase.util.FutureUtils.addListener;
 import static org.apache.hadoop.hbase.util.FutureUtils.unwrapCompletionException;
+
 import edu.umd.cs.findbugs.annotations.Nullable;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -230,10 +231,14 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
@@ -660,6 +665,18 @@ public CompletableFuture<Void> modifyTable(TableDescriptor desc) {
       (resp) -> resp.getProcId(), new ModifyTableProcedureBiConsumer(this, desc.getTableName()));
   }
 
+  @Override
+  public CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT) {
+    return this
+      .<ModifyTableStoreFileTrackerRequest, ModifyTableStoreFileTrackerResponse> procedureCall(
+        tableName,
+        RequestConverter.buildModifyTableStoreFileTrackerRequest(tableName, dstSFT,
+          ng.getNonceGroup(), ng.newNonce()),
+        (s, c, req, done) -> s.modifyTableStoreFileTracker(c, req, done),
+        (resp) -> resp.getProcId(),
+        new ModifyTableStoreFileTrackerProcedureBiConsumer(this, tableName));
+  }
+
   @Override
   public CompletableFuture<Void> deleteTable(TableName tableName) {
     return this.<DeleteTableRequest, DeleteTableResponse> procedureCall(tableName,
@@ -808,6 +825,19 @@ public CompletableFuture<Void> modifyColumnFamily(TableName tableName,
       (resp) -> resp.getProcId(), new ModifyColumnFamilyProcedureBiConsumer(tableName));
   }
 
+  @Override
+  public CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName,
+    byte[] family, String dstSFT) {
+    return this
+      .<ModifyColumnStoreFileTrackerRequest, ModifyColumnStoreFileTrackerResponse> procedureCall(
+        tableName,
+        RequestConverter.buildModifyColumnStoreFileTrackerRequest(tableName, family, dstSFT,
+          ng.getNonceGroup(), ng.newNonce()),
+        (s, c, req, done) -> s.modifyColumnStoreFileTracker(c, req, done),
+        (resp) -> resp.getProcId(),
+        new ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer(tableName));
+  }
+
   @Override
   public CompletableFuture<Void> createNamespace(NamespaceDescriptor descriptor) {
     return this.<CreateNamespaceRequest, CreateNamespaceResponse> procedureCall(
@@ -2590,7 +2620,20 @@ private static class ModifyTableProcedureBiConsumer extends TableProcedureBiCons
 
     @Override
     String getOperationType() {
-      return "ENABLE";
+      return "MODIFY";
+    }
+  }
+
+  private static class ModifyTableStoreFileTrackerProcedureBiConsumer
+    extends TableProcedureBiConsumer {
+
+    ModifyTableStoreFileTrackerProcedureBiConsumer(AsyncAdmin admin, TableName tableName) {
+      super(tableName);
+    }
+
+    @Override
+    String getOperationType() {
+      return "MODIFY_TABLE_STORE_FILE_TRACKER";
     }
   }
 
@@ -2684,6 +2727,19 @@ String getOperationType() {
     }
   }
 
+  private static class ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer
+    extends TableProcedureBiConsumer {
+
+    ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer(TableName tableName) {
+      super(tableName);
+    }
+
+    @Override
+    String getOperationType() {
+      return "MODIFY_COLUMN_FAMILY_STORE_FILE_TRACKER";
+    }
+  }
+
   private static class CreateNamespaceProcedureBiConsumer extends NamespaceProcedureBiConsumer {
 
     CreateNamespaceProcedureBiConsumer(String namespaceName) {
@@ -4282,5 +4338,4 @@ public CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNam
         return CompletableFuture.completedFuture(Collections.emptyList());
     }
   }
-
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java

@@ -69,6 +69,7 @@
 import org.apache.yetus.audience.InterfaceAudience;
 
 import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
 import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
 import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils;
 
@@ -128,8 +129,10 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineRegionRequest;
@@ -1022,6 +1025,19 @@ public static ModifyColumnRequest buildModifyColumnRequest(
     return builder.build();
   }
 
+  public static ModifyColumnStoreFileTrackerRequest
+    buildModifyColumnStoreFileTrackerRequest(final TableName tableName, final byte[] family,
+      final String dstSFT, final long nonceGroup, final long nonce) {
+    ModifyColumnStoreFileTrackerRequest.Builder builder =
+      ModifyColumnStoreFileTrackerRequest.newBuilder();
+    builder.setTableName(ProtobufUtil.toProtoTableName((tableName)));
+    builder.setFamily(ByteString.copyFrom(family));
+    builder.setDstSft(dstSFT);
+    builder.setNonceGroup(nonceGroup);
+    builder.setNonce(nonce);
+    return builder.build();
+  }
+
   /**
    * Create a protocol buffer MoveRegionRequest
    * @param encodedRegionName
@@ -1220,6 +1236,17 @@ public static ModifyTableRequest buildModifyTableRequest(
     return builder.build();
   }
 
+  public static ModifyTableStoreFileTrackerRequest buildModifyTableStoreFileTrackerRequest(
+    final TableName tableName, final String dstSFT, final long nonceGroup, final long nonce) {
+    ModifyTableStoreFileTrackerRequest.Builder builder =
+      ModifyTableStoreFileTrackerRequest.newBuilder();
+    builder.setTableName(ProtobufUtil.toProtoTableName((tableName)));
+    builder.setDstSft(dstSFT);
+    builder.setNonceGroup(nonceGroup);
+    builder.setNonce(nonce);
+    return builder.build();
+  }
+
   /**
    * Creates a protocol buffer GetTableDescriptorsRequest
    *

hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto

@@ -732,6 +732,29 @@ message BalancerRejectionsResponse {
   repeated BalancerRejection balancer_rejection = 1;
 }
 
+message ModifyTableStoreFileTrackerRequest {
+  required TableName table_Name = 1;
+  required string dst_sft = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyTableStoreFileTrackerResponse {
+  optional uint64 proc_id = 1;
+}
+
+message ModifyColumnStoreFileTrackerRequest {
+  required TableName table_Name = 1;
+  required bytes family = 2;
+  required string dst_sft = 3;
+  optional uint64 nonce_group = 4 [default = 0];
+  optional uint64 nonce = 5 [default = 0];
+}
+
+message ModifyColumnStoreFileTrackerResponse {
+  optional uint64 proc_id = 1;
+}
+
 service MasterService {
   /** Used by the client to get the number of regions that have received the updated schema */
   rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
@@ -1165,6 +1188,12 @@ service MasterService {
 
   rpc GetLogEntries(LogRequest)
     returns(LogEntry);
+
+  rpc ModifyTableStoreFileTracker(ModifyTableStoreFileTrackerRequest)
+    returns(ModifyTableStoreFileTrackerResponse);
+
+  rpc ModifyColumnStoreFileTracker(ModifyColumnStoreFileTrackerRequest)
+    returns(ModifyColumnStoreFileTrackerResponse);
 }
 
 // HBCK Service definitions.

hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto

@@ -658,3 +658,18 @@ message ModifyTableDescriptorStateData {
   required TableName table_name = 1;
   optional TableSchema modified_table_schema = 2;
 }
+
+enum ModifyStoreFileTrackerState {
+  MODIFY_STORE_FILE_TRACKER_FINISH_PREVIOUS_MIGRATION = 1;
+  MODIFY_STORE_FILE_TRACKER_START_MIGRATION = 2;
+  MODIFY_STORE_FILE_TRACKER_FINISH_MIGRATION = 3;
+}
+
+message ModifyStoreFileTrackerStateData {
+  required TableName table_name = 1;
+  required string dst_sft = 2;
+}
+
+message ModifyColumnFamilyStoreFileTrackerStateData {
+  required bytes family = 1;
+}