HBASE-29448 Modern backup failures can cause backup system to lock up

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java

@@ -1403,9 +1403,7 @@ public static void restoreFromSnapshot(Connection conn) throws IOException {
     try (Admin admin = conn.getAdmin()) {
       String snapshotName = BackupSystemTable.getSnapshotName(conf);
       if (snapshotExists(admin, snapshotName)) {
-        admin.disableTable(BackupSystemTable.getTableName(conf));
-        admin.restoreSnapshot(snapshotName);
-        admin.enableTable(BackupSystemTable.getTableName(conf));
+        admin.restoreBackupSystemTable(snapshotName);
         LOG.debug("Done restoring backup system table");
       } else {
         // Snapshot does not exists, i.e completeBackup failed after

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestRestoreBackupSystemTable.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.master;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestRestoreBackupSystemTable {
+  private static final String BACKUP_ROOT = "root";
+  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    UTIL.startMiniCluster();
+  }
+
+  @Test
+  public void itRestoresFromSnapshot() throws Exception {
+    BackupSystemTable table = new BackupSystemTable(UTIL.getConnection());
+    Set<TableName> tables = new HashSet<>();
+
+    tables.add(TableName.valueOf("test1"));
+    tables.add(TableName.valueOf("test2"));
+    tables.add(TableName.valueOf("test3"));
+
+    Map<String, Long> rsTimestampMap = new HashMap<>();
+    rsTimestampMap.put("rs1:100", 100L);
+    rsTimestampMap.put("rs2:100", 101L);
+    rsTimestampMap.put("rs3:100", 103L);
+
+    table.writeRegionServerLogTimestamp(tables, rsTimestampMap, BACKUP_ROOT);
+    BackupSystemTable.snapshot(UTIL.getConnection());
+
+    Admin admin = UTIL.getAdmin();
+    TableName backupSystemTn = BackupSystemTable.getTableName(UTIL.getConfiguration());
+    admin.disableTable(backupSystemTn);
+    admin.truncateTable(backupSystemTn, true);
+
+    BackupSystemTable.restoreFromSnapshot(UTIL.getConnection());
+    Map<TableName, Map<String, Long>> results = table.readLogTimestampMap(BACKUP_ROOT);
+
+    assertEquals(results.size(), tables.size());
+
+    for (TableName tableName : tables) {
+      Map<String, Long> resultMap = results.get(tableName);
+      assertEquals(resultMap, rsTimestampMap);
+    }
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+}

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

@@ -2661,4 +2661,7 @@ List<LogEntry> getLogEntries(Set<ServerName> serverNames, String logType, Server
    * Get the list of cached files
    */
   List<String> getCachedFilesList(ServerName serverName) throws IOException;
+
+  @InterfaceAudience.Private
+  void restoreBackupSystemTable(String snapshotName) throws IOException;
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java

@@ -1141,4 +1141,9 @@ public void flushMasterStore() throws IOException {
   public List<String> getCachedFilesList(ServerName serverName) throws IOException {
     return get(admin.getCachedFilesList(serverName));
   }
+
+  @Override
+  public void restoreBackupSystemTable(String snapshotName) throws IOException {
+    get(admin.restoreBackupSystemTable(snapshotName));
+  }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java

@@ -1871,4 +1871,7 @@ CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNames, Str
    * Get the list of cached files
    */
   CompletableFuture<List<String>> getCachedFilesList(ServerName serverName);
+
+  @InterfaceAudience.Private
+  CompletableFuture<Void> restoreBackupSystemTable(String snapshotName);
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java

@@ -1010,4 +1010,9 @@ public CompletableFuture<Void> flushMasterStore() {
   public CompletableFuture<List<String>> getCachedFilesList(ServerName serverName) {
     return wrap(rawAdmin.getCachedFilesList(serverName));
   }
+
+  @Override
+  public CompletableFuture<Void> restoreBackupSystemTable(String snapshotName) {
+    return wrap(rawAdmin.restoreBackupSystemTable(snapshotName));
+  }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java

@@ -2795,6 +2795,19 @@ void onError(Throwable error) {
     }
   }
 
+  private static class RestoreBackupSystemTableProcedureBiConsumer extends ProcedureBiConsumer {
+
+    @Override
+    void onFinished() {
+      LOG.info("RestoreBackupSystemTableProcedure completed");
+    }
+
+    @Override
+    void onError(Throwable error) {
+      LOG.info("RestoreBackupSystemTableProcedure failed with {}", error.getMessage());
+    }
+  }
+
   private static class CreateTableProcedureBiConsumer extends TableProcedureBiConsumer {
 
     CreateTableProcedureBiConsumer(TableName tableName) {
@@ -4637,4 +4650,16 @@ List<String>> adminCall(controller, stub, request.build(),
           resp -> resp.getCachedFilesList()))
       .serverName(serverName).call();
   }
+
+  @Override
+  public CompletableFuture<Void> restoreBackupSystemTable(String snapshotName) {
+    MasterProtos.RestoreBackupSystemTableRequest request =
+      MasterProtos.RestoreBackupSystemTableRequest.newBuilder().setSnapshotName(snapshotName)
+        .build();
+    return this.<MasterProtos.RestoreBackupSystemTableRequest,
+      MasterProtos.RestoreBackupSystemTableResponse> procedureCall(request,
+        MasterService.Interface::restoreBackupSystemTable,
+        MasterProtos.RestoreBackupSystemTableResponse::getProcId,
+        new RestoreBackupSystemTableProcedureBiConsumer());
+  }
 }

hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto

@@ -1280,6 +1280,9 @@ service MasterService {
   rpc FlushTable(FlushTableRequest)
     returns(FlushTableResponse);
 
+  rpc RestoreBackupSystemTable(RestoreBackupSystemTableRequest)
+    returns(RestoreBackupSystemTableResponse);
+
   rpc rollAllWALWriters(RollAllWALWritersRequest)
     returns(RollAllWALWritersResponse);
 }
@@ -1369,6 +1372,13 @@ message FixMetaRequest {}
 
 message FixMetaResponse {}
 
+message RestoreBackupSystemTableRequest {
+  required string snapshot_name = 1;
+}
+message RestoreBackupSystemTableResponse {
+  optional uint64 proc_id = 1;
+}
+
 service HbckService {
   /** Update state of the table in meta only*/
   rpc SetTableStateInMeta(SetTableStateInMetaRequest)

hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto

@@ -840,6 +840,13 @@ message ReloadQuotasProcedureStateData {
   optional ForeignExceptionMessage error = 2;
 }
 
+enum RestoreBackupSystemTableState {
+  RESTORE_BACKUP_SYSTEM_TABLE_PREPARE = 1;
+  RESTORE_BACKUP_SYSTEM_TABLE_DISABLE = 2;
+  RESTORE_BACKUP_SYSTEM_TABLE_RESTORE = 3;
+  RESTORE_BACKUP_SYSTEM_TABLE_ENABLE = 4;
+}
+
 enum LogRollProcedureState {
   LOG_ROLL_ROLL_LOG_ON_RS = 1;
   LOG_ROLL_COLLECT_RS_HIGHEST_WAL_FILENUM = 2;

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java

@@ -76,6 +76,7 @@
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.NonceProcedureRunnable;
+import org.apache.hadoop.hbase.master.procedure.RestoreBackupSystemTableProcedure;
 import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
 import org.apache.hadoop.hbase.master.replication.AbstractPeerNoLockProcedure;
 import org.apache.hadoop.hbase.mob.MobUtils;
@@ -3667,6 +3668,24 @@ public FlushTableResponse flushTable(RpcController controller, FlushTableRequest
     }
   }
 
+  @Override
+  public MasterProtos.RestoreBackupSystemTableResponse restoreBackupSystemTable(
+    RpcController rpcController,
+    MasterProtos.RestoreBackupSystemTableRequest restoreBackupSystemTableRequest)
+    throws ServiceException {
+    try {
+      String snapshotName = restoreBackupSystemTableRequest.getSnapshotName();
+      SnapshotDescription snapshot = server.snapshotManager.getCompletedSnapshots().stream()
+        .filter(s -> s.getName().equals(snapshotName)).findFirst()
+        .orElseThrow(() -> new ServiceException("Snapshot %s not found".formatted(snapshotName)));
+      long pid = server.getMasterProcedureExecutor()
+        .submitProcedure(new RestoreBackupSystemTableProcedure(snapshot));
+      return MasterProtos.RestoreBackupSystemTableResponse.newBuilder().setProcId(pid).build();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
   @Override
   public RollAllWALWritersResponse rollAllWALWriters(RpcController rpcController,
     RollAllWALWritersRequest request) throws ServiceException {