HBASE-27794: Tooling for parsing/reading the prefetch files list file

apache · Oct 18, 2023 · e1dde58 · e1dde58
1 parent 391dfda
commit e1dde58
Show file tree

Hide file tree

Showing 13 changed files with 122 additions and 0 deletions.
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -2615,4 +2615,9 @@ List<LogEntry> getLogEntries(Set<ServerName> serverNames, String logType, Server
    * Flush master local region
    */
   void flushMasterStore() throws IOException;
+
+  /**
+   * Get the list of prefetched files
+   */
+  List<String> getPrefetchedFilesList(ServerName serverName) throws IOException;
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
@@ -1115,4 +1115,9 @@ public List<LogEntry> getLogEntries(Set<ServerName> serverNames, String logType,
   public void flushMasterStore() throws IOException {
     get(admin.flushMasterStore());
   }
+
+  @Override
+  public List<String> getPrefetchedFilesList(ServerName serverName) throws IOException {
+    return get(admin.getPrefetchedFilesList(serverName));
+  }
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -1837,4 +1837,9 @@ CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNames, Str
    * Flush master local region
    */
   CompletableFuture<Void> flushMasterStore();
+
+  /**
+   * Get the list of prefetched files
+   */
+  CompletableFuture<List<String>> getPrefetchedFilesList(ServerName serverName);
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -990,4 +990,9 @@ public CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNam
   public CompletableFuture<Void> flushMasterStore() {
     return wrap(rawAdmin.flushMasterStore());
   }
+
+  @Override
+  public CompletableFuture<List<String>> getPrefetchedFilesList(ServerName serverName) {
+    return wrap(rawAdmin.getPrefetchedFilesList(serverName));
+  }
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -134,6 +134,8 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
@@ -4453,4 +4455,15 @@ Void> call(controller, stub, request.build(),
           (s, c, req, done) -> s.flushMasterStore(c, req, done), resp -> null))
       .call();
   }
+
+  @Override
+  public CompletableFuture<List<String>> getPrefetchedFilesList(ServerName serverName) {
+    GetPrefetchedFilesListRequest.Builder request = GetPrefetchedFilesListRequest.newBuilder();
+    return this.<List<String>> newAdminCaller()
+      .action((controller, stub) -> this.<GetPrefetchedFilesListRequest,
+        GetPrefetchedFilesListResponse, List<String>> adminCall(controller, stub, request.build(),
+          (s, c, req, done) -> s.getPrefetchedFilesList(c, req, done),
+          resp -> resp.getPrefetchedFilesList()))
+      .serverName(serverName).call();
+  }
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -155,6 +155,8 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
@@ -1780,6 +1782,21 @@ public static List<org.apache.hadoop.hbase.client.RegionInfo> getOnlineRegions(
     return getRegionInfos(response);
   }
 
+  /**
+   * Get the list of prefetched files
+   */
+  public static List<String> getFullyPrefetchedFiles(final RpcController controller,
+    final AdminService.BlockingInterface admin) throws IOException {
+    GetPrefetchedFilesListRequest request = GetPrefetchedFilesListRequest.newBuilder().build();
+    GetPrefetchedFilesListResponse response = null;
+    try {
+      response = admin.getPrefetchedFilesList(controller, request);
+    } catch (ServiceException se) {
+      throw getRemoteException(se);
+    }
+    return new ArrayList<>(response.getPrefetchedFilesList());
+  }
+
   /**
    * Get the list of region info from a GetOnlineRegionResponse
    * @param proto the GetOnlineRegionResponse

diff --git a/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto
@@ -283,6 +283,13 @@ message ExecuteProceduresRequest {
 message ExecuteProceduresResponse {
 }
 
+message GetPrefetchedFilesListRequest {
+}
+
+message GetPrefetchedFilesListResponse {
+  repeated string prefetched_files = 1;
+}
+
 /**
  * Slow/Large log (LogRequest) use-case specific RPC request. This request payload will be
  * converted in bytes and sent to generic RPC API: GetLogEntries
@@ -405,4 +412,7 @@ service AdminService {
   rpc GetLogEntries(LogRequest)
     returns(LogEntry);
 
+  rpc GetPrefetchedFilesList(GetPrefetchedFilesListRequest)
+    returns(GetPrefetchedFilesListResponse);
+
 }
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -166,6 +166,8 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
@@ -3563,6 +3565,11 @@ public ExecuteProceduresResponse executeProcedures(RpcController controller,
     throw new ServiceException(new DoNotRetryIOException("Unsupported method on master"));
   }
 
+  public GetPrefetchedFilesListResponse getPrefetchedFilesList(RpcController controller,
+    GetPrefetchedFilesListRequest request) throws ServiceException {
+    throw new ServiceException(new DoNotRetryIOException("Unsupported method on master"));
+  }
+
   @Override
   public GetLiveRegionServersResponse getLiveRegionServers(RpcController controller,
     GetLiveRegionServersRequest request) throws ServiceException {

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -87,6 +87,8 @@
 import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;
 import org.apache.hadoop.hbase.io.ByteBuffAllocator;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
+import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
 import org.apache.hadoop.hbase.ipc.HBaseRpcController;
 import org.apache.hadoop.hbase.ipc.PriorityFunction;
 import org.apache.hadoop.hbase.ipc.QosPriority;
@@ -171,6 +173,8 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
@@ -3933,4 +3937,25 @@ public void onConfigurationChange(Configuration conf) {
     super.onConfigurationChange(conf);
     setReloadableGuardrails(conf);
   }
+
+  @Override
+  public GetPrefetchedFilesListResponse getPrefetchedFilesList(RpcController controller,
+    GetPrefetchedFilesListRequest request) throws ServiceException {
+    GetPrefetchedFilesListResponse.Builder responseBuilder =
+      GetPrefetchedFilesListResponse.newBuilder();
+
+    List<String> fullyCachedFiles = new ArrayList<>();
+    server.getBlockCache().ifPresent(blockCache -> {
+      if (blockCache instanceof CombinedBlockCache) {
+        BlockCache l2 = ((CombinedBlockCache) blockCache).getSecondLevelCache();
+        if (l2 instanceof BucketCache) {
+          if (((BucketCache) l2).isCachePersistenceEnabled()) {
+            LOG.info("Fetching fully cached files");
+            fullyCachedFiles.addAll(((BucketCache) l2).getFullyCachedFiles().keySet());
+          }
+        }
+      }
+    });
+    return responseBuilder.addAllPrefetchedFiles(fullyCachedFiles).build();
+  }
 }
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java
@@ -23,18 +23,21 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
+import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
 import org.apache.hadoop.hbase.StartTestingClusterOption;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.IOTests;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -114,6 +117,15 @@ public void testPrefetchPersistence() throws Exception {
     // should exist.
     assertTrue(new File(testDir + "/bucket.persistence").exists());
 
+    HRegionServer regionServingRS = cluster.getRegionServer(1).getRegions(tableName).size() == 1
+      ? cluster.getRegionServer(1)
+      : cluster.getRegionServer(0);
+
+    Admin admin = TEST_UTIL.getAdmin();
+    List<String> prefetchedFilesList =
+      admin.getPrefetchedFilesList(regionServingRS.getServerName());
+    assertEquals(1, ((List<?>) prefetchedFilesList).size());
+
     // Stop the RS
     cluster.stopRegionServer(0);
     LOG.info("Stopped Region Server 0.");

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -99,6 +99,8 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetPrefetchedFilesListResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
@@ -452,6 +454,12 @@ public GetOnlineRegionResponse getOnlineRegion(RpcController controller,
     return null;
   }
 
+  @Override
+  public GetPrefetchedFilesListResponse getPrefetchedFilesList(RpcController controller,
+    GetPrefetchedFilesListRequest request) throws ServiceException {
+    return null;
+  }
+
   @Override
   public List<HRegion> getRegions() {
     return null;

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
@@ -959,6 +959,11 @@ public void flushMasterStore() throws IOException {
     admin.flushMasterStore();
   }
 
+  @Override
+  public List<String> getPrefetchedFilesList(ServerName serverName) throws IOException {
+    return admin.getPrefetchedFilesList(serverName);
+  }
+
   @Override
   public boolean replicationPeerModificationSwitch(boolean on, boolean drainProcedures)
     throws IOException {

diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
@@ -1335,6 +1335,11 @@ public void flushMasterStore() throws IOException {
     throw new NotImplementedException("flushMasterStore not supported in ThriftAdmin");
   }
 
+  @Override
+  public List<String> getPrefetchedFilesList(ServerName serverName) throws IOException {
+    throw new NotImplementedException("getPrefetchedFilesList not supported in ThriftAdmin");
+  }
+
   @Override
   public boolean replicationPeerModificationSwitch(boolean on, boolean drainProcedures)
     throws IOException {