diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 417e0013523a..1f22c0fe0a86 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -2629,4 +2629,9 @@ List getLogEntries(Set serverNames, String logType, Server * Flush master local region */ void flushMasterStore() throws IOException; + + /** + * Get the list of cached files + */ + List getCachedFilesList(ServerName serverName) throws IOException; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java index bb620aa3cdaa..9c8f03a1057c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java @@ -1125,4 +1125,9 @@ public List getLogEntries(Set serverNames, String logType, public void flushMasterStore() throws IOException { get(admin.flushMasterStore()); } + + @Override + public List getCachedFilesList(ServerName serverName) throws IOException { + return get(admin.getCachedFilesList(serverName)); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 1097abbbf5e2..313d2f01c881 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1843,4 +1843,9 @@ CompletableFuture> getLogEntries(Set serverNames, Str * Flush master local region */ CompletableFuture flushMasterStore(); + + /** + * Get the list of cached files + */ + CompletableFuture> getCachedFilesList(ServerName serverName); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 0c7fd0f7b354..247acf485889 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -995,4 +995,9 @@ public CompletableFuture> getLogEntries(Set serverNam public CompletableFuture flushMasterStore() { return wrap(rawAdmin.flushMasterStore()); } + + @Override + public CompletableFuture> getCachedFilesList(ServerName serverName) { + return wrap(rawAdmin.getCachedFilesList(serverName)); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 953dd2024767..92a95d491f0d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -132,6 +132,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactionSwitchResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -4519,4 +4521,15 @@ Void> call(controller, stub, request.build(), (s, c, req, done) -> s.flushMasterStore(c, req, done), resp -> null)) .call(); } + + @Override + public CompletableFuture> getCachedFilesList(ServerName serverName) { + GetCachedFilesListRequest.Builder request = GetCachedFilesListRequest.newBuilder(); + return this.> newAdminCaller() + .action((controller, stub) -> this.> adminCall(controller, stub, request.build(), + (s, c, req, done) -> s.getCachedFilesList(c, req, done), + resp -> resp.getCachedFilesList())) + .serverName(serverName).call(); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index c14a0d042823..d2e14df1c8d6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -153,6 +153,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearSlowLogResponses; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -1780,6 +1782,21 @@ public static List getOnlineRegions( return getRegionInfos(response); } + /** + * Get the list of cached files + */ + public static List getCachedFilesList(final RpcController controller, + final AdminService.BlockingInterface admin) throws IOException { + GetCachedFilesListRequest request = GetCachedFilesListRequest.newBuilder().build(); + GetCachedFilesListResponse response = null; + try { + response = admin.getCachedFilesList(controller, request); + } catch (ServiceException se) { + throw getRemoteException(se); + } + return new ArrayList<>(response.getCachedFilesList()); + } + /** * Get the list of region info from a GetOnlineRegionResponse * @param proto the GetOnlineRegionResponse diff --git a/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto index cd88a0ca7cdb..308b1a8b6d62 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto @@ -283,6 +283,13 @@ message ExecuteProceduresRequest { message ExecuteProceduresResponse { } +message GetCachedFilesListRequest { +} + +message GetCachedFilesListResponse { + repeated string cached_files = 1; +} + /** * Slow/Large log (LogRequest) use-case specific RPC request. This request payload will be * converted in bytes and sent to generic RPC API: GetLogEntries @@ -405,4 +412,7 @@ service AdminService { rpc GetLogEntries(LogRequest) returns(LogEntry); + rpc GetCachedFilesList(GetCachedFilesListRequest) + returns(GetCachedFilesListResponse); + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 4e795ec75e75..91ebaaabd422 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hbase.io.hfile; import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; /** @@ -161,4 +164,11 @@ default Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repe default boolean isMetaBlock(BlockType blockType) { return blockType != null && blockType.getCategory() != BlockType.BlockCategory.DATA; } + + /** + * Returns the list of fully cached files + */ + default Optional>> getFullyCachedFiles() { + return Optional.empty(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index d616d6f40d9f..04f6336f7e2b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -18,8 +18,11 @@ package org.apache.hadoop.hbase.io.hfile; import java.util.Iterator; +import java.util.Map; +import java.util.Optional; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; +import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; /** @@ -382,6 +385,14 @@ public BlockCache[] getBlockCaches() { return new BlockCache[] { this.l1Cache, this.l2Cache }; } + /** + * Returns the list of fully cached files + */ + @Override + public Optional>> getFullyCachedFiles() { + return this.l2Cache.getFullyCachedFiles(); + } + @Override public void setMaxSize(long size) { this.l1Cache.setMaxSize(size); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java index bc44dd20f9a7..9ab195d5fa26 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java @@ -42,8 +42,9 @@ public HFilePreadReader(ReaderContext context, HFileInfo fileInfo, CacheConfig c final MutableBoolean fileAlreadyCached = new MutableBoolean(false); Optional bucketCacheOptional = BucketCache.getBucketCacheFromCacheConfig(cacheConf); - bucketCacheOptional.ifPresent(bc -> fileAlreadyCached - .setValue(bc.getFullyCachedFiles().get(path.getName()) == null ? false : true)); + bucketCacheOptional.flatMap(BucketCache::getFullyCachedFiles).ifPresent(fcf -> { + fileAlreadyCached.setValue(fcf.get(path.getName()) == null ? false : true); + }); // Prefetch file blocks upon open if requested if ( cacheConf.shouldPrefetchOnOpen() && cacheIfCompactionsOff() diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 60a06d6f3303..7008f65cf963 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -2002,8 +2002,9 @@ public AtomicBoolean getBackingMapValidated() { return backingMapValidated; } - public Map> getFullyCachedFiles() { - return fullyCachedFiles; + @Override + public Optional>> getFullyCachedFiles() { + return Optional.of(fullyCachedFiles); } public static Optional getBucketCacheFromCacheConfig(CacheConfig cacheConf) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 736fbae0dea9..9ff69d436469 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -164,6 +164,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -3575,6 +3577,12 @@ public ExecuteProceduresResponse executeProcedures(RpcController controller, throw new ServiceException(new DoNotRetryIOException("Unsupported method on master")); } + @Override + public GetCachedFilesListResponse getCachedFilesList(RpcController controller, + GetCachedFilesListRequest request) throws ServiceException { + throw new ServiceException(new DoNotRetryIOException("Unsupported method on master")); + } + @Override public GetLiveRegionServersResponse getLiveRegionServers(RpcController controller, GetLiveRegionServersRequest request) throws ServiceException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 57efe505c126..4f04457e91b6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -169,6 +169,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -3933,4 +3935,15 @@ public void onConfigurationChange(Configuration conf) { super.onConfigurationChange(conf); setReloadableGuardrails(conf); } + + @Override + public GetCachedFilesListResponse getCachedFilesList(RpcController controller, + GetCachedFilesListRequest request) throws ServiceException { + GetCachedFilesListResponse.Builder responseBuilder = GetCachedFilesListResponse.newBuilder(); + List fullyCachedFiles = new ArrayList<>(); + server.getBlockCache().flatMap(BlockCache::getFullyCachedFiles).ifPresent(fcf -> { + fullyCachedFiles.addAll(fcf.keySet()); + }); + return responseBuilder.addAllCachedFiles(fullyCachedFiles).build(); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java index 64db9158333d..879d8566c82e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -30,11 +31,14 @@ import org.apache.hadoop.hbase.SingleProcessHBaseCluster; import org.apache.hadoop.hbase.StartTestingClusterOption; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -112,7 +116,17 @@ public void testPrefetchPersistence() throws Exception { // Default interval for cache persistence is 1000ms. So after 1000ms, both the persistence files // should exist. - assertTrue(new File(testDir + "/bucket.persistence").exists()); + + HRegionServer regionServingRS = cluster.getRegionServer(1).getRegions(tableName).size() == 1 + ? cluster.getRegionServer(1) + : cluster.getRegionServer(0); + + Admin admin = TEST_UTIL.getAdmin(); + List cachedFilesList = admin.getCachedFilesList(regionServingRS.getServerName()); + assertEquals(1, cachedFilesList.size()); + for (HStoreFile h : regionServingRS.getRegions().get(0).getStores().get(0).getStorefiles()) { + assertTrue(cachedFilesList.contains(h.getPath().getName())); + } // Stop the RS cluster.stopRegionServer(0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java index e5c6b42fcc40..93f09231f740 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java @@ -123,7 +123,7 @@ public void testPrefetchDoesntOverwork() throws Exception { BlockCacheKey key = snapshot.keySet().stream().findFirst().get(); LOG.debug("removing block {}", key); bc.getBackingMap().remove(key); - bc.getFullyCachedFiles().remove(storeFile.getName()); + bc.getFullyCachedFiles().ifPresent(fcf -> fcf.remove(storeFile.getName())); assertTrue(snapshot.size() > bc.getBackingMap().size()); LOG.debug("Third read should prefetch again, as we removed one block for the file."); readStoreFile(storeFile); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index 9974c824f889..a7164a6fab64 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -97,6 +97,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -452,6 +454,12 @@ public GetOnlineRegionResponse getOnlineRegion(RpcController controller, return null; } + @Override + public GetCachedFilesListResponse getCachedFilesList(RpcController controller, + GetCachedFilesListRequest request) throws ServiceException { + return null; + } + @Override public List getRegions() { return null; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java index 02087fb0a661..f98a1c78ad5f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java @@ -969,6 +969,11 @@ public void flushMasterStore() throws IOException { admin.flushMasterStore(); } + @Override + public List getCachedFilesList(ServerName serverName) throws IOException { + return admin.getCachedFilesList(serverName); + } + @Override public boolean replicationPeerModificationSwitch(boolean on, boolean drainProcedures) throws IOException { diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 70ce37faf47e..c88572b9f3c7 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -1345,6 +1345,11 @@ public void flushMasterStore() throws IOException { throw new NotImplementedException("flushMasterStore not supported in ThriftAdmin"); } + @Override + public List getCachedFilesList(ServerName serverName) throws IOException { + throw new NotImplementedException("getCachedFilesList not supported in ThriftAdmin"); + } + @Override public boolean replicationPeerModificationSwitch(boolean on, boolean drainProcedures) throws IOException {