diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index ab3df597bfaf..871c1b82149c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -3317,4 +3317,9 @@ List getLogEntries(Set serverNames, String logType, Server * @throws IOException if a remote or network exception occurs */ Future truncateRegionAsync(byte[] regionName) throws IOException; + + /** + * Get the list of cached files + */ + List getCachedFilesList(ServerName serverName) throws IOException; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 6b95e19a22a2..bde7e950e56b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1702,4 +1702,9 @@ CompletableFuture> getLogEntries(Set serverNames, Str * Flush master local region */ CompletableFuture flushMasterStore(); + + /** + * Get the list of cached files + */ + CompletableFuture> getCachedFilesList(ServerName serverName); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 4c9138e73061..cf473291319c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -911,4 +911,9 @@ public CompletableFuture> getLogEntries(Set serverNam public CompletableFuture flushMasterStore() { return wrap(rawAdmin.flushMasterStore()); } + + @Override + public CompletableFuture> getCachedFilesList(ServerName serverName) { + return wrap(rawAdmin.getCachedFilesList(serverName)); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index c54e561cc211..601f13b48895 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -2078,7 +2078,7 @@ public Future truncateRegionAsync(byte[] regionName) throws IOException { throw new IllegalArgumentException("Invalid region: " + Bytes.toStringBinary(regionName)); } - TableName tableName = hri.getTable(); + TableName tableName = (hri != null) ? hri.getTable() : null; MasterProtos.TruncateRegionResponse response = executeCallable(getTruncateRegionCallable(tableName, hri)); @@ -2086,6 +2086,15 @@ public Future truncateRegionAsync(byte[] regionName) throws IOException { return new TruncateRegionFuture(this, tableName, response); } + /** + * Get the list of cached files + */ + @Override + public List getCachedFilesList(ServerName serverName) throws IOException { + return ProtobufUtil.getCachedFilesList(rpcControllerFactory.newController(), + this.connection.getAdmin(serverName)); + } + private MasterCallable getTruncateRegionCallable(TableName tableName, RegionInfo hri) { return new MasterCallable(getConnection(), diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 7a560e51e0a0..cf103f401fcb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -129,6 +129,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactionSwitchResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -4264,4 +4266,15 @@ Void> call(controller, stub, request.build(), (s, c, req, done) -> s.flushMasterStore(c, req, done), resp -> null)) .call(); } + + @Override + public CompletableFuture> getCachedFilesList(ServerName serverName) { + GetCachedFilesListRequest.Builder request = GetCachedFilesListRequest.newBuilder(); + return this.> newAdminCaller() + .action((controller, stub) -> this.> adminCall(controller, stub, request.build(), + (s, c, req, done) -> s.getCachedFilesList(c, req, done), + resp -> resp.getCachedFilesList())) + .serverName(serverName).call(); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index a3b55e5ef16b..477e869296ad 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -149,6 +149,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearSlowLogResponses; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -1746,6 +1748,21 @@ public static List getOnlineRegions( return getRegionInfos(response); } + /** + * Get the list of cached files + */ + public static List getCachedFilesList(final RpcController controller, + final AdminService.BlockingInterface admin) throws IOException { + GetCachedFilesListRequest request = GetCachedFilesListRequest.newBuilder().build(); + GetCachedFilesListResponse response = null; + try { + response = admin.getCachedFilesList(controller, request); + } catch (ServiceException se) { + throw getRemoteException(se); + } + return new ArrayList<>(response.getCachedFilesList()); + } + /** * Get the list of region info from a GetOnlineRegionResponse * @param proto the GetOnlineRegionResponse diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/Admin.proto index 0c25be1a7f24..ae0bd78fbf98 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto @@ -283,6 +283,13 @@ message ExecuteProceduresRequest { message ExecuteProceduresResponse { } +message GetCachedFilesListRequest { +} + +message GetCachedFilesListResponse { + repeated string cached_files = 1; +} + /** * Slow/Large log (LogRequest) use-case specific RPC request. This request payload will be * converted in bytes and sent to generic RPC API: GetLogEntries @@ -406,4 +413,7 @@ service AdminService { rpc GetLogEntries(LogRequest) returns(LogEntry); + rpc GetCachedFilesList(GetCachedFilesListRequest) + returns(GetCachedFilesListResponse); + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 6c05402f5a55..fee9325fae22 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.io.hfile; import java.util.Iterator; +import java.util.Map; +import java.util.Optional; import org.apache.yetus.audience.InterfaceAudience; /** @@ -146,4 +148,11 @@ Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, default boolean isMetaBlock(BlockType blockType) { return blockType != null && blockType.getCategory() != BlockType.BlockCategory.DATA; } + + /** + * Returns the list of fully cached files + */ + default Optional> getFullyCachedFiles() { + return Optional.empty(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 4ad6dc3683f8..5ff2ee02a6e5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.io.hfile; import java.util.Iterator; +import java.util.Map; +import java.util.Optional; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.yetus.audience.InterfaceAudience; @@ -369,6 +371,14 @@ public BlockCache[] getBlockCaches() { return new BlockCache[] { this.l1Cache, this.l2Cache }; } + /** + * Returns the list of fully cached files + */ + @Override + public Optional> getFullyCachedFiles() { + return this.l2Cache.getFullyCachedFiles(); + } + @Override public void setMaxSize(long size) { this.l1Cache.setMaxSize(size); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java index 1ac9a4ffb842..f1579ea53b8e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java @@ -42,8 +42,9 @@ public HFilePreadReader(ReaderContext context, HFileInfo fileInfo, CacheConfig c final MutableBoolean fileAlreadyCached = new MutableBoolean(false); Optional bucketCacheOptional = BucketCache.getBucketCacheFromCacheConfig(cacheConf); - bucketCacheOptional.ifPresent(bc -> fileAlreadyCached - .setValue(bc.getFullyCachedFiles().get(path.getName()) == null ? false : true)); + bucketCacheOptional.flatMap(BucketCache::getFullyCachedFiles).ifPresent(fcf -> { + fileAlreadyCached.setValue(fcf.get(path.getName()) == null ? false : true); + }); // Prefetch file blocks upon open if requested if ( cacheConf.shouldPrefetchOnOpen() && cacheIfCompactionsOff() diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index f7f6f8a8ee0a..5b1757b85079 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -1950,8 +1950,9 @@ public AtomicBoolean getBackingMapValidated() { return backingMapValidated; } - public Map getFullyCachedFiles() { - return fullyCachedFiles; + @Override + public Optional> getFullyCachedFiles() { + return Optional.of(fullyCachedFiles); } public static Optional getBucketCacheFromCacheConfig(CacheConfig cacheConf) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index d4cfaaaf47a7..2924ebef000d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -152,6 +152,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos.RevokeResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; @@ -3065,6 +3067,12 @@ public HBaseProtos.LogEntry getLogEntries(RpcController controller, throw new ServiceException("Invalid request params"); } + @Override + public GetCachedFilesListResponse getCachedFilesList(RpcController controller, + GetCachedFilesListRequest request) throws ServiceException { + throw new ServiceException(new DoNotRetryIOException("Unsupported method on master")); + } + private MasterProtos.BalancerDecisionsResponse getBalancerDecisions(MasterProtos.BalancerDecisionsRequest request) { final NamedQueueRecorder namedQueueRecorder = this.regionServer.getNamedQueueRecorder(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 834e3a6301ae..09dc35963c22 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -188,6 +188,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -4033,6 +4035,17 @@ public HBaseProtos.LogEntry getLogEntries(RpcController controller, throw new ServiceException("Invalid request params"); } + @Override + public GetCachedFilesListResponse getCachedFilesList(RpcController controller, + GetCachedFilesListRequest request) throws ServiceException { + GetCachedFilesListResponse.Builder responseBuilder = GetCachedFilesListResponse.newBuilder(); + List fullyCachedFiles = new ArrayList<>(); + regionServer.getBlockCache().flatMap(BlockCache::getFullyCachedFiles).ifPresent(fcf -> { + fullyCachedFiles.addAll(fcf.keySet()); + }); + return responseBuilder.addAllCachedFiles(fullyCachedFiles).build(); + } + public RpcScheduler getRpcScheduler() { return rpcServer.getScheduler(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java index 3794c6c28d77..5164d582fa03 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -30,11 +31,14 @@ import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.StartMiniClusterOption; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -111,7 +115,17 @@ public void testPrefetchPersistence() throws Exception { // Default interval for cache persistence is 1000ms. So after 1000ms, both the persistence files // should exist. - assertTrue(new File(testDir + "/bucket.persistence").exists()); + + HRegionServer regionServingRS = cluster.getRegionServer(1).getRegions(tableName).size() == 1 + ? cluster.getRegionServer(1) + : cluster.getRegionServer(0); + + Admin admin = TEST_UTIL.getAdmin(); + List cachedFilesList = admin.getCachedFilesList(regionServingRS.getServerName()); + assertEquals(1, cachedFilesList.size()); + for (HStoreFile h : regionServingRS.getRegions().get(0).getStores().get(0).getStorefiles()) { + assertTrue(cachedFilesList.contains(h.getPath().getName())); + } // Stop the RS cluster.stopRegionServer(0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java index 768e41019d8a..f9f48ff8535f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java @@ -123,7 +123,7 @@ public void testPrefetchDoesntOverwork() throws Exception { BlockCacheKey key = snapshot.keySet().stream().findFirst().get(); LOG.debug("removing block {}", key); bc.getBackingMap().remove(key); - bc.getFullyCachedFiles().remove(storeFile.getName()); + bc.getFullyCachedFiles().ifPresent(fcf -> fcf.remove(storeFile.getName())); assertTrue(snapshot.size() > bc.getBackingMap().size()); LOG.debug("Third read should prefetch again, as we removed one block for the file."); readStoreFile(storeFile); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index 72a3ad044c3e..736560e18cb0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -95,6 +95,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetCachedFilesListResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; @@ -694,6 +696,12 @@ public HBaseProtos.LogEntry getLogEntries(RpcController controller, return null; } + @Override + public GetCachedFilesListResponse getCachedFilesList(RpcController controller, + GetCachedFilesListRequest request) throws ServiceException { + return null; + } + @Override public GetSpaceQuotaSnapshotsResponse getSpaceQuotaSnapshots(RpcController controller, GetSpaceQuotaSnapshotsRequest request) throws ServiceException { diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 183efd6288a4..76deb6fdd6ad 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -898,6 +898,14 @@ public Future truncateRegionAsync(byte[] regionName) { throw new NotImplementedException("Truncate Region Async not supported in ThriftAdmin"); } + /** + * Get the list of cached files + */ + @Override + public List getCachedFilesList(ServerName serverName) { + throw new NotImplementedException("getCachedFilesList not supported in ThriftAdmin"); + } + @Override public void splitRegion(byte[] regionName, byte[] splitPoint) { throw new NotImplementedException("splitRegion not supported in ThriftAdmin");