HBASE-15560 W-TinyLFU based BlockCache

Signed-off-by: Andrew Purtell <apurtell@apache.org>

hbase-common/src/main/resources/hbase-default.xml

@@ -900,6 +900,11 @@ possible configurations would overwhelm and obscure the important.
     <description>
       The default thread pool size if parallel-seeking feature enabled.</description>
   </property>
+  <property>
+    <name>hfile.block.cache.policy</name>
+    <value>LRU</value>
+    <description>The eviction policy for the L1 block cache (LRU or TinyLFU).</description>
+  </property>
   <property>
     <name>hfile.block.cache.size</name>
     <value>0.4</value>

hbase-resource-bundle/src/main/resources/supplemental-models.xml

@@ -1858,4 +1858,30 @@ Copyright (c) 2007-2017 The JRuby project
       </licenses>
     </project>
   </supplement>
+  <supplement>
+    <project>
+      <groupId>com.github.ben-manes.caffeine</groupId>
+      <artifactId>caffeine</artifactId>
+      <licenses>
+        <license>
+          <name>Apache License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.google.errorprone</groupId>
+      <artifactId>error_prone_annotations</artifactId>
+      <licenses>
+        <license>
+          <name>Apache License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
 </supplementalDataModels>

hbase-server/pom.xml

@@ -455,6 +455,10 @@
        <artifactId>findbugs-annotations</artifactId>
        <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>com.github.ben-manes.caffeine</groupId>
+      <artifactId>caffeine</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.dropwizard.metrics</groupId>
       <artifactId>metrics-core</artifactId>

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java

@@ -21,6 +21,7 @@
 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
 
 import java.io.IOException;
+import java.util.concurrent.ForkJoinPool;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
@@ -41,6 +42,12 @@ public final class BlockCacheFactory {
    * Configuration keys for Bucket cache
    */
 
+  /**
+   * Configuration key to cache block policy (Lru, TinyLfu).
+   */
+  public static final String BLOCKCACHE_POLICY_KEY = "hfile.block.cache.policy";
+  public static final String BLOCKCACHE_POLICY_DEFAULT = "LRU";
+
   /**
    * If the chosen ioengine can persist its state across restarts, the path to the file to persist
    * to. This file is NOT the data file. It is a file into which we will serialize the map of
@@ -85,16 +92,16 @@ private BlockCacheFactory() {
   }
 
   public static BlockCache createBlockCache(Configuration conf) {
-    LruBlockCache onHeapCache = createOnHeapCache(conf);
-    if (onHeapCache == null) {
+    FirstLevelBlockCache l1Cache = createFirstLevelCache(conf);
+    if (l1Cache == null) {
       return null;
     }
     boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
     if (useExternal) {
       BlockCache l2CacheInstance = createExternalBlockcache(conf);
       return l2CacheInstance == null ?
-          onHeapCache :
-          new InclusiveCombinedBlockCache(onHeapCache, l2CacheInstance);
+          l1Cache :
+          new InclusiveCombinedBlockCache(l1Cache, l2CacheInstance);
     } else {
       // otherwise use the bucket cache.
       BucketCache bucketCache = createBucketCache(conf);
@@ -103,20 +110,26 @@ public static BlockCache createBlockCache(Configuration conf) {
         LOG.warn(
             "From HBase 2.0 onwards only combined mode of LRU cache and bucket cache is available");
       }
-      return bucketCache == null ? onHeapCache : new CombinedBlockCache(onHeapCache, bucketCache);
+      return bucketCache == null ? l1Cache : new CombinedBlockCache(l1Cache, bucketCache);
     }
   }
 
-  private static LruBlockCache createOnHeapCache(final Configuration c) {
+  private static FirstLevelBlockCache createFirstLevelCache(final Configuration c) {
     final long cacheSize = MemorySizeUtil.getOnHeapCacheSize(c);
     if (cacheSize < 0) {
       return null;
     }
+    String policy = c.get(BLOCKCACHE_POLICY_KEY, BLOCKCACHE_POLICY_DEFAULT);
     int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
-    LOG.info(
-        "Allocating onheap LruBlockCache size=" + StringUtils.byteDesc(cacheSize) + ", blockSize="
-            + StringUtils.byteDesc(blockSize));
-    return new LruBlockCache(cacheSize, blockSize, true, c);
+    LOG.info("Allocating BlockCache size=" +
+        StringUtils.byteDesc(cacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
+    if (policy.equalsIgnoreCase("LRU")) {
+      return new LruBlockCache(cacheSize, blockSize, true, c);
+    } else if (policy.equalsIgnoreCase("TinyLFU")) {
+      return new TinyLfuBlockCache(cacheSize, blockSize, ForkJoinPool.commonPool(), c);
+    } else {
+      throw new IllegalArgumentException("Unknown policy: " + policy);
+    }
   }
 
   /**

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java

@@ -30,22 +30,23 @@
 
 /**
  * CombinedBlockCache is an abstraction layer that combines
- * {@link LruBlockCache} and {@link BucketCache}. The smaller lruCache is used
+ * {@link FirstLevelBlockCache} and {@link BucketCache}. The smaller lruCache is used
  * to cache bloom blocks and index blocks.  The larger Cache is used to
  * cache data blocks. {@link #getBlock(BlockCacheKey, boolean, boolean, boolean)} reads
- * first from the smaller lruCache before looking for the block in the l2Cache.
+ * first from the smaller l1Cache before looking for the block in the l2Cache.  Blocks evicted
+ * from l1Cache are put into the bucket cache.
  * Metrics are the combined size and hits and misses of both caches.
  */
 @InterfaceAudience.Private
 public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
-  protected final LruBlockCache onHeapCache;
+  protected final FirstLevelBlockCache l1Cache;
   protected final BlockCache l2Cache;
   protected final CombinedCacheStats combinedCacheStats;
 
-  public CombinedBlockCache(LruBlockCache onHeapCache, BlockCache l2Cache) {
-    this.onHeapCache = onHeapCache;
+  public CombinedBlockCache(FirstLevelBlockCache l1Cache, BlockCache l2Cache) {
+    this.l1Cache = l1Cache;
     this.l2Cache = l2Cache;
-    this.combinedCacheStats = new CombinedCacheStats(onHeapCache.getStats(),
+    this.combinedCacheStats = new CombinedCacheStats(l1Cache.getStats(),
         l2Cache.getStats());
   }
 
@@ -55,14 +56,14 @@ public long heapSize() {
     if (l2Cache instanceof HeapSize) {
       l2size = ((HeapSize) l2Cache).heapSize();
     }
-    return onHeapCache.heapSize() + l2size;
+    return l1Cache.heapSize() + l2size;
   }
 
   @Override
   public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
     boolean metaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA;
     if (metaBlock) {
-      onHeapCache.cacheBlock(cacheKey, buf, inMemory);
+      l1Cache.cacheBlock(cacheKey, buf, inMemory);
     } else {
       l2Cache.cacheBlock(cacheKey, buf, inMemory);
     }
@@ -80,19 +81,19 @@ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching,
     // we end up calling l2Cache.getBlock.
     // We are not in a position to exactly look at LRU cache or BC as BlockType may not be getting
     // passed always.
-    return onHeapCache.containsBlock(cacheKey)?
-        onHeapCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics):
+    return l1Cache.containsBlock(cacheKey)?
+        l1Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics):
         l2Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
   }
 
   @Override
   public boolean evictBlock(BlockCacheKey cacheKey) {
-    return onHeapCache.evictBlock(cacheKey) || l2Cache.evictBlock(cacheKey);
+    return l1Cache.evictBlock(cacheKey) || l2Cache.evictBlock(cacheKey);
   }
 
   @Override
   public int evictBlocksByHfileName(String hfileName) {
-    return onHeapCache.evictBlocksByHfileName(hfileName)
+    return l1Cache.evictBlocksByHfileName(hfileName)
         + l2Cache.evictBlocksByHfileName(hfileName);
   }
 
@@ -103,43 +104,43 @@ public CacheStats getStats() {
 
   @Override
   public void shutdown() {
-    onHeapCache.shutdown();
+    l1Cache.shutdown();
     l2Cache.shutdown();
   }
 
   @Override
   public long size() {
-    return onHeapCache.size() + l2Cache.size();
+    return l1Cache.size() + l2Cache.size();
   }
 
   @Override
   public long getMaxSize() {
-    return onHeapCache.getMaxSize() + l2Cache.getMaxSize();
+    return l1Cache.getMaxSize() + l2Cache.getMaxSize();
   }
 
   @Override
   public long getCurrentDataSize() {
-    return onHeapCache.getCurrentDataSize() + l2Cache.getCurrentDataSize();
+    return l1Cache.getCurrentDataSize() + l2Cache.getCurrentDataSize();
   }
 
   @Override
   public long getFreeSize() {
-    return onHeapCache.getFreeSize() + l2Cache.getFreeSize();
+    return l1Cache.getFreeSize() + l2Cache.getFreeSize();
   }
 
   @Override
   public long getCurrentSize() {
-    return onHeapCache.getCurrentSize() + l2Cache.getCurrentSize();
+    return l1Cache.getCurrentSize() + l2Cache.getCurrentSize();
   }
 
   @Override
   public long getBlockCount() {
-    return onHeapCache.getBlockCount() + l2Cache.getBlockCount();
+    return l1Cache.getBlockCount() + l2Cache.getBlockCount();
   }
 
   @Override
   public long getDataBlockCount() {
-    return onHeapCache.getDataBlockCount() + l2Cache.getDataBlockCount();
+    return l1Cache.getDataBlockCount() + l2Cache.getDataBlockCount();
   }
 
   public static class CombinedCacheStats extends CacheStats {
@@ -332,7 +333,7 @@ public void rollMetricsPeriod() {
       lruCacheStats.rollMetricsPeriod();
       bucketCacheStats.rollMetricsPeriod();
     }
-    
+
     @Override
     public long getFailedInserts() {
       return lruCacheStats.getFailedInserts() + bucketCacheStats.getFailedInserts();
@@ -343,13 +344,13 @@ public long getSumHitCountsPastNPeriods() {
       return lruCacheStats.getSumHitCountsPastNPeriods()
           + bucketCacheStats.getSumHitCountsPastNPeriods();
     }
-    
+
     @Override
     public long getSumRequestCountsPastNPeriods() {
       return lruCacheStats.getSumRequestCountsPastNPeriods()
           + bucketCacheStats.getSumRequestCountsPastNPeriods();
     }
-    
+
     @Override
     public long getSumHitCachingCountsPastNPeriods() {
       return lruCacheStats.getSumHitCachingCountsPastNPeriods()
@@ -370,12 +371,12 @@ public Iterator<CachedBlock> iterator() {
 
   @Override
   public BlockCache[] getBlockCaches() {
-    return new BlockCache [] {this.onHeapCache, this.l2Cache};
+    return new BlockCache [] {this.l1Cache, this.l2Cache};
   }
 
   @Override
   public void setMaxSize(long size) {
-    this.onHeapCache.setMaxSize(size);
+    this.l1Cache.setMaxSize(size);
   }
 
   @Override
@@ -390,7 +391,7 @@ public int getRefCount(BlockCacheKey cacheKey) {
         ? ((BucketCache) this.l2Cache).getRefCount(cacheKey) : 0;
   }
 
-  public LruBlockCache getOnHeapCache() {
-    return onHeapCache;
+  public FirstLevelBlockCache getFirstLevelCache() {
+    return l1Cache;
   }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * In-memory BlockCache that may be backed by secondary layer(s).
+ */
+@InterfaceAudience.Private
+public interface FirstLevelBlockCache extends ResizableBlockCache, HeapSize {
+
+  /**
+   * Whether the cache contains the block with specified cacheKey
+   *
+   * @param cacheKey cache key for the block
+   * @return true if it contains the block
+   */
+  boolean containsBlock(BlockCacheKey cacheKey);
+
+  /**
+   * Specifies the secondary cache. An entry that is evicted from this cache due to a size
+   * constraint will be inserted into the victim cache.
+   *
+   * @param victimCache the second level cache
+   * @throws IllegalArgumentException if the victim cache had already been set
+   */
+  void setVictimCache(BlockCache victimCache);
+}

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java

@@ -23,7 +23,7 @@
 
 @InterfaceAudience.Private
 public class InclusiveCombinedBlockCache extends CombinedBlockCache {
-  public InclusiveCombinedBlockCache(LruBlockCache l1, BlockCache l2) {
+  public InclusiveCombinedBlockCache(FirstLevelBlockCache l1, BlockCache l2) {
     super(l1,l2);
     l1.setVictimCache(l2);
   }
@@ -34,7 +34,7 @@ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching,
     // On all external cache set ups the lru should have the l2 cache set as the victimHandler
     // Because of that all requests that miss inside of the lru block cache will be
     // tried in the l2 block cache.
-    return onHeapCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
+    return l1Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
   }
 
   /**
@@ -48,15 +48,15 @@ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching,
   public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
     // This is the inclusive part of the combined block cache.
     // Every block is placed into both block caches.
-    onHeapCache.cacheBlock(cacheKey, buf, inMemory);
+    l1Cache.cacheBlock(cacheKey, buf, inMemory);
 
     // This assumes that insertion into the L2 block cache is either async or very fast.
     l2Cache.cacheBlock(cacheKey, buf, inMemory);
   }
 
   @Override
   public boolean evictBlock(BlockCacheKey cacheKey) {
-    boolean l1Result = this.onHeapCache.evictBlock(cacheKey);
+    boolean l1Result = this.l1Cache.evictBlock(cacheKey);
     boolean l2Result = this.l2Cache.evictBlock(cacheKey);
     return l1Result || l2Result;
   }