Skip to content

Commit

Permalink
Potential Fix facebook#10434
Browse files Browse the repository at this point in the history
  • Loading branch information
@gangliao
gangliao committed Aug 1, 2022
1 parent 87649d3 commit e69183b
Show file tree
Hide file tree
Showing 10 changed files with 63 additions and 63 deletions.
2 changes: 1 addition & 1 deletion cache/clock_cache.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -700,7 +700,7 @@ std::shared_ptr<Cache> NewClockCache(
return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
/* high_pri_pool_ratio */ 0.5, nullptr,
kDefaultToAdaptiveMutex, metadata_charge_policy,
/* low_pri_pool_ratio */ 0.5);
/* low_pri_pool_ratio */ 0.0);
}

std::shared_ptr<Cache> ExperimentalNewClockCache(
Expand Down
42 changes: 21 additions & 21 deletions cache/compressed_secondary_cache_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
lru_cache_opts.secondary_cache = secondary_cache;
std::shared_ptr<Cache> cache = NewLRUCache(lru_cache_opts);
std::shared_ptr<Statistics> stats = CreateDBStatistics();
Expand Down Expand Up @@ -324,11 +324,11 @@ class CompressedSecondaryCacheTest : public testing::Test {
std::shared_ptr<SecondaryCache> secondary_cache =
NewCompressedSecondaryCache(secondary_cache_opts);

LRUCacheOptions opts(
1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */,
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
opts.secondary_cache = secondary_cache;
std::shared_ptr<Cache> cache = NewLRUCache(opts);

Expand Down Expand Up @@ -373,11 +373,11 @@ class CompressedSecondaryCacheTest : public testing::Test {
std::shared_ptr<SecondaryCache> secondary_cache =
NewCompressedSecondaryCache(secondary_cache_opts);

LRUCacheOptions opts(
1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */,
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
opts.secondary_cache = secondary_cache;
std::shared_ptr<Cache> cache = NewLRUCache(opts);

Expand Down Expand Up @@ -432,11 +432,11 @@ class CompressedSecondaryCacheTest : public testing::Test {
std::shared_ptr<SecondaryCache> secondary_cache =
NewCompressedSecondaryCache(secondary_cache_opts);

LRUCacheOptions opts(
1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */,
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
opts.secondary_cache = secondary_cache;
std::shared_ptr<Cache> cache = NewLRUCache(opts);

Expand Down Expand Up @@ -491,11 +491,11 @@ class CompressedSecondaryCacheTest : public testing::Test {
std::shared_ptr<SecondaryCache> secondary_cache =
NewCompressedSecondaryCache(secondary_cache_opts);

LRUCacheOptions opts(
1024 /* capacity */, 0 /* num_shard_bits */,
true /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */,
true /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */,
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
opts.secondary_cache = secondary_cache;
std::shared_ptr<Cache> cache = NewLRUCache(opts);

Expand Down
6 changes: 4 additions & 2 deletions cache/lru_cache.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,8 @@ void LRUCacheShard::LRU_Insert(LRUHandle* e) {
e->SetInLowPriPool(false);
high_pri_pool_usage_ += e->total_charge;
MaintainPoolSize();
} else if (low_pri_pool_ratio_ > 0 && (e->IsLowPri() || e->HasHit())) {
} else if (low_pri_pool_ratio_ > 0 &&
(e->IsHighPri() || e->IsLowPri() || e->HasHit())) {
// Insert "e" to the head of low-pri pool.
e->next = lru_low_pri_->next;
e->prev = lru_low_pri_;
Expand Down Expand Up @@ -842,7 +843,8 @@ std::shared_ptr<Cache> NewLRUCache(
return nullptr;
}
if (low_pri_pool_ratio < 0.0 || low_pri_pool_ratio > 1.0) {
low_pri_pool_ratio = 1.0 - high_pri_pool_ratio;
// Invalid high_pri_pool_ratio
return nullptr;
}
if (low_pri_pool_ratio + high_pri_pool_ratio > 1.0) {
// Invalid high_pri_pool_ratio and low_pri_pool_ratio combination
Expand Down
53 changes: 26 additions & 27 deletions cache/lru_cache_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -705,8 +705,7 @@ class TestSecondaryCache : public SecondaryCache {
: num_inserts_(0), num_lookups_(0), inject_failure_(false) {
cache_ =
NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */, nullptr,
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata,
0.5 /* low_pri_pool_ratio */);
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
}
~TestSecondaryCache() override { cache_.reset(); }

Expand Down Expand Up @@ -949,7 +948,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache =
std::make_shared<TestSecondaryCache>(2048);
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -996,7 +995,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicFailTest) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache =
std::make_shared<TestSecondaryCache>(2048);
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1028,7 +1027,7 @@ TEST_F(LRUCacheSecondaryCacheTest, SaveFailTest) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache =
std::make_shared<TestSecondaryCache>(2048);
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1071,7 +1070,7 @@ TEST_F(LRUCacheSecondaryCacheTest, CreateFailTest) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache =
std::make_shared<TestSecondaryCache>(2048);
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1115,7 +1114,7 @@ TEST_F(LRUCacheSecondaryCacheTest, FullCapacityTest) {
true /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache =
std::make_shared<TestSecondaryCache>(2048);
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1167,7 +1166,7 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheCorrectness1) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1267,7 +1266,7 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheCorrectness2) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1363,7 +1362,7 @@ TEST_F(DBSecondaryCacheTest, NoSecondaryCacheInsertion) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1420,7 +1419,7 @@ TEST_F(DBSecondaryCacheTest, SecondaryCacheIntensiveTesting) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1472,7 +1471,7 @@ TEST_F(DBSecondaryCacheTest, SecondaryCacheFailureTest) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1567,7 +1566,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicWaitAllTest) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache =
std::make_shared<TestSecondaryCache>(32 * 1024);
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1626,7 +1625,7 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheMultiGet) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -1760,11 +1759,11 @@ class LRUCacheWithStat : public LRUCache {
#ifndef ROCKSDB_LITE

TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
LRUCacheOptions cache_opts(
1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
LRUCacheOptions cache_opts(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */,
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
LRUCacheWithStat* tmp_cache = new LRUCacheWithStat(
cache_opts.capacity, cache_opts.num_shard_bits,
cache_opts.strict_capacity_limit, cache_opts.high_pri_pool_ratio,
Expand Down Expand Up @@ -1900,11 +1899,11 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
}

TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
LRUCacheOptions cache_opts(
1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
LRUCacheOptions cache_opts(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */,
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
LRUCacheWithStat* tmp_cache = new LRUCacheWithStat(
cache_opts.capacity, cache_opts.num_shard_bits,
cache_opts.strict_capacity_limit, cache_opts.high_pri_pool_ratio,
Expand Down Expand Up @@ -2080,7 +2079,7 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheOptionBasic) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -2178,7 +2177,7 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheOptionChange) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down Expand Up @@ -2276,7 +2275,7 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheOptionTwoDB) {
false /* strict_capacity_limit */,
0.5 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 0.5 /* low_pri_pool_ratio */);
kDontChargeCacheMetadata);
std::shared_ptr<TestSecondaryCache> secondary_cache(
new TestSecondaryCache(2048 * 1024));
opts.secondary_cache = secondary_cache;
Expand Down
2 changes: 1 addition & 1 deletion db/db_block_cache_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -820,7 +820,7 @@ class MockCache : public LRUCache {
MockCache()
: LRUCache((size_t)1 << 25 /*capacity*/, 0 /*num_shard_bits*/,
false /*strict_capacity_limit*/, 0.0 /*high_pri_pool_ratio*/,
1.0 /*low_pri_pool_ratio*/) {}
0.0 /*low_pri_pool_ratio*/) {}

using ShardedCache::Insert;

Expand Down
2 changes: 1 addition & 1 deletion db/db_test2.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,7 @@ TEST_F(DBTest2, TestWriteBufferNoLimitWithCache) {
10000000 /* capacity */, 1 /* num_shard_bits */,
false /* strict_capacity_limit */, 0.0 /* high_pri_pool_ratio */,
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
kDontChargeCacheMetadata, 1.0 /* low_pri_pool_ratio */));
kDontChargeCacheMetadata));

options.write_buffer_size = 50000; // this is never hit
// Use a write buffer total size so that the soft limit is about
Expand Down
11 changes: 5 additions & 6 deletions include/rocksdb/cache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,9 @@ struct LRUCacheOptions {
// the low-pri list (the midpoint) and bottom-pri entries will be first
// inserted to the bottom-pri list.
//
// The default value is -1.0, which means that uses (1 - high_pri_pool_ratio).
//
// See also high_pri_pool_ratio.
double low_pri_pool_ratio = -1.0;
double low_pri_pool_ratio = 0.0;

// If non-nullptr will use this allocator instead of system allocator when
// allocating memory for cache blocks. Call this method before you start using
Expand Down Expand Up @@ -112,7 +111,7 @@ struct LRUCacheOptions {
bool _use_adaptive_mutex = kDefaultToAdaptiveMutex,
CacheMetadataChargePolicy _metadata_charge_policy =
kDefaultCacheMetadataChargePolicy,
double _low_pri_pool_ratio = -1.0)
double _low_pri_pool_ratio = 0.0)
: capacity(_capacity),
num_shard_bits(_num_shard_bits),
strict_capacity_limit(_strict_capacity_limit),
Expand All @@ -138,7 +137,7 @@ extern std::shared_ptr<Cache> NewLRUCache(
bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
CacheMetadataChargePolicy metadata_charge_policy =
kDefaultCacheMetadataChargePolicy,
double low_pri_pool_ratio = -1.0);
double low_pri_pool_ratio = 0.0);

extern std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts);

Expand Down Expand Up @@ -166,7 +165,7 @@ struct CompressedSecondaryCacheOptions : LRUCacheOptions {
CacheMetadataChargePolicy _metadata_charge_policy =
kDefaultCacheMetadataChargePolicy,
CompressionType _compression_type = CompressionType::kLZ4Compression,
uint32_t _compress_format_version = 2, double _low_pri_pool_ratio = -1.0)
uint32_t _compress_format_version = 2, double _low_pri_pool_ratio = 0.0)
: LRUCacheOptions(_capacity, _num_shard_bits, _strict_capacity_limit,
_high_pri_pool_ratio, std::move(_memory_allocator),
_use_adaptive_mutex, _metadata_charge_policy,
Expand All @@ -185,7 +184,7 @@ extern std::shared_ptr<SecondaryCache> NewCompressedSecondaryCache(
CacheMetadataChargePolicy metadata_charge_policy =
kDefaultCacheMetadataChargePolicy,
CompressionType compression_type = CompressionType::kLZ4Compression,
uint32_t compress_format_version = 2, double low_pri_pool_ratio = -1.0);
uint32_t compress_format_version = 2, double low_pri_pool_ratio = 0.0);

extern std::shared_ptr<SecondaryCache> NewCompressedSecondaryCache(
const CompressedSecondaryCacheOptions& opts);
Expand Down
2 changes: 1 addition & 1 deletion table/block_based/block_based_table_factory.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,7 @@ void BlockBasedTableFactory::InitializeOptions() {
// It makes little sense to pay overhead for mid-point insertion while the
// block size is only 8MB.
co.high_pri_pool_ratio = 0.0;
co.low_pri_pool_ratio = 1.0;
co.low_pri_pool_ratio = 0.0;
table_options_.block_cache = NewLRUCache(co);
}
if (table_options_.block_size_deviation < 0 ||
Expand Down
Loading

0 comments on commit e69183b

Please sign in to comment.