Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for FastLRUCache in stress and crash tests. #10081

Closed
wants to merge 16 commits into from
Closed

Add support for FastLRUCache in stress and crash tests. #10081

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
f1b8f64
Add support for FastLRUCache in stress test and crash test.
May 31, 2022
ec67dd5
Cache type is an enum. Validate --secondary_cache_uri and --cache_type.
Jun 1, 2022
4fddac5
Remove comment.
Jun 1, 2022
ffdc007
Cache types and conversions defined separately.
Jun 1, 2022
74224cf
Format.
Jun 1, 2022
31e3034
Amend revert.
Jun 1, 2022
a2f7176
Revert "Amend revert."
Jun 1, 2022
9fb8df9
Revert "Format."
Jun 1, 2022
c66c95e
Revert "Revert "Format.""
Jun 1, 2022
a066e3a
Revert "Revert "Amend revert.""
Jun 1, 2022
e8ca385
Revert "Amend revert."
Jun 1, 2022
527b0a2
Revert "Format."
Jun 1, 2022
2e01a95
Revert "Cache types and conversions defined separately."
Jun 1, 2022
42e146b
Comparison without going through an enum.
Jun 1, 2022
943adac
Dummy push to trigger CircleCI.
Jun 1, 2022
f9d1aea
Rollback trivial change.
Jun 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion db_stress_tool/db_stress_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ DECLARE_bool(charge_table_reader);
DECLARE_int32(top_level_index_pinning);
DECLARE_int32(partition_pinning);
DECLARE_int32(unpartitioned_pinning);
DECLARE_bool(use_clock_cache);
DECLARE_string(cache_type);
DECLARE_uint64(subcompactions);
DECLARE_uint64(periodic_compaction_seconds);
DECLARE_uint64(compaction_ttl);
Expand Down
3 changes: 1 addition & 2 deletions db_stress_tool/db_stress_gflags.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,8 +343,7 @@ DEFINE_int32(
"Type of pinning for unpartitioned metadata blocks (see `enum PinningTier` "
"in table.h)");

DEFINE_bool(use_clock_cache, false,
"Replace default LRU block cache with clock cache.");
DEFINE_string(cache_type, "lru_cache", "Type of block cache.");

DEFINE_uint64(subcompactions, 1,
"Maximum number of subcompactions to divide L0-L1 compactions "
Expand Down
11 changes: 9 additions & 2 deletions db_stress_tool/db_stress_test_base.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

#include "util/compression.h"
#ifdef GFLAGS
#include "cache/fast_lru_cache.h"
#include "db_stress_tool/db_stress_common.h"
#include "db_stress_tool/db_stress_compaction_filter.h"
#include "db_stress_tool/db_stress_driver.h"
Expand Down Expand Up @@ -130,14 +131,17 @@ std::shared_ptr<Cache> StressTest::NewCache(size_t capacity,
if (capacity <= 0) {
return nullptr;
}
if (FLAGS_use_clock_cache) {

if (FLAGS_cache_type == "clock_cache") {
auto cache = NewClockCache((size_t)capacity);
if (!cache) {
fprintf(stderr, "Clock cache not supported.");
exit(1);
}
return cache;
} else {
} else if (FLAGS_cache_type == "fast_lru_cache") {
return NewFastLRUCache((size_t)capacity, num_shard_bits);
} else if (FLAGS_cache_type == "lru_cache") {
LRUCacheOptions opts;
opts.capacity = capacity;
opts.num_shard_bits = num_shard_bits;
Expand All @@ -161,6 +165,9 @@ std::shared_ptr<Cache> StressTest::NewCache(size_t capacity,
}
#endif
return NewLRUCache(opts);
} else {
fprintf(stderr, "Cache type not supported.");
exit(1);
}
}

Expand Down
16 changes: 10 additions & 6 deletions tools/db_crashtest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@
"use_direct_reads": lambda: random.randint(0, 1),
"use_direct_io_for_flush_and_compaction": lambda: random.randint(0, 1),
"mock_direct_io": False,
"use_clock_cache": 0, # currently broken
"cache_type": lambda: random.choice(["fast_lru_cache", "lru_cache"]), # clock_cache is broken
"use_full_merge_v1": lambda: random.randint(0, 1),
"use_merge": lambda: random.randint(0, 1),
# 999 -> use Bloom API
Expand Down Expand Up @@ -176,6 +176,7 @@
"async_io": lambda: random.choice([0, 1]),
"wal_compression": lambda: random.choice(["none", "zstd"]),
"verify_sst_unique_id_in_manifest": 1, # always do unique_id verification
"secondary_cache_uri": "",
}

_TEST_DIR_ENV_VAR = 'TEST_TMPDIR'
Expand Down Expand Up @@ -525,11 +526,14 @@ def finalize_and_sanitize(src_params):
if dest_params.get("two_write_queues") == 1:
dest_params["enable_pipelined_write"] = 0
if dest_params.get("best_efforts_recovery") == 1:
dest_params["disable_wal"] = 1
dest_params["atomic_flush"] = 0
dest_params["enable_compaction_filter"] = 0
dest_params["sync"] = 0
dest_params["write_fault_one_in"] = 0
dest_params["disable_wal"] = 1
dest_params["atomic_flush"] = 0
dest_params["enable_compaction_filter"] = 0
dest_params["sync"] = 0
dest_params["write_fault_one_in"] = 0
if dest_params["secondary_cache_uri"] != "":
# Currently the only cache type compatible with a secondary cache is LRUCache
dest_params["cache_type"] = "lru_cache"

return dest_params

Expand Down