From ed562d10733f17b67dc16361a110817ea8d77ec8 Mon Sep 17 00:00:00 2001 From: Amnon Hanuhov Date: Sun, 21 May 2023 17:08:54 +0300 Subject: [PATCH] include/rocksdb: Add an options validation against db crash/stress test --- CMakeLists.txt | 1 + db/db_crashtest_use_case.cc | 336 ++++++++++++++++++++++++ include/rocksdb/db_crashtest_use_case.h | 199 ++++++++++++++ 3 files changed, 536 insertions(+) create mode 100644 db/db_crashtest_use_case.cc create mode 100644 include/rocksdb/db_crashtest_use_case.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f273f617e0..77c8f39ce0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -731,6 +731,7 @@ set(SOURCES db/compaction/sst_partitioner.cc db/compaction/subcompaction_state.cc db/convenience.cc + db/db_crashtest_use_case.cc db/db_filesnapshot.cc db/db_impl/compacted_db_impl.cc db/db_impl/db_impl.cc diff --git a/db/db_crashtest_use_case.cc b/db/db_crashtest_use_case.cc new file mode 100644 index 0000000000..85c5533350 --- /dev/null +++ b/db/db_crashtest_use_case.cc @@ -0,0 +1,336 @@ +#include + +#include "rocksdb/db_crashtest_use_case.h" + +#include "rocksdb/advanced_options.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/customizable_util.h" + +namespace ROCKSDB_NAMESPACE { + +void DBCrashtestUseCase::InitDefaultParams() { + memtable_protection_bytes_per_key = {0, 1, 2, 4, 8}; + block_size = {4096, 16384}; + bloom_bits_per_key = std::make_tuple(0, 19); + max_background_compactions = 20; + max_bytes_for_level_base = 10485760; + max_write_buffer_number = 3; + max_open_files = {-1, 100, 500000}; + recycle_log_file_num = {0, 1}; + max_subcompactions = std::make_tuple(1, 4); + target_file_size_base = 2097152; + target_file_size_multiplier = 2; + write_buffer_size = {1024 * 1024, 8 * 1024 * 1024, 128 * 1024 * 1024, + 1024 * 1024 * 1024}; + format_version = {2, 3, 4, 5}; + index_block_restart_interval = std::make_tuple(1, 16); + periodic_compaction_seconds = {0, 1, 2, 10, 100, 1000}; + stats_dump_period_sec = {0, 10, 600}; + max_manifest_file_size = {1 * 16384, 2 * 16384, 1024 * 1024 * 1024}; + bytes_per_sync = {0, 262144}; + wal_bytes_per_sync = {0, 524288}; + db_write_buffer_size = {0, 1024 * 1024, 8 * 1024 * 1024, 128 * 1024 * 1024, + 1024 * 1024 * 1024}; + max_write_batch_group_size_bytes = {16, 64, 1024 * 1024, 16 * 1024 * 1024}; + level_compaction_dynamic_level_bytes = true; + max_write_buffer_size_to_maintain = {0, 1024 * 1024, 2 * 1024 * 1024, + 4 * 1024 * 1024, 8 * 1024 * 1024}; + memtable_prefix_bloom_size_ratio = {0.001, 0.01, 0.1, 0.5}; + wal_compression = {CompressionType::kNoCompression, CompressionType::kZSTD}; + verify_sst_unique_id_in_manifest = true; + allow_data_in_errors = true; + initial_auto_readahead_size = {0, 16384, 524288}; + max_auto_readahead_size = {0, 16384, 524288}; + num_file_reads_for_auto_readahead = std::make_tuple(0, 2); + min_write_buffer_number_to_merge = {1, 2}; + preserve_internal_time_seconds = {0, 60, 3600, 36000}; +} + +DBCrashtestUseCase::DBCrashtestUseCase() { InitDefaultParams(); } + +bool DBCrashtestUseCase::Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) { + bool result = + ValueMatches("max_background_compactions", + db_opts.max_background_compactions, + max_background_compactions, valid_opts, invalid_opts) && + ValueExists("recycle_log_file_num", db_opts.recycle_log_file_num, + recycle_log_file_num, valid_opts, invalid_opts) && + ValueInRange("max_subcompactions", db_opts.max_subcompactions, + max_subcompactions, valid_opts, invalid_opts) && + ValueExists("stats_dump_period_sec", db_opts.stats_dump_period_sec, + stats_dump_period_sec, valid_opts, invalid_opts) && + ValueExists("max_manifest_file_size", db_opts.max_manifest_file_size, + max_manifest_file_size, valid_opts, invalid_opts) && + ValueExists("bytes_per_sync", db_opts.bytes_per_sync, bytes_per_sync, + valid_opts, invalid_opts) && + ValueExists("wal_bytes_per_sync", db_opts.wal_bytes_per_sync, + wal_bytes_per_sync, valid_opts, invalid_opts) && + ValueExists("db_write_buffer_size", db_opts.db_write_buffer_size, + db_write_buffer_size, valid_opts, invalid_opts) && + ValueExists("wal_compression", db_opts.wal_compression, wal_compression, + valid_opts, invalid_opts) && + ValueExists("max_write_batch_group_size_bytes", + db_opts.max_write_batch_group_size_bytes, + max_write_batch_group_size_bytes, valid_opts, invalid_opts) && + ValueMatches("verify_sst_unique_id_in_manifest", + db_opts.verify_sst_unique_id_in_manifest, + verify_sst_unique_id_in_manifest, valid_opts, + invalid_opts) && + ValueMatches("allow_data_in_errors", db_opts.allow_data_in_errors, + allow_data_in_errors, valid_opts, invalid_opts); + + return result; +} + +bool DBCrashtestUseCase::Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) { + bool result = + ValueExists("memtable_protection_bytes_per_key", + cf_opts.memtable_protection_bytes_per_key, + memtable_protection_bytes_per_key, valid_opts, + invalid_opts) && + ValueMatches("max_bytes_for_level_base", cf_opts.max_bytes_for_level_base, + max_bytes_for_level_base, valid_opts, invalid_opts) && + ValueMatches("max_write_buffer_number", cf_opts.max_write_buffer_number, + max_write_buffer_number, valid_opts, invalid_opts) && + ValueMatches("target_file_size_base", cf_opts.target_file_size_base, + target_file_size_base, valid_opts, invalid_opts) && + ValueMatches("target_file_size_multiplier", + cf_opts.target_file_size_multiplier, + target_file_size_multiplier, valid_opts, invalid_opts) && + ValueExists("periodic_compaction_seconds", + cf_opts.periodic_compaction_seconds, + periodic_compaction_seconds, valid_opts, invalid_opts) && + ValueMatches("level_compaction_dynamic_level_bytes", + cf_opts.level_compaction_dynamic_level_bytes, + level_compaction_dynamic_level_bytes, valid_opts, + invalid_opts) && + ValueExists("max_write_buffer_size_to_maintain", + cf_opts.max_write_buffer_size_to_maintain, + max_write_buffer_size_to_maintain, valid_opts, + invalid_opts) && + ValueExists("memtable_prefix_bloom_size_ratio", + cf_opts.memtable_prefix_bloom_size_ratio, + memtable_prefix_bloom_size_ratio, valid_opts, invalid_opts) && + ValueExists("min_write_buffer_number_to_merge", + cf_opts.min_write_buffer_number_to_merge, + min_write_buffer_number_to_merge, valid_opts, invalid_opts) && + ValueExists("preserve_internal_time_seconds", + cf_opts.preserve_internal_time_seconds, + preserve_internal_time_seconds, valid_opts, invalid_opts); + + return result; +} + +bool DBCrashtestUseCase::Validate(const ConfigOptions& cfg_opts, + const BlockBasedTableOptions& bbt_opts, + std::set& valid_opts, + std::set& invalid_opts) { + bool result = + ValueExists("block_size", bbt_opts.block_size, block_size, valid_opts, + invalid_opts) && + ValueExists("format_version", bbt_opts.format_version, format_version, + valid_opts, invalid_opts) && + ValueInRange("index_block_restart_interval", + bbt_opts.index_block_restart_interval, + index_block_restart_interval, valid_opts, invalid_opts) && + ValueExists("initial_auto_readahead_size", + bbt_opts.initial_auto_readahead_size, + initial_auto_readahead_size, valid_opts, invalid_opts) && + ValueExists("max_auto_readahead_size", bbt_opts.max_auto_readahead_size, + max_auto_readahead_size, valid_opts, invalid_opts) && + ValueInRange("num_file_reads_for_auto_readahead", + bbt_opts.num_file_reads_for_auto_readahead, + num_file_reads_for_auto_readahead, valid_opts, invalid_opts); + + return result; +} + +bool DBCrashtestUseCase::Validate( + const ConfigOptions& cfg_opts, const Options& opts, + std::set& valid_opts, + std::set& invalid_opts) { + const DBOptions* db_opts = &opts; + const ColumnFamilyOptions* cf_opts = &opts; + return Validate(cfg_opts, *db_opts, valid_opts, invalid_opts) && + Validate(cfg_opts, *cf_opts, valid_opts, invalid_opts); +} + +SimpleDefaultParams::SimpleDefaultParams() { + max_background_compactions = 1; + max_bytes_for_level_base = 67108864; + target_file_size_base = 16777216; + target_file_size_multiplier = 1; + write_buffer_size = {32 * 1024 * 1024}; + level_compaction_dynamic_level_bytes = false; +} + +bool SimpleDefaultParams::Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts, + invalid_opts); +} + +bool SimpleDefaultParams::Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts, + invalid_opts); +} + +TxnParams::TxnParams() { enable_pipelined_write = false; } + +bool TxnParams::Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts, + invalid_opts) && + ValueMatches("enable_pipelined_write", + db_opts.enable_pipelined_write, enable_pipelined_write, + valid_opts, invalid_opts); +} + +bool TxnParams::Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts, + invalid_opts); +} + +BestEffortsRecoveryParams::BestEffortsRecoveryParams() { + best_efforts_recovery = true; + atomic_flush = false; +} + +bool BestEffortsRecoveryParams::Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts, + invalid_opts) && + ValueMatches("best_efforts_recovery", db_opts.best_efforts_recovery, + best_efforts_recovery, valid_opts, invalid_opts) && + ValueMatches("atomic_flush", db_opts.atomic_flush, atomic_flush, + valid_opts, invalid_opts); +} + +bool BestEffortsRecoveryParams::Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts, + invalid_opts); +} + +BlobParams::BlobParams() { + min_blob_size = {0, 8, 16}; + blob_file_size = {1048576, 16777216, 268435456, 1073741824}; + blob_compression_type = { + CompressionType::kNoCompression, CompressionType::kSnappyCompression, + CompressionType::kLZ4Compression, CompressionType::kZSTD}; + blob_garbage_collection_age_cutoff = {0.0, 0.25, 0.5, 0.75, 1.0}; + blob_garbage_collection_force_threshold = {0.5, 0.75, 1.0}; + blob_compaction_readahead_size = {0, 1048576, 4194304}; + blob_file_starting_level = {0, 1, 2, 3}; +} + +bool BlobParams::Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts, + invalid_opts); +} + +bool BlobParams::Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts, + invalid_opts) && + ValueExists("min_blob_size", cf_opts.min_blob_size, min_blob_size, + valid_opts, invalid_opts) && + ValueExists("blob_file_size", cf_opts.blob_file_size, blob_file_size, + valid_opts, invalid_opts) && + ValueExists("blob_compression_type", cf_opts.blob_compression_type, + blob_compression_type, valid_opts, invalid_opts) && + ValueExists("blob_garbage_collection_age_cutoff", + cf_opts.blob_garbage_collection_age_cutoff, + blob_garbage_collection_age_cutoff, valid_opts, + invalid_opts) && + ValueExists("blob_garbage_collection_force_threshold", + cf_opts.blob_garbage_collection_force_threshold, + blob_garbage_collection_force_threshold, valid_opts, + invalid_opts) && + ValueExists("blob_compaction_readahead_size", + cf_opts.blob_compaction_readahead_size, + blob_compaction_readahead_size, valid_opts, + invalid_opts) && + ValueExists("blob_file_starting_level", + cf_opts.blob_file_starting_level, + blob_file_starting_level, valid_opts, invalid_opts); +} + +TieredParams::TieredParams() { + preclude_last_level_data_seconds = {60, 3600, 36000}; + compaction_style = CompactionStyle::kCompactionStyleUniversal; + enable_blob_files = false; +} + +bool TieredParams::Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts, + invalid_opts); +} + +bool TieredParams::Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts, + invalid_opts) && + ValueExists("preclude_last_level_data_seconds", + cf_opts.preclude_last_level_data_seconds, + preclude_last_level_data_seconds, valid_opts, + invalid_opts) && + ValueMatches("compaction_style", cf_opts.compaction_style, + compaction_style, valid_opts, invalid_opts) && + ValueMatches("enable_blob_files", cf_opts.enable_blob_files, + enable_blob_files, valid_opts, invalid_opts); +} + +MultiopsTxnDefaultParams::MultiopsTxnDefaultParams() { + write_buffer_size = {65536}; +} + +bool MultiopsTxnDefaultParams::Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts, + invalid_opts); +} + +bool MultiopsTxnDefaultParams::Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) { + return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts, + invalid_opts); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/db_crashtest_use_case.h b/include/rocksdb/db_crashtest_use_case.h new file mode 100644 index 0000000000..3df25fd132 --- /dev/null +++ b/include/rocksdb/db_crashtest_use_case.h @@ -0,0 +1,199 @@ +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/options.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/use_case.h" +#include "rocksdb/utilities/customizable_util.h" + +namespace ROCKSDB_NAMESPACE { + +class DBCrashtestUseCase : public UseCase { + public: + DBCrashtestUseCase(); + static const char* kClassName() { return "rocksdb.DBCrashtestUseCase"; } + const char* Name() const override { return kClassName(); } + Status Populate(const ConfigOptions& cfg_opts, DBOptions& db_opts) override { + return Status::OK(); + } + Status Populate(const ConfigOptions& cfg_opts, + ColumnFamilyOptions& cf_opts) override { + return Status::OK(); + } + Status Populate(const ConfigOptions& cfg_opts, + BlockBasedTableOptions& bbt_opts) const { + return Status::OK(); + } + virtual bool Validate(const ConfigOptions& cfg_opts, const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) override; + virtual bool Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) override; + virtual bool Validate(const ConfigOptions& cfg_opts, + const BlockBasedTableOptions& bbt_opts, + std::set& valid_opts, + std::set& invalid_opts); + bool Validate(const ConfigOptions& cfg_opts, const Options& opts, + std::set& valid_opts, + std::set& invalid_opts) override; + + protected: + std::vector memtable_protection_bytes_per_key; + std::vector block_size; + std::tuple bloom_bits_per_key; + int max_background_compactions; + uint64_t max_bytes_for_level_base; + int max_write_buffer_number; + std::vector max_open_files; + std::vector recycle_log_file_num; + std::tuple max_subcompactions; + uint64_t target_file_size_base; + int target_file_size_multiplier; + std::vector write_buffer_size; + std::vector format_version; + std::tuple index_block_restart_interval; + std::vector periodic_compaction_seconds; + std::vector stats_dump_period_sec; + std::vector max_manifest_file_size; + std::vector bytes_per_sync; + std::vector wal_bytes_per_sync; + std::vector db_write_buffer_size; + std::vector max_write_batch_group_size_bytes; + bool level_compaction_dynamic_level_bytes; + std::vector max_write_buffer_size_to_maintain; + std::vector memtable_prefix_bloom_size_ratio; + std::vector wal_compression; + bool verify_sst_unique_id_in_manifest; + bool allow_data_in_errors; + std::vector initial_auto_readahead_size; + std::vector max_auto_readahead_size; + std::tuple num_file_reads_for_auto_readahead; + std::vector min_write_buffer_number_to_merge; + std::vector preserve_internal_time_seconds; + bool enable_pipelined_write; + bool best_efforts_recovery; + bool atomic_flush; + std::vector min_blob_size; + std::vector blob_file_size; + std::vector blob_compression_type; + std::vector blob_garbage_collection_age_cutoff; + std::vector blob_garbage_collection_force_threshold; + std::vector blob_compaction_readahead_size; + std::vector blob_file_starting_level; + std::vector preclude_last_level_data_seconds; + CompactionStyle compaction_style; + bool enable_blob_files; + + private: + void InitDefaultParams(); +}; + +class SimpleDefaultParams : public DBCrashtestUseCase { + public: + SimpleDefaultParams(); + static const char* kClassName() { return "rocksdb.SimpleDefaultParams"; } + const char* Name() const override { return kClassName(); } + + virtual bool Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) override; + + virtual bool Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) override; +}; + +class TxnParams : public DBCrashtestUseCase { + public: + TxnParams(); + static const char* kClassName() { return "rocksdb.TxnParams"; } + const char* Name() const override { return kClassName(); } + + virtual bool Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) override; + + virtual bool Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) override; +}; + +class BestEffortsRecoveryParams : public DBCrashtestUseCase { + public: + BestEffortsRecoveryParams(); + static const char* kClassName() { return "rocksdb.BestEffortsRecoveryParams"; } + const char* Name() const override { return kClassName(); } + + virtual bool Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) override; + + virtual bool Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) override; +}; + +class BlobParams : public DBCrashtestUseCase { + public: + BlobParams(); + static const char* kClassName() { return "rocksdb.BlobParams"; } + const char* Name() const override { return kClassName(); } + + virtual bool Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) override; + + virtual bool Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) override; +}; + +class TieredParams : public DBCrashtestUseCase { + public: + TieredParams(); + static const char* kClassName() { return "rocksdb.TieredParams"; } + const char* Name() const override { return kClassName(); } + + virtual bool Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) override; + + virtual bool Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) override; +}; + +class MultiopsTxnDefaultParams : public DBCrashtestUseCase { + public: + MultiopsTxnDefaultParams(); + static const char* kClassName() { return "rocksdb.MultiopsTxnDefaultParams"; } + const char* Name() const override { return kClassName(); } + + virtual bool Validate(const ConfigOptions& cfg_opts, + const DBOptions& db_opts, + std::set& valid_opts, + std::set& invalid_opts) override; + + virtual bool Validate(const ConfigOptions& cfg_opts, + const ColumnFamilyOptions& cf_opts, + std::set& valid_opts, + std::set& invalid_opts) override; +}; +} // namespace ROCKSDB_NAMESPACE