include/rocksdb: Add an options validation against db crash/stress test

CMakeLists.txt

@@ -731,6 +731,7 @@ set(SOURCES
         db/compaction/sst_partitioner.cc
         db/compaction/subcompaction_state.cc
         db/convenience.cc
+        db/db_crashtest_use_case.cc
         db/db_filesnapshot.cc
         db/db_impl/compacted_db_impl.cc
         db/db_impl/db_impl.cc

db/db_crashtest_use_case.cc

@@ -0,0 +1,336 @@
+#include <memory>
+
+#include "rocksdb/db_crashtest_use_case.h"
+
+#include "rocksdb/advanced_options.h"
+#include "rocksdb/compression_type.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table.h"
+#include "rocksdb/utilities/customizable_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+void DBCrashtestUseCase::InitDefaultParams() {
+  memtable_protection_bytes_per_key = {0, 1, 2, 4, 8};
+  block_size = {4096, 16384};
+  bloom_bits_per_key = std::make_tuple(0, 19);
+  max_background_compactions = 20;
+  max_bytes_for_level_base = 10485760;
+  max_write_buffer_number = 3;
+  max_open_files = {-1, 100, 500000};
+  recycle_log_file_num = {0, 1};
+  max_subcompactions = std::make_tuple(1, 4);
+  target_file_size_base = 2097152;
+  target_file_size_multiplier = 2;
+  write_buffer_size = {1024 * 1024, 8 * 1024 * 1024, 128 * 1024 * 1024,
+                       1024 * 1024 * 1024};
+  format_version = {2, 3, 4, 5};
+  index_block_restart_interval = std::make_tuple(1, 16);
+  periodic_compaction_seconds = {0, 1, 2, 10, 100, 1000};
+  stats_dump_period_sec = {0, 10, 600};
+  max_manifest_file_size = {1 * 16384, 2 * 16384, 1024 * 1024 * 1024};
+  bytes_per_sync = {0, 262144};
+  wal_bytes_per_sync = {0, 524288};
+  db_write_buffer_size = {0, 1024 * 1024, 8 * 1024 * 1024, 128 * 1024 * 1024,
+                          1024 * 1024 * 1024};
+  max_write_batch_group_size_bytes = {16, 64, 1024 * 1024, 16 * 1024 * 1024};
+  level_compaction_dynamic_level_bytes = true;
+  max_write_buffer_size_to_maintain = {0, 1024 * 1024, 2 * 1024 * 1024,
+                                       4 * 1024 * 1024, 8 * 1024 * 1024};
+  memtable_prefix_bloom_size_ratio = {0.001, 0.01, 0.1, 0.5};
+  wal_compression = {CompressionType::kNoCompression, CompressionType::kZSTD};
+  verify_sst_unique_id_in_manifest = true;
+  allow_data_in_errors = true;
+  initial_auto_readahead_size = {0, 16384, 524288};
+  max_auto_readahead_size = {0, 16384, 524288};
+  num_file_reads_for_auto_readahead = std::make_tuple(0, 2);
+  min_write_buffer_number_to_merge = {1, 2};
+  preserve_internal_time_seconds = {0, 60, 3600, 36000};
+}
+
+DBCrashtestUseCase::DBCrashtestUseCase() { InitDefaultParams(); }
+
+bool DBCrashtestUseCase::Validate(const ConfigOptions& cfg_opts,
+                                  const DBOptions& db_opts,
+                                  std::set<std::string>& valid_opts,
+                                  std::set<std::string>& invalid_opts) {
+  bool result =
+      ValueMatches("max_background_compactions",
+                   db_opts.max_background_compactions,
+                   max_background_compactions, valid_opts, invalid_opts) &&
+      ValueExists("recycle_log_file_num", db_opts.recycle_log_file_num,
+                  recycle_log_file_num, valid_opts, invalid_opts) &&
+      ValueInRange("max_subcompactions", db_opts.max_subcompactions,
+                   max_subcompactions, valid_opts, invalid_opts) &&
+      ValueExists("stats_dump_period_sec", db_opts.stats_dump_period_sec,
+                  stats_dump_period_sec, valid_opts, invalid_opts) &&
+      ValueExists("max_manifest_file_size", db_opts.max_manifest_file_size,
+                  max_manifest_file_size, valid_opts, invalid_opts) &&
+      ValueExists("bytes_per_sync", db_opts.bytes_per_sync, bytes_per_sync,
+                  valid_opts, invalid_opts) &&
+      ValueExists("wal_bytes_per_sync", db_opts.wal_bytes_per_sync,
+                  wal_bytes_per_sync, valid_opts, invalid_opts) &&
+      ValueExists("db_write_buffer_size", db_opts.db_write_buffer_size,
+                  db_write_buffer_size, valid_opts, invalid_opts) &&
+      ValueExists("wal_compression", db_opts.wal_compression, wal_compression,
+                  valid_opts, invalid_opts) &&
+      ValueExists("max_write_batch_group_size_bytes",
+                  db_opts.max_write_batch_group_size_bytes,
+                  max_write_batch_group_size_bytes, valid_opts, invalid_opts) &&
+      ValueMatches("verify_sst_unique_id_in_manifest",
+                   db_opts.verify_sst_unique_id_in_manifest,
+                   verify_sst_unique_id_in_manifest, valid_opts,
+                   invalid_opts) &&
+      ValueMatches("allow_data_in_errors", db_opts.allow_data_in_errors,
+                   allow_data_in_errors, valid_opts, invalid_opts);
+
+  return result;
+}
+
+bool DBCrashtestUseCase::Validate(const ConfigOptions& cfg_opts,
+                                  const ColumnFamilyOptions& cf_opts,
+                                  std::set<std::string>& valid_opts,
+                                  std::set<std::string>& invalid_opts) {
+  bool result =
+      ValueExists("memtable_protection_bytes_per_key",
+                  cf_opts.memtable_protection_bytes_per_key,
+                  memtable_protection_bytes_per_key, valid_opts,
+                  invalid_opts) &&
+      ValueMatches("max_bytes_for_level_base", cf_opts.max_bytes_for_level_base,
+                   max_bytes_for_level_base, valid_opts, invalid_opts) &&
+      ValueMatches("max_write_buffer_number", cf_opts.max_write_buffer_number,
+                   max_write_buffer_number, valid_opts, invalid_opts) &&
+      ValueMatches("target_file_size_base", cf_opts.target_file_size_base,
+                   target_file_size_base, valid_opts, invalid_opts) &&
+      ValueMatches("target_file_size_multiplier",
+                   cf_opts.target_file_size_multiplier,
+                   target_file_size_multiplier, valid_opts, invalid_opts) &&
+      ValueExists("periodic_compaction_seconds",
+                  cf_opts.periodic_compaction_seconds,
+                  periodic_compaction_seconds, valid_opts, invalid_opts) &&
+      ValueMatches("level_compaction_dynamic_level_bytes",
+                   cf_opts.level_compaction_dynamic_level_bytes,
+                   level_compaction_dynamic_level_bytes, valid_opts,
+                   invalid_opts) &&
+      ValueExists("max_write_buffer_size_to_maintain",
+                  cf_opts.max_write_buffer_size_to_maintain,
+                  max_write_buffer_size_to_maintain, valid_opts,
+                  invalid_opts) &&
+      ValueExists("memtable_prefix_bloom_size_ratio",
+                  cf_opts.memtable_prefix_bloom_size_ratio,
+                  memtable_prefix_bloom_size_ratio, valid_opts, invalid_opts) &&
+      ValueExists("min_write_buffer_number_to_merge",
+                  cf_opts.min_write_buffer_number_to_merge,
+                  min_write_buffer_number_to_merge, valid_opts, invalid_opts) &&
+      ValueExists("preserve_internal_time_seconds",
+                  cf_opts.preserve_internal_time_seconds,
+                  preserve_internal_time_seconds, valid_opts, invalid_opts);
+
+  return result;
+}
+
+bool DBCrashtestUseCase::Validate(const ConfigOptions& cfg_opts,
+                                  const BlockBasedTableOptions& bbt_opts,
+                                  std::set<std::string>& valid_opts,
+                                  std::set<std::string>& invalid_opts) {
+  bool result =
+      ValueExists("block_size", bbt_opts.block_size, block_size, valid_opts,
+                  invalid_opts) &&
+      ValueExists("format_version", bbt_opts.format_version, format_version,
+                  valid_opts, invalid_opts) &&
+      ValueInRange("index_block_restart_interval",
+                   bbt_opts.index_block_restart_interval,
+                   index_block_restart_interval, valid_opts, invalid_opts) &&
+      ValueExists("initial_auto_readahead_size",
+                  bbt_opts.initial_auto_readahead_size,
+                  initial_auto_readahead_size, valid_opts, invalid_opts) &&
+      ValueExists("max_auto_readahead_size", bbt_opts.max_auto_readahead_size,
+                  max_auto_readahead_size, valid_opts, invalid_opts) &&
+      ValueInRange("num_file_reads_for_auto_readahead",
+                   bbt_opts.num_file_reads_for_auto_readahead,
+                   num_file_reads_for_auto_readahead, valid_opts, invalid_opts);
+
+  return result;
+}
+
+bool DBCrashtestUseCase::Validate(
+    const ConfigOptions& cfg_opts, const Options& opts,
+    std::set<std::string>& valid_opts,
+    std::set<std::string>& invalid_opts) {
+  const DBOptions* db_opts = &opts;
+  const ColumnFamilyOptions* cf_opts = &opts;
+  return Validate(cfg_opts, *db_opts, valid_opts, invalid_opts) &&
+         Validate(cfg_opts, *cf_opts, valid_opts, invalid_opts);
+}
+
+SimpleDefaultParams::SimpleDefaultParams() {
+  max_background_compactions = 1;
+  max_bytes_for_level_base = 67108864;
+  target_file_size_base = 16777216;
+  target_file_size_multiplier = 1;
+  write_buffer_size = {32 * 1024 * 1024};
+  level_compaction_dynamic_level_bytes = false;
+}
+
+bool SimpleDefaultParams::Validate(const ConfigOptions& cfg_opts,
+                                   const DBOptions& db_opts,
+                                   std::set<std::string>& valid_opts,
+                                   std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts,
+                                      invalid_opts);
+}
+
+bool SimpleDefaultParams::Validate(const ConfigOptions& cfg_opts,
+                                   const ColumnFamilyOptions& cf_opts,
+                                   std::set<std::string>& valid_opts,
+                                   std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts,
+                                      invalid_opts);
+}
+
+TxnParams::TxnParams() { enable_pipelined_write = false; }
+
+bool TxnParams::Validate(const ConfigOptions& cfg_opts,
+                         const DBOptions& db_opts,
+                         std::set<std::string>& valid_opts,
+                         std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts,
+                                      invalid_opts) &&
+         ValueMatches("enable_pipelined_write",
+                      db_opts.enable_pipelined_write, enable_pipelined_write,
+                      valid_opts, invalid_opts);
+}
+
+bool TxnParams::Validate(const ConfigOptions& cfg_opts,
+                         const ColumnFamilyOptions& cf_opts,
+                         std::set<std::string>& valid_opts,
+                         std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts,
+                                      invalid_opts);
+}
+
+BestEffortsRecoveryParams::BestEffortsRecoveryParams() {
+  best_efforts_recovery = true;
+  atomic_flush = false;
+}
+
+bool BestEffortsRecoveryParams::Validate(const ConfigOptions& cfg_opts,
+                                         const DBOptions& db_opts,
+                                         std::set<std::string>& valid_opts,
+                                         std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts,
+                                      invalid_opts) &&
+         ValueMatches("best_efforts_recovery", db_opts.best_efforts_recovery,
+                      best_efforts_recovery, valid_opts, invalid_opts) &&
+         ValueMatches("atomic_flush", db_opts.atomic_flush, atomic_flush,
+                      valid_opts, invalid_opts);
+}
+
+bool BestEffortsRecoveryParams::Validate(const ConfigOptions& cfg_opts,
+                                         const ColumnFamilyOptions& cf_opts,
+                                         std::set<std::string>& valid_opts,
+                                         std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts,
+                                      invalid_opts);
+}
+
+BlobParams::BlobParams() {
+  min_blob_size = {0, 8, 16};
+  blob_file_size = {1048576, 16777216, 268435456, 1073741824};
+  blob_compression_type = {
+      CompressionType::kNoCompression, CompressionType::kSnappyCompression,
+      CompressionType::kLZ4Compression, CompressionType::kZSTD};
+  blob_garbage_collection_age_cutoff = {0.0, 0.25, 0.5, 0.75, 1.0};
+  blob_garbage_collection_force_threshold = {0.5, 0.75, 1.0};
+  blob_compaction_readahead_size = {0, 1048576, 4194304};
+  blob_file_starting_level = {0, 1, 2, 3};
+}
+
+bool BlobParams::Validate(const ConfigOptions& cfg_opts,
+                          const DBOptions& db_opts,
+                          std::set<std::string>& valid_opts,
+                          std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts,
+                                      invalid_opts);
+}
+
+bool BlobParams::Validate(const ConfigOptions& cfg_opts,
+                          const ColumnFamilyOptions& cf_opts,
+                          std::set<std::string>& valid_opts,
+                          std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts,
+                                      invalid_opts) &&
+         ValueExists("min_blob_size", cf_opts.min_blob_size, min_blob_size,
+                      valid_opts, invalid_opts) &&
+         ValueExists("blob_file_size", cf_opts.blob_file_size, blob_file_size,
+                      valid_opts, invalid_opts) &&
+         ValueExists("blob_compression_type", cf_opts.blob_compression_type,
+                      blob_compression_type, valid_opts, invalid_opts) &&
+         ValueExists("blob_garbage_collection_age_cutoff",
+                      cf_opts.blob_garbage_collection_age_cutoff,
+                      blob_garbage_collection_age_cutoff, valid_opts,
+                      invalid_opts) &&
+         ValueExists("blob_garbage_collection_force_threshold",
+                      cf_opts.blob_garbage_collection_force_threshold,
+                      blob_garbage_collection_force_threshold, valid_opts,
+                      invalid_opts) &&
+         ValueExists("blob_compaction_readahead_size",
+                      cf_opts.blob_compaction_readahead_size,
+                      blob_compaction_readahead_size, valid_opts,
+                      invalid_opts) &&
+         ValueExists("blob_file_starting_level",
+                      cf_opts.blob_file_starting_level,
+                      blob_file_starting_level, valid_opts, invalid_opts);
+}
+
+TieredParams::TieredParams() {
+  preclude_last_level_data_seconds = {60, 3600, 36000};
+  compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  enable_blob_files = false;
+}
+
+bool TieredParams::Validate(const ConfigOptions& cfg_opts,
+                            const DBOptions& db_opts,
+                            std::set<std::string>& valid_opts,
+                            std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts,
+                                      invalid_opts);
+}
+
+bool TieredParams::Validate(const ConfigOptions& cfg_opts,
+                            const ColumnFamilyOptions& cf_opts,
+                            std::set<std::string>& valid_opts,
+                            std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts,
+                                      invalid_opts) &&
+         ValueExists("preclude_last_level_data_seconds",
+                      cf_opts.preclude_last_level_data_seconds,
+                      preclude_last_level_data_seconds, valid_opts,
+                      invalid_opts) &&
+         ValueMatches("compaction_style", cf_opts.compaction_style,
+                      compaction_style, valid_opts, invalid_opts) &&
+         ValueMatches("enable_blob_files", cf_opts.enable_blob_files,
+                      enable_blob_files, valid_opts, invalid_opts);
+}
+
+MultiopsTxnDefaultParams::MultiopsTxnDefaultParams() { 
+  write_buffer_size = {65536};
+}
+
+bool MultiopsTxnDefaultParams::Validate(const ConfigOptions& cfg_opts,
+                                        const DBOptions& db_opts,
+                                        std::set<std::string>& valid_opts,
+                                        std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, db_opts, valid_opts,
+                                      invalid_opts);
+}
+
+bool MultiopsTxnDefaultParams::Validate(const ConfigOptions& cfg_opts,
+                                        const ColumnFamilyOptions& cf_opts,
+                                        std::set<std::string>& valid_opts,
+                                        std::set<std::string>& invalid_opts) {
+  return DBCrashtestUseCase::Validate(cfg_opts, cf_opts, valid_opts,
+                                      invalid_opts);
+}
+}  // namespace ROCKSDB_NAMESPACE