new compact strategy(OpenAtomFoundation#2)

conf/pika.conf

@@ -636,4 +636,38 @@ cache-lfu-decay-time: 1
 # 'internal-used-unfinished-full-sync' is used to generate a metric 'is_eligible_for_master_election'
 # which serves for the scenario of codis-pika cluster reelection
 # You'd better [DO NOT MODIFY IT UNLESS YOU KNOW WHAT YOU ARE DOING]
-internal-used-unfinished-full-sync :
+internal-used-unfinished-full-sync :
+
+# Pika automatic compact compact strategy, a complement to rocksdb compact.
+# Trigger the compact background task periodically according to `compact-interval`
+# Can choose `full-compact` or `obd-compact`.
+# obd-compact https://github.com/OpenAtomFoundation/pika/issues/2255
+compaction-strategy : obd-compact
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+compact-every-num-of-files : 10
+
+# For OBD_Compact 
+# In another search, if the file creation time is 
+# greater than `force-compact-file-age-seconds`, 
+# a compaction of the upper and lower boundaries 
+# of the file will be performed at the same time 
+# `compact-every-num-of-files` -1
+force-compact-file-age-seconds : 300
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+force-compact-min-delete-ratio : 10
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+dont-compact-sst-created-in-seconds : 600
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+best-delete-min-ratio : 10

conf/s_pika.conf

@@ -0,0 +1,101 @@
+block_cache_size
+# 360 dba pika conf pika3.5.2
+port : 9221
+thread-num : 8
+log-path : ./log/
+loglevel : info
+db-path : ./db/
+write-buffer-size : 256M
+timeout : 30
+#requirepass : 06154eee364854d5
+#masterauth : 06154eee364854d5
+#userpass : 06154eee364854d5360
+#userblacklist : bgsave,dumpoff,client
+dump-prefix : pika-
+dump-expire : 1
+pidfile : .pika.pid
+daemonize : yes
+dump-path : ./dump/block_cache_size
+maxclients : 20000
+target-file-size-base : 20971520
+expire-logs-days : 7
+expire-logs-nums : 300
+root-connection-num : 10
+slowlog-log-slower-than : 100000
+binlog-file-size : 104857600
+compression : snappy
+db-sync-path : ./dbsync
+db-sync-speed : 60
+slowlog-write-errorlog : yes
+small-compaction-threshold : 5000
+max-write-buffer-size : 20737418240
+max-cache-files : 8000
+replication-num : 0
+consensus-level : 0
+max-cache-statistic-keys : 0
+thread-pool-size : 30
+slowlog-write-errorlog : yes
+default-slot-num : 1024
+instance-mode : classic
+databases : 1
+sync-thread-num : 1
+arena-block-size : 33554432
+max-background-jobs : 12
+max-background-flushes : 3
+max-background-compactions : 9
+rate-limiter-bandwidth : 1099511627776
+db-instance-num : 1
+block-size : 4096
+#block-cache : 5368709120
+block-cache : 4294967296
+max-subcompactions : 8
+
+#cache-maxmemory : 5368709120
+cache-lfu-decay-time : 1
+cache-maxmemory-samples : 5
+cache-maxmemory-policy : 1
+cache-num : 8
+cache-model : 0
+zset-cache-field-num-per-key : 512
+zset-cache-start-direction : 0
+cache-type :
+
+share-block-cache : yes
+throttle-bytes-per-second : 102400000
+max-rsync-parallel-num : 4
+write-binlog : no
+slotmigrate : no
+
+# Pika automatic compact compact strategy, a complement to rocksdb compact.
+# Trigger the compact background task periodically according to `compact-interval`
+# Can choose `full-compact` or `obd-compact`.
+# obd-compact https://github.com/OpenAtomFoundation/pika/issues/2255
+compaction-strategy : obd-compact
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+compact-every-num-of-files : 10
+
+# For OBD_Compact 
+# In another search, if the file creation time is 
+# greater than `force-compact-file-age-seconds`, 
+# a compaction of the upper and lower boundaries 
+# of the file will be performed at the same time 
+# `compact-every-num-of-files` -1
+force-compact-file-age-seconds : 300
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+force-compact-min-delete-ratio : 10
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+dont-compact-sst-created-in-seconds : 5
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+best-delete-min-ratio : 10

include/pika_conf.h

@@ -28,6 +28,12 @@ const uint32_t configReplicationIDSize = 50;
 
 // global class, class members well initialized
 class PikaConf : public pstd::BaseConf {
+ public:
+  enum CompactionStrategy {
+    FullCompact,
+    OldestOrBestDeleteRatioSstCompact
+  };
+
  public:
   PikaConf(const std::string& path);
   ~PikaConf() override = default;
@@ -114,6 +120,30 @@ class PikaConf : public pstd::BaseConf {
     std::shared_lock l(rwlock_);
     return max_subcompactions_;
   }
+  int compact_every_num_of_files() {
+    std::shared_lock l(rwlock_);
+    return compact_every_num_of_files_;
+  }
+  int force_compact_file_age_seconds() {
+    std::shared_lock l(rwlock_);
+    return force_compact_file_age_seconds_;
+  }
+  int force_compact_min_delete_ratio() {
+    std::shared_lock l(rwlock_);
+    return force_compact_min_delete_ratio_;
+  }
+  int dont_compact_sst_created_in_seconds() {
+    std::shared_lock l(rwlock_);
+    return dont_compact_sst_created_in_seconds_;
+  }
+  int best_delete_min_ratio() {
+    std::shared_lock l(rwlock_);
+    return best_delete_min_ratio_;
+  }
+  CompactionStrategy compaction_strategy() {
+    std::shared_lock l(rwlock_);
+    return compaction_strategy_;
+  }
   bool disable_auto_compactions() {
     std::shared_lock l(rwlock_);
     return disable_auto_compactions_;
@@ -914,6 +944,15 @@ class PikaConf : public pstd::BaseConf {
   std::string compact_interval_;
   int max_subcompactions_ = 1;
   bool disable_auto_compactions_ = false;
+
+  // for obd_compact
+  int compact_every_num_of_files_;
+  int force_compact_file_age_seconds_;
+  int force_compact_min_delete_ratio_;
+  int dont_compact_sst_created_in_seconds_;
+  int best_delete_min_ratio_;
+  CompactionStrategy compaction_strategy_;
+
   int64_t resume_check_interval_ = 60; // seconds
   int64_t least_free_disk_to_resume_ = 268435456; // 256 MB
   double min_check_resume_ratio_ = 0.7;

include/pika_db.h

@@ -125,6 +125,8 @@ class DB : public std::enable_shared_from_this<DB>, public pstd::noncopyable {
   // Compact use;
   void Compact(const storage::DataType& type);
   void CompactRange(const storage::DataType& type, const std::string& start, const std::string& end);
+  // void FullCompact();
+  void LongestNotCompactiontSstCompact(const storage::DataType& type);
 
   void SetCompactRangeOptions(const bool is_canceled);
 

include/pika_server.h

@@ -59,6 +59,7 @@ enum TaskType {
   kStopKeyScan,
   kBgSave,
   kCompactRangeAll,
+  kCompactOldestOrBestDeleteRatioSst,
 };
 
 struct TaskArg {

src/pika_conf.cc

@@ -301,6 +301,41 @@ int PikaConf::Load() {
     max_subcompactions_ = 1;
   }
 
+  GetConfInt("compact-every-num-of-files", &compact_every_num_of_files_);
+  if (compact_every_num_of_files_ < 10) {
+    compact_every_num_of_files_ = 10;
+  }
+
+  GetConfInt("force-compact-file-age-seconds", &force_compact_file_age_seconds_);
+  if (force_compact_file_age_seconds_ < 300) {
+    force_compact_file_age_seconds_ = 300;
+  }
+
+  GetConfInt("force-compact-min-delete-ratio", &force_compact_min_delete_ratio_);
+  if (force_compact_min_delete_ratio_ < 10) {
+    force_compact_min_delete_ratio_ = 10;
+  }
+
+  GetConfInt("dont-compact-sst-created-in-seconds", &dont_compact_sst_created_in_seconds_);
+  if (dont_compact_sst_created_in_seconds_ < 600) {
+    dont_compact_sst_created_in_seconds_ = 600;
+  }
+
+  GetConfInt("best-delete-min-ratio", &best_delete_min_ratio_);
+  if (best_delete_min_ratio_ < 10) {
+    best_delete_min_ratio_ = 10;
+  }
+
+  std::string cs_;
+  GetConfStr("compaction-strategy", &cs_);
+  if (cs_ == "full-compact") {
+    compaction_strategy_ = FullCompact;
+  } else if (cs_ == "obd-compact") {
+    compaction_strategy_ = OldestOrBestDeleteRatioSstCompact;
+  } else {
+    compaction_strategy_ = FullCompact;
+  }
+
   // least-free-disk-resume-size
   GetConfInt64Human("least-free-disk-resume-size", &least_free_disk_to_resume_);
   if (least_free_disk_to_resume_ <= 0) {
@@ -410,6 +445,11 @@ int PikaConf::Load() {
     max_write_buffer_num_ = 2;  // 1 for immutable memtable, 1 for mutable memtable
   }
 
+  GetConfInt("min-write-buffer-number-to-merge", &min_write_buffer_number_to_merge_);
+  if (min_write_buffer_number_to_merge_ < 1) {
+    min_write_buffer_number_to_merge_ = 1;  // 1 for immutable memtable to merge
+  }
+
   // max_client_response_size
   GetConfInt64Human("max-client-response-size", &max_client_response_size_);
   if (max_client_response_size_ <= 0) {
@@ -774,8 +814,40 @@ int PikaConf::ConfigRewrite() {
   SetConfInt("small-compaction-duration-threshold", small_compaction_duration_threshold_);
   SetConfInt("max-client-response-size", static_cast<int32_t>(max_client_response_size_));
   SetConfInt("db-sync-speed", db_sync_speed_);
+  // compact
   SetConfStr("compact-cron", compact_cron_);
   SetConfStr("compact-interval", compact_interval_);
+  SetConfInt("compact-every-num-of-files", compact_every_num_of_files_);
+  if (compact_every_num_of_files_ < 1) {
+    compact_every_num_of_files_ = 1;
+  }
+  SetConfInt("force-compact-file-age-seconds", force_compact_file_age_seconds_);
+  if (force_compact_file_age_seconds_ < 300) {
+    force_compact_file_age_seconds_ = 300;
+  }
+  SetConfInt("force-compact-min-delete-ratio", force_compact_min_delete_ratio_);
+  if (force_compact_min_delete_ratio_ < 5) {
+    force_compact_min_delete_ratio_ = 5;
+  }
+  SetConfInt("dont-compact-sst-created-in-seconds", dont_compact_sst_created_in_seconds_);
+  if (dont_compact_sst_created_in_seconds_ < 300) {
+    dont_compact_sst_created_in_seconds_ = 300;
+  }
+  SetConfInt("best-delete-min-ratio", best_delete_min_ratio_);
+  if (best_delete_min_ratio_ < 10) {
+    best_delete_min_ratio_ = 10;
+  }
+
+  std::string cs_;
+  SetConfStr("compaction-strategy", cs_);
+  if (cs_ == "full_compact") {
+    compaction_strategy_ = FullCompact;
+  } else if (cs_ == "obd_compact") {
+    compaction_strategy_ = OldestOrBestDeleteRatioSstCompact;
+  } else {
+    compaction_strategy_ = FullCompact;
+  }
+
   SetConfStr("disable_auto_compactions", disable_auto_compactions_ ? "true" : "false");
   SetConfStr("cache-type", scachetype);
   SetConfInt64("least-free-disk-resume-size", least_free_disk_to_resume_);

src/pika_db.cc

@@ -166,6 +166,22 @@ void DB::CompactRange(const storage::DataType& type, const std::string& start, c
   storage_->CompactRange(type, start, end);
 }
 
+// void DB::FullCompact() {
+//   std::lock_guard rwl(dbs_rw_);
+//   if (!opened_) {
+//     return;
+//   }
+//   storage_->FullCompact();
+// }
+
+void DB::LongestNotCompactiontSstCompact(const storage::DataType& type) {
+  std::lock_guard rwl(dbs_rw_);
+  if (!opened_) {
+    return;
+  }
+  storage_->LongestNotCompactiontSstCompact(type);
+}
+
 void DB::DoKeyScan(void* arg) {
   std::unique_ptr <BgTaskArg> bg_task_arg(static_cast<BgTaskArg*>(arg));
   bg_task_arg->db->RunKeyScan();

src/pika_server.cc

@@ -424,6 +424,9 @@ Status PikaServer::DoSameThingSpecificDB(const std::set<std::string>& dbs, const
       case TaskType::kCompactRangeAll:
         db_item.second->CompactRange(storage::DataType::kAll, arg.argv[0], arg.argv[1]);
         break;
+      case TaskType::kCompactOldestOrBestDeleteRatioSst:
+        db_item.second->LongestNotCompactiontSstCompact(storage::DataType::kAll);
+        break;
       default:
         break;
     }
@@ -1138,7 +1141,12 @@ void PikaServer::AutoCompactRange() {
       gettimeofday(&last_check_compact_time_, nullptr);
       if ((static_cast<double>(free_size) / static_cast<double>(total_size)) * 100 >= usage) {
         std::set<std::string> dbs = g_pika_server->GetAllDBName();
-        Status s = DoSameThingSpecificDB(dbs, {TaskType::kCompactAll});
+        Status s;
+        if (g_pika_conf->compaction_strategy() == PikaConf::FullCompact) {
+          s = DoSameThingSpecificDB(dbs, {TaskType::kCompactAll});
+        } else if (g_pika_conf->compaction_strategy() == PikaConf::OldestOrBestDeleteRatioSstCompact) {
+          s = DoSameThingSpecificDB(dbs, {TaskType::kCompactOldestOrBestDeleteRatioSst});
+        }
         if (s.ok()) {
           LOG(INFO) << "[Interval]schedule compactRange, freesize: " << free_size / 1048576
                     << "MB, disksize: " << total_size / 1048576 << "MB";
@@ -1333,6 +1341,12 @@ void PikaServer::InitStorageOptions() {
   storage_options_.options.max_bytes_for_level_base = g_pika_conf->level0_file_num_compaction_trigger() * g_pika_conf->write_buffer_size();
   storage_options_.options.max_subcompactions = g_pika_conf->max_subcompactions();
   storage_options_.options.target_file_size_base = g_pika_conf->target_file_size_base();
+  storage_options_.options.level0_file_num_compaction_trigger = g_pika_conf->level0_file_num_compaction_trigger();
+  storage_options_.options.level0_stop_writes_trigger = g_pika_conf->level0_stop_writes_trigger();
+  storage_options_.options.level0_slowdown_writes_trigger = g_pika_conf->level0_slowdown_writes_trigger();
+  storage_options_.options.min_write_buffer_number_to_merge = g_pika_conf->min_write_buffer_number_to_merge();
+  storage_options_.options.max_bytes_for_level_base = g_pika_conf->level0_file_num_compaction_trigger() * g_pika_conf->write_buffer_size();
+  storage_options_.options.max_subcompactions = g_pika_conf->max_subcompactions();
   storage_options_.options.max_compaction_bytes = g_pika_conf->max_compaction_bytes();
   storage_options_.options.max_background_flushes = g_pika_conf->max_background_flushes();
   storage_options_.options.max_background_compactions = g_pika_conf->max_background_compactions();
@@ -1386,6 +1400,13 @@ void PikaServer::InitStorageOptions() {
   storage_options_.statistics_max_size = g_pika_conf->max_cache_statistic_keys();
   storage_options_.small_compaction_threshold = g_pika_conf->small_compaction_threshold();
 
+  // For Storage compaction
+  storage_options_.compact_param_.best_delete_min_ratio_ = g_pika_conf->best_delete_min_ratio();
+  storage_options_.compact_param_.dont_compact_sst_created_in_seconds_ = g_pika_conf->dont_compact_sst_created_in_seconds();
+  storage_options_.compact_param_.force_compact_file_age_seconds_ = g_pika_conf->force_compact_file_age_seconds();
+  storage_options_.compact_param_.force_compact_min_delete_ratio_ = g_pika_conf->force_compact_min_delete_ratio();
+  storage_options_.compact_param_.compact_every_num_of_files_ = g_pika_conf->compact_every_num_of_files();
+
   // rocksdb blob
   if (g_pika_conf->enable_blob_files()) {
     storage_options_.options.enable_blob_files = g_pika_conf->enable_blob_files();