fix(meta): limit max orm param (#19617)

src/common/src/config.rs

@@ -1433,15 +1433,15 @@ pub mod default {
         use crate::config::{DefaultParallelism, MetaBackend};
 
         pub fn min_sst_retention_time_sec() -> u64 {
-            3600 * 3
+            3600 * 6
         }
 
         pub fn gc_history_retention_time_sec() -> u64 {
             3600 * 6
         }
 
         pub fn full_gc_interval_sec() -> u64 {
-            600
+            3600
         }
 
         pub fn full_gc_object_limit() -> u64 {

src/config/docs.md

@@ -37,7 +37,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | enable_hummock_data_archive | If enabled, `SSTable` object file and version delta will be retained. `SSTable` object file need to be deleted via full GC. version delta need to be manually deleted. | false |
 | event_log_channel_max_size | Keeps the latest N events per channel. | 10 |
 | event_log_enabled |  | true |
-| full_gc_interval_sec | Interval of automatic hummock full GC. | 600 |
+| full_gc_interval_sec | Interval of automatic hummock full GC. | 3600 |
 | full_gc_object_limit | Max number of object per full GC job can fetch. | 100000 |
 | gc_history_retention_time_sec | Duration in seconds to retain garbage collection history data. | 21600 |
 | hummock_time_travel_snapshot_interval | The interval at which a Hummock version snapshot is taken for time travel. Larger value indicates less storage overhead but worse query performance. | 100 |
@@ -47,7 +47,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | max_inflight_time_travel_query | Max number of inflight time travel query. | 1000 |
 | meta_leader_lease_secs |  | 30 |
 | min_delta_log_num_for_hummock_version_checkpoint | The minimum delta log number a new checkpoint should compact, otherwise the checkpoint attempt is rejected. | 10 |
-| min_sst_retention_time_sec | Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they are dangling. | 10800 |
+| min_sst_retention_time_sec | Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they are dangling. | 21600 |
 | move_table_size_limit |  | 10737418240 |
 | node_num_monitor_interval_sec |  | 10 |
 | parallelism_control_batch_size | The number of streaming jobs per scaling operation. | 10 |

src/config/example.toml

@@ -14,8 +14,8 @@ threshold_auto = 0.8999999761581421
 dir = "./"
 
 [meta]
-min_sst_retention_time_sec = 10800
-full_gc_interval_sec = 600
+min_sst_retention_time_sec = 21600
+full_gc_interval_sec = 3600
 full_gc_object_limit = 100000
 gc_history_retention_time_sec = 21600
 max_inflight_time_travel_query = 1000

src/meta/src/hummock/manager/gc.rs

@@ -443,7 +443,7 @@ impl HummockManager {
         }
         let now = self.now().await?;
         let dt = DateTime::from_timestamp(now.try_into().unwrap(), 0).unwrap();
-        let models = object_ids.map(|o| hummock_gc_history::ActiveModel {
+        let mut models = object_ids.map(|o| hummock_gc_history::ActiveModel {
             object_id: Set(o.try_into().unwrap()),
             mark_delete_at: Set(dt.naive_utc()),
         });
@@ -459,10 +459,27 @@ impl HummockManager {
             .filter(hummock_gc_history::Column::MarkDeleteAt.lt(gc_history_low_watermark))
             .exec(db)
             .await?;
-        hummock_gc_history::Entity::insert_many(models)
-            .on_conflict_do_nothing()
-            .exec(db)
-            .await?;
+        const BATCH_SIZE: usize = 1000;
+        let mut is_finished = false;
+        while !is_finished {
+            let mut batch = vec![];
+            let mut count: usize = BATCH_SIZE;
+            while count > 0 {
+                let Some(m) = models.next() else {
+                    is_finished = true;
+                    break;
+                };
+                count -= 1;
+                batch.push(m);
+            }
+            if batch.is_empty() {
+                break;
+            }
+            hummock_gc_history::Entity::insert_many(batch)
+                .on_conflict_do_nothing()
+                .exec(db)
+                .await?;
+        }
         Ok(())
     }
 
@@ -518,7 +535,7 @@ impl HummockManager {
                 break;
             }
             let delete_batch: HashSet<_> = objects_to_delete.drain(..batch_size).collect();
-            tracing::debug!(?objects_to_delete, "Attempt to delete objects.");
+            tracing::debug!(?delete_batch, "Attempt to delete objects.");
             let deleted_object_ids = delete_batch.clone();
             self.gc_manager
                 .delete_objects(delete_batch.into_iter())