refactor(storage): distinguish SST id and object id (close risingwave…

…labs#8434) (risingwavelabs#8436)
Little-Wallace · Mar 14, 2023 · 9d5ff78 · 9d5ff78
1 parent 6d28cf6
commit 9d5ff78
Show file tree

Hide file tree

Showing 69 changed files with 1,440 additions and 1,165 deletions.
diff --git a/dashboard/proto/gen/hummock.ts b/dashboard/proto/gen/hummock.ts
diff --git a/proto/hummock.proto b/proto/hummock.proto
@@ -10,15 +10,14 @@ option java_package = "com.risingwave.proto";
 option optimize_for = SPEED;
 
 message SstableInfo {
-  uint64 id = 1;
-  KeyRange key_range = 2;
-  uint64 file_size = 3;
-  repeated uint32 table_ids = 4;
-  uint64 meta_offset = 5;
-  uint64 stale_key_count = 6;
-  uint64 total_key_count = 7;
-  // When a SST is divided, its divide_version will increase one.
-  uint64 divide_version = 8;
+  uint64 object_id = 1;
+  uint64 sst_id = 2;
+  KeyRange key_range = 3;
+  uint64 file_size = 4;
+  repeated uint32 table_ids = 5;
+  uint64 meta_offset = 6;
+  uint64 stale_key_count = 7;
+  uint64 total_key_count = 8;
   uint64 min_epoch = 9;
   uint64 max_epoch = 10;
   uint64 uncompressed_file_size = 11;
@@ -64,6 +63,7 @@ message GroupConstruct {
   uint64 parent_group_id = 2;
   repeated uint32 table_ids = 3;
   uint64 group_id = 4;
+  uint64 new_sst_start_id = 5;
 }
 
 message GroupMetaChange {
@@ -117,7 +117,7 @@ message HummockVersionDelta {
   // Reads against such an epoch will fail.
   uint64 safe_epoch = 5;
   bool trivial_move = 6;
-  repeated uint64 gc_sst_ids = 7;
+  repeated uint64 gc_object_ids = 7;
 }
 
 message HummockVersionDeltas {
@@ -232,7 +232,7 @@ message CompactTask {
     INVALID_GROUP_CANCELED = 8;
     EXECUTE_FAILED = 9;
     JOIN_HANDLE_FAILED = 10;
-    TRACK_SST_ID_FAILED = 11;
+    TRACK_SST_OBJECT_ID_FAILED = 11;
   }
   // SSTs to be compacted, which will be removed from LSM after compaction
   repeated InputLevel input_ssts = 1;
@@ -390,7 +390,7 @@ message SubscribeCompactTasksResponse {
 
 // Delete SSTs in object store
 message VacuumTask {
-  repeated uint64 sstable_ids = 1;
+  repeated uint64 sstable_object_ids = 1;
 }
 
 // Scan object store to get candidate orphan SSTs.
@@ -425,7 +425,7 @@ message TriggerManualCompactionResponse {
 }
 
 message ReportFullScanTaskRequest {
-  repeated uint64 sst_ids = 1;
+  repeated uint64 object_ids = 1;
 }
 
 message ReportFullScanTaskResponse {

diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs
@@ -179,7 +179,7 @@ pub async fn compute_node_serve(
 
     let mut extra_info_sources: Vec<ExtraInfoSourceRef> = vec![];
     if let Some(storage) = state_store.as_hummock_trait() {
-        extra_info_sources.push(storage.sstable_id_manager().clone());
+        extra_info_sources.push(storage.sstable_object_id_manager().clone());
         if embedded_compactor_enabled {
             tracing::info!("start embedded compactor");
             let read_memory_limiter = Arc::new(MemoryLimiter::new(
@@ -194,7 +194,7 @@ pub async fn compute_node_serve(
                 compaction_executor: Arc::new(CompactionExecutor::new(Some(1))),
                 filter_key_extractor_manager: storage.filter_key_extractor_manager().clone(),
                 read_memory_limiter,
-                sstable_id_manager: storage.sstable_id_manager().clone(),
+                sstable_object_id_manager: storage.sstable_object_id_manager().clone(),
                 task_progress_manager: Default::default(),
                 compactor_runtime_config: Arc::new(tokio::sync::Mutex::new(
                     CompactorRuntimeConfig {