Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(storage): distinguish SST id and object id (close #8434) #8436

Merged
merged 11 commits into from
Mar 14, 2023
Merged

refactor(storage): distinguish SST id and object id (close #8434) #8436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
e0b5ba1
refactor(storage): distinguish SST id and object id (close #8434)
Mar 11, 2023
431d435
rename
Mar 9, 2023
45f807e
gen proto
Mar 9, 2023
040f828
add doc
Mar 11, 2023
0e56713
apply #8484
Mar 11, 2023
d0a838f
same SST in one group
Mar 11, 2023
931a20a
unit test
Mar 11, 2023
dbcd0be
address comments
Mar 14, 2023
7f718b4
address comments
Mar 14, 2023
ab9c4e6
refactor `HummockSstableId`
Mar 14, 2023
ee1d3d6
license header check fails
Mar 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 46 additions & 39 deletions dashboard/proto/gen/hummock.ts
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

26 changes: 13 additions & 13 deletions proto/hummock.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,14 @@ option java_package = "com.risingwave.proto";
option optimize_for = SPEED;

message SstableInfo {
uint64 id = 1;
KeyRange key_range = 2;
uint64 file_size = 3;
repeated uint32 table_ids = 4;
uint64 meta_offset = 5;
uint64 stale_key_count = 6;
uint64 total_key_count = 7;
// When a SST is divided, its divide_version will increase one.
uint64 divide_version = 8;
uint64 object_id = 1;
uint64 sst_id = 2;
KeyRange key_range = 3;
uint64 file_size = 4;
repeated uint32 table_ids = 5;
uint64 meta_offset = 6;
uint64 stale_key_count = 7;
uint64 total_key_count = 8;
uint64 min_epoch = 9;
uint64 max_epoch = 10;
uint64 uncompressed_file_size = 11;
Expand Down Expand Up @@ -64,6 +63,7 @@ message GroupConstruct {
uint64 parent_group_id = 2;
repeated uint32 table_ids = 3;
uint64 group_id = 4;
uint64 new_sst_start_id = 5;
}

message GroupMetaChange {
Expand Down Expand Up @@ -117,7 +117,7 @@ message HummockVersionDelta {
// Reads against such an epoch will fail.
uint64 safe_epoch = 5;
bool trivial_move = 6;
repeated uint64 gc_sst_ids = 7;
repeated uint64 gc_object_ids = 7;
}

message HummockVersionDeltas {
Expand Down Expand Up @@ -232,7 +232,7 @@ message CompactTask {
INVALID_GROUP_CANCELED = 8;
EXECUTE_FAILED = 9;
JOIN_HANDLE_FAILED = 10;
TRACK_SST_ID_FAILED = 11;
TRACK_SST_OBJECT_ID_FAILED = 11;
}
// SSTs to be compacted, which will be removed from LSM after compaction
repeated InputLevel input_ssts = 1;
Expand Down Expand Up @@ -390,7 +390,7 @@ message SubscribeCompactTasksResponse {

// Delete SSTs in object store
message VacuumTask {
repeated uint64 sstable_ids = 1;
repeated uint64 sstable_object_ids = 1;
}

// Scan object store to get candidate orphan SSTs.
Expand Down Expand Up @@ -425,7 +425,7 @@ message TriggerManualCompactionResponse {
}

message ReportFullScanTaskRequest {
repeated uint64 sst_ids = 1;
repeated uint64 object_ids = 1;
}

message ReportFullScanTaskResponse {
Expand Down
4 changes: 2 additions & 2 deletions src/compute/src/server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ pub async fn compute_node_serve(

let mut extra_info_sources: Vec<ExtraInfoSourceRef> = vec![];
if let Some(storage) = state_store.as_hummock_trait() {
extra_info_sources.push(storage.sstable_id_manager().clone());
extra_info_sources.push(storage.sstable_object_id_manager().clone());
if embedded_compactor_enabled {
tracing::info!("start embedded compactor");
let read_memory_limiter = Arc::new(MemoryLimiter::new(
Expand All @@ -194,7 +194,7 @@ pub async fn compute_node_serve(
compaction_executor: Arc::new(CompactionExecutor::new(Some(1))),
filter_key_extractor_manager: storage.filter_key_extractor_manager().clone(),
read_memory_limiter,
sstable_id_manager: storage.sstable_id_manager().clone(),
sstable_object_id_manager: storage.sstable_object_id_manager().clone(),
task_progress_manager: Default::default(),
compactor_runtime_config: Arc::new(tokio::sync::Mutex::new(
CompactorRuntimeConfig {
Expand Down
Loading