Skip to content

Commit

Permalink
Change two constants ledger backup to config and add pruning block st…
Browse files Browse the repository at this point in the history 
…oring
  • Loading branch information
@AurelienFT
AurelienFT committed May 29, 2024
1 parent 3ca8e45 commit 2a0869c
Show file tree
Hide file tree
Showing 27 changed files with 158 additions and 55 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,12 @@
**/target
massa-client/config/*
massa-client/wallet.dat
massa-execution-worker/test_db/*
massa-execution-worker/**.bin
massa-node/config/*
massa-node/logs.txt
massa-node/storage/*
massa-node/dump/*
massa-node/storage/peers.json
massa-node/src/tests/storage
shell.nix
Expand Down
6 changes: 6 additions & 0 deletions massa-async-pool/src/pool.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1099,6 +1099,7 @@ mod tests {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 100,
};
let db: ShareableMassaDBController = Arc::new(RwLock::new(
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
Expand Down Expand Up @@ -1140,6 +1141,7 @@ mod tests {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 100,
};
let db: ShareableMassaDBController = Arc::new(RwLock::new(
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
Expand Down Expand Up @@ -1201,6 +1203,7 @@ mod tests {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 100,
};
let db: ShareableMassaDBController = Arc::new(RwLock::new(
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
Expand Down Expand Up @@ -1257,6 +1260,7 @@ mod tests {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 100,
};
let db: ShareableMassaDBController = Arc::new(RwLock::new(
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
Expand Down Expand Up @@ -1313,6 +1317,7 @@ mod tests {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 100,
};
let db: ShareableMassaDBController = Arc::new(RwLock::new(
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
Expand Down Expand Up @@ -1362,6 +1367,7 @@ mod tests {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 100,
};
let db: ShareableMassaDBController = Arc::new(RwLock::new(Box::new(MassaDB::new(
db_config.clone(),
Expand Down
3 changes: 3 additions & 0 deletions massa-bootstrap/src/tests/binders.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,7 @@ fn test_staying_connected_without_message_trigger_read_timeout() {
max_final_state_elements_size: 100_000_000,
max_versioning_elements_size: 100_000_000,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
}))
as Box<(dyn MassaDBController + 'static)>));
let rolls_path = PathBuf::from_str("../massa-node/base_config/initial_rolls.json").unwrap();
Expand Down Expand Up @@ -547,6 +548,7 @@ fn test_staying_connected_pass_handshake_but_deadline_after() {
max_final_state_elements_size: 100_000_000,
max_versioning_elements_size: 100_000_000,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
}))
as Box<(dyn MassaDBController + 'static)>));
let rolls_path = PathBuf::from_str("../massa-node/base_config/initial_rolls.json").unwrap();
Expand Down Expand Up @@ -644,6 +646,7 @@ fn test_staying_connected_pass_handshake_but_deadline_during_data_exchange() {
max_final_state_elements_size: 100_000_000,
max_versioning_elements_size: 100_000_000,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
}))
as Box<(dyn MassaDBController + 'static)>));
let rolls_path = PathBuf::from_str("../massa-node/base_config/initial_rolls.json").unwrap();
Expand Down
1 change: 1 addition & 0 deletions massa-bootstrap/src/tests/universe_client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ impl TestUniverse for BootstrapClientTestUniverse {
max_versioning_elements_size: MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE as usize,
max_final_state_elements_size: MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE as usize,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
}))
as Box<(dyn MassaDBController + 'static)>));
controllers
Expand Down
1 change: 1 addition & 0 deletions massa-bootstrap/src/tests/universe_server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ impl BootstrapServerForeignControllers {
max_versioning_elements_size: MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE as usize,
max_final_state_elements_size: MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE as usize,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
}))
as Box<(dyn MassaDBController + 'static)>));
Self {
Expand Down
2 changes: 2 additions & 0 deletions massa-db-exports/src/settings.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,6 @@ pub struct MassaDBConfig {
pub max_final_state_elements_size: usize,
/// Thread count for slot serialization
pub thread_count: u8,
/// Maximum number of ledger backups to keep
pub max_ledger_backups: u64,
}
69 changes: 39 additions & 30 deletions massa-db-worker/src/massa_db.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ use massa_db_exports::{
};
use massa_hash::{HashXof, HASH_XOF_SIZE_BYTES};
use massa_models::{
config::MAX_BACKUPS_TO_KEEP,
error::ModelsError,
slot::{Slot, SlotDeserializer, SlotSerializer},
streaming_step::StreamingStep,
Expand Down Expand Up @@ -636,38 +635,35 @@ impl MassaDBController for RawMassaDB<Slot, SlotSerializer, SlotDeserializer> {
let db = &self.db;
let subpath = format!("backup_{}_{}", slot.period, slot.thread);

if let Some(max_backups) = MAX_BACKUPS_TO_KEEP {
let previous_backups_paths = std::fs::read_dir(db.path())
.expect("Cannot walk db path")
.map(|res| res.map(|e| e.path()))
.collect::<Result<Vec<_>, std::io::Error>>()
.expect("Cannot walk db path");
let previous_backups_paths = std::fs::read_dir(db.path())
.expect("Cannot walk db path")
.map(|res| res.map(|e| e.path()))
.collect::<Result<Vec<_>, std::io::Error>>()
.expect("Cannot walk db path");

let mut previous_backups = BTreeMap::new();
let mut previous_backups = BTreeMap::new();

for backup_path in previous_backups_paths.iter() {
let Some(path_str) = backup_path.file_name().and_then(|f| f.to_str()) else {
for backup_path in previous_backups_paths.iter() {
let Some(path_str) = backup_path.file_name().and_then(|f| f.to_str()) else {
continue;
};
let vec = path_str.split('_').collect::<Vec<&str>>();
if vec.len() == 3 && vec[0] == "backup" {
let Ok(period) = vec[1].parse::<u64>() else {
continue;
};
let vec = path_str.split('_').collect::<Vec<&str>>();
if vec.len() == 3 && vec[0] == "backup" {
let Ok(period) = vec[1].parse::<u64>() else {
continue;
};
let Ok(thread) = vec[2].parse::<u8>() else {
continue;
};
let backup_slot = Slot::new(period, thread);
previous_backups.insert(backup_slot, backup_path);
}
let Ok(thread) = vec[2].parse::<u8>() else {
continue;
};
let backup_slot = Slot::new(period, thread);
previous_backups.insert(backup_slot, backup_path);
}
}

// Remove the oldest backups if we have too many
while previous_backups.len() >= max_backups {
if let Some((_, oldest_backup_path)) = previous_backups.pop_first() {
std::fs::remove_dir_all(oldest_backup_path)
.expect("Cannot remove oldest backup");
}
// Remove the oldest backups if we have too many
while previous_backups.len() >= self.config.max_ledger_backups as usize {
if let Some((_, oldest_backup_path)) = previous_backups.pop_first() {
std::fs::remove_dir_all(oldest_backup_path).expect("Cannot remove oldest backup");
}
}

Expand Down Expand Up @@ -928,6 +924,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
Expand Down Expand Up @@ -957,6 +954,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
Expand Down Expand Up @@ -1040,6 +1038,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
Expand Down Expand Up @@ -1124,6 +1123,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
Expand Down Expand Up @@ -1172,6 +1172,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_backup_1_opts = MassaDB::default_db_opts();
db_backup_1_opts.create_if_missing(false);
Expand All @@ -1195,6 +1196,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_backup_2_opts = MassaDB::default_db_opts();
db_backup_2_opts.create_if_missing(false);
Expand Down Expand Up @@ -1227,19 +1229,20 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
db_opts.set_paranoid_checks(true);

let db = Arc::new(RwLock::new(Box::new(
MassaDB::new_with_options(db_config, db_opts.clone()).unwrap(),
MassaDB::new_with_options(db_config.clone(), db_opts.clone()).unwrap(),
)
as Box<(dyn MassaDBController + 'static)>));

let mut backups = BTreeMap::default();

for i in 0..(MAX_BACKUPS_TO_KEEP.unwrap() + 1) {
for i in 0..(db_config.max_ledger_backups + 1) {
let slot = Slot::new(i as u64, 0);

Check failure on line 1246 in massa-db-worker/src/massa_db.rs

View workflow job for this annotation

GitHub Actions / clippy

casting to the same type is unnecessary (`u64` -> `u64`) 

error: casting to the same type is unnecessary (`u64` -> `u64`)
    --> massa-db-worker/src/massa_db.rs:1246:34
     |
1246 |             let slot = Slot::new(i as u64, 0);
     |                                  ^^^^^^^^ help: try: `i`
     |
     = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_cast
     = note: `-D clippy::unnecessary-cast` implied by `-D warnings`
     = help: to override `-D warnings` add `#[allow(clippy::unnecessary_cast)]`
let i_ = i as u8;
let batch = DBBatch::from([(vec![i_], Some(vec![i_ + 10]))]);
Expand All @@ -1258,7 +1261,7 @@ mod test {
let mut db_opts_no_create = db_opts.clone();
db_opts_no_create.create_if_missing(false);

for i in 0..(MAX_BACKUPS_TO_KEEP.unwrap() + 1) {
for i in 0..(db_config.max_ledger_backups + 1) {
let slot = Slot::new(i as u64, 0);

Check failure on line 1265 in massa-db-worker/src/massa_db.rs

View workflow job for this annotation

GitHub Actions / clippy

casting to the same type is unnecessary (`u64` -> `u64`) 

error: casting to the same type is unnecessary (`u64` -> `u64`)
    --> massa-db-worker/src/massa_db.rs:1265:34
     |
1265 |             let slot = Slot::new(i as u64, 0);
     |                                  ^^^^^^^^ help: try: `i`
     |
     = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_cast
let (backup_xof, backup_path) = backups.get(&slot).unwrap();

Expand All @@ -1273,6 +1276,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
// let db_backup_2_opts = MassaDB::default_db_opts();

Expand Down Expand Up @@ -1321,6 +1325,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
Expand Down Expand Up @@ -1413,6 +1418,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
Expand Down Expand Up @@ -1500,6 +1506,7 @@ mod test {
max_final_state_elements_size: 10,
max_versioning_elements_size: 10,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
Expand Down Expand Up @@ -1584,6 +1591,7 @@ mod test {
max_final_state_elements_size: 20,
max_versioning_elements_size: 20,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let mut db_opts = MassaDB::default_db_opts();
// Additional checks (only for testing)
Expand Down Expand Up @@ -1695,6 +1703,7 @@ mod test {
max_final_state_elements_size: 20,
max_versioning_elements_size: 20,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};

let slot_1 = Slot::new(1, 0);
Expand Down
1 change: 1 addition & 0 deletions massa-executed-ops/src/executed_denunciations.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ mod test {
max_history_length: 100,
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
max_ledger_backups: 10,
thread_count: THREAD_COUNT,
};
let db = Arc::new(RwLock::new(
Expand Down
3 changes: 3 additions & 0 deletions massa-executed-ops/src/executed_ops.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};
let db = Arc::new(RwLock::new(
Box::new(MassaDB::new(db_config.clone())) as Box<(dyn MassaDBController + 'static)>
Expand Down Expand Up @@ -360,13 +361,15 @@ mod test {
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count,
max_ledger_backups: 10,
};
let db_c_config = MassaDBConfig {
path: tempdir_c.path().to_path_buf(),
max_history_length: 10,
max_final_state_elements_size: 100,
max_versioning_elements_size: 100,
thread_count,
max_ledger_backups: 10,
};

let db_a = Arc::new(RwLock::new(
Expand Down
1 change: 1 addition & 0 deletions massa-execution-exports/src/types.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ pub struct ExecutionOutput {
pub slot_trace: Option<(SlotAbiCallStack, Vec<Transfer>)>,
/// storage
#[cfg(feature = "dump-block")]
#[serde(skip_serializing)]
pub storage: Option<Storage>,
/// Deferred credits execution (empty if execution-info feature is NOT enabled)
pub deferred_credits_execution: Vec<(Address, Result<Amount, String>)>,
Expand Down
1 change: 1 addition & 0 deletions massa-execution-worker/src/interface_impl.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ impl InterfaceImpl {
max_final_state_elements_size: 100_000,
max_versioning_elements_size: 100_000,
thread_count: THREAD_COUNT,
max_ledger_backups: 10,
};

let db = Arc::new(RwLock::new(
Expand Down
Loading

0 comments on commit 2a0869c

Please sign in to comment.