use sandbox folder for txhashset validation on state sync

chain/src/chain.rs

@@ -36,7 +36,9 @@ use crate::util::secp::pedersen::{Commitment, RangeProof};
 use crate::util::{Mutex, RwLock, StopState};
 use grin_store::Error::NotFoundErr;
 use std::collections::HashMap;
+use std::env;
 use std::fs::File;
+use std::path::PathBuf;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -180,7 +182,8 @@ impl Chain {
 			let store = Arc::new(store::ChainStore::new(db_env)?);
 
 			// open the txhashset, creating a new one if necessary
-			let mut txhashset = txhashset::TxHashSet::open(db_root.clone(), store.clone(), None)?;
+			let mut txhashset =
+				txhashset::TxHashSet::open(db_root.clone(), store.clone(), None, None)?;
 
 			setup_head(&genesis, &store, &mut txhashset)?;
 			Chain::log_heads(&store)?;
@@ -842,6 +845,11 @@ impl Chain {
 		}
 	}
 
+	/// Clean the temporary sandbox folder
+	pub fn clean_txhashset_sandbox(&self) {
+		txhashset::clean_txhashset_folder(&env::temp_dir());
+	}
+
 	/// Writes a reading view on a txhashset state that's been provided to us.
 	/// If we're willing to accept that new state, the data stream will be
 	/// read as a zip file, unzipped and the resulting state files should be
@@ -863,17 +871,21 @@ impl Chain {
 
 		let header = self.get_block_header(&h)?;
 
-		{
-			let mut txhashset_ref = self.txhashset.write();
-			// Drop file handles in underlying txhashset
-			txhashset_ref.release_backend_files();
-		}
-
-		// Rewrite hashset
-		txhashset::zip_write(self.db_root.clone(), txhashset_data, &header)?;
-
-		let mut txhashset =
-			txhashset::TxHashSet::open(self.db_root.clone(), self.store.clone(), Some(&header))?;
+		// Write txhashset to sandbox
+		txhashset::clean_txhashset_folder(&env::temp_dir());
+		txhashset::zip_write(env::temp_dir(), txhashset_data.try_clone()?, &header)?;
+
+		let mut txhashset = txhashset::TxHashSet::open(
+			self.db_root.clone(),
+			self.store.clone(),
+			Some(&header),
+			Some(
+				env::temp_dir()
+					.to_str()
+					.expect("invalid temp folder")
+					.to_owned(),
+			),
+		)?;
 
 		// The txhashset.zip contains the output, rangeproof and kernel MMRs.
 		// We must rebuild the header MMR ourselves based on the headers in our db.
@@ -925,6 +937,24 @@ impl Chain {
 
 		debug!("txhashset_write: finished committing the batch (head etc.)");
 
+		// sandbox full validation ok, go to overwrite txhashset on db root
+		{
+			// Before overwriting, drop file handles in underlying txhashset
+			{
+				let mut txhashset_ref = self.txhashset.write();
+				txhashset_ref.release_backend_files();
+			}
+
+			txhashset::txhashset_replace(env::temp_dir(), PathBuf::from(self.db_root.clone()))?;
+
+			txhashset = txhashset::TxHashSet::open(
+				self.db_root.clone(),
+				self.store.clone(),
+				Some(&header),
+				None,
+			)?;
+		}
+
 		// Replace the chain txhashset with the newly built one.
 		{
 			let mut txhashset_ref = self.txhashset.write();

chain/src/txhashset/txhashset.rs

@@ -114,7 +114,13 @@ impl TxHashSet {
 		root_dir: String,
 		commit_index: Arc<ChainStore>,
 		header: Option<&BlockHeader>,
+		sandbox_dir: Option<String>,
 	) -> Result<TxHashSet, Error> {
+		let txhashset_rootdir = if let Some(sandbox) = sandbox_dir {
+			sandbox.clone()
+		} else {
+			root_dir.clone()
+		};
 		Ok(TxHashSet {
 			header_pmmr_h: PMMRHandle::new(
 				&root_dir,
@@ -131,21 +137,21 @@ impl TxHashSet {
 				None,
 			)?,
 			output_pmmr_h: PMMRHandle::new(
-				&root_dir,
+				&txhashset_rootdir,
 				TXHASHSET_SUBDIR,
 				OUTPUT_SUBDIR,
 				true,
 				header,
 			)?,
 			rproof_pmmr_h: PMMRHandle::new(
-				&root_dir,
+				&txhashset_rootdir,
 				TXHASHSET_SUBDIR,
 				RANGE_PROOF_SUBDIR,
 				true,
 				header,
 			)?,
 			kernel_pmmr_h: PMMRHandle::new(
-				&root_dir,
+				&txhashset_rootdir,
 				TXHASHSET_SUBDIR,
 				KERNEL_SUBDIR,
 				false,
@@ -1450,17 +1456,49 @@ pub fn zip_read(root_dir: String, header: &BlockHeader, rand: Option<u32>) -> Re
 /// Extract the txhashset data from a zip file and writes the content into the
 /// txhashset storage dir
 pub fn zip_write(
-	root_dir: String,
+	root_dir: PathBuf,
 	txhashset_data: File,
 	header: &BlockHeader,
 ) -> Result<(), Error> {
-	let txhashset_path = Path::new(&root_dir).join(TXHASHSET_SUBDIR);
+	debug!("zip_write on path: {:?}", root_dir);
+	let txhashset_path = root_dir.clone().join(TXHASHSET_SUBDIR);
 	fs::create_dir_all(txhashset_path.clone())?;
 	zip::decompress(txhashset_data, &txhashset_path, expected_file)
 		.map_err(|ze| ErrorKind::Other(ze.to_string()))?;
 	check_and_remove_files(&txhashset_path, header)
 }
 
+/// Rename a folder to another
+pub fn txhashset_replace(from: PathBuf, to: PathBuf) -> Result<(), Error> {
+	debug!("txhashset_replace: move from {:?} to {:?}", from, to);
+
+	// clean the 'to' folder firstly
+	clean_txhashset_folder(&to);
+
+	// rename the 'from' folder as the 'to' folder
+	let txhashset_from_path = from.clone().join(TXHASHSET_SUBDIR);
+	let txhashset_to_path = to.clone().join(TXHASHSET_SUBDIR);
+	if let Err(e) = fs::rename(txhashset_from_path, txhashset_to_path) {
+		error!("txhashset_replace fail. err: {}", e);
+		Err(ErrorKind::TxHashSetErr(format!("txhashset replacing fail")).into())
+	} else {
+		Ok(())
+	}
+}
+
+/// Clean the txhashset folder
+pub fn clean_txhashset_folder(root_dir: &PathBuf) {
+	let txhashset_path = root_dir.clone().join(TXHASHSET_SUBDIR);
+	if txhashset_path.exists() {
+		if let Err(e) = fs::remove_dir_all(txhashset_path.clone()) {
+			warn!(
+				"clean_txhashset_folder: fail on {:?}. err: {}",
+				txhashset_path, e
+			);
+		}
+	}
+}
+
 fn expected_file(path: &Path) -> bool {
 	use lazy_static::lazy_static;
 	use regex::Regex;

chain/tests/test_txhashset.rs

@@ -44,7 +44,7 @@ fn test_unexpected_zip() {
 	let db_env = Arc::new(store::new_env(db_root.clone()));
 	let chain_store = ChainStore::new(db_env).unwrap();
 	let store = Arc::new(chain_store);
-	txhashset::TxHashSet::open(db_root.clone(), store.clone(), None).unwrap();
+	txhashset::TxHashSet::open(db_root.clone(), store.clone(), None, None).unwrap();
 	// First check if everything works out of the box
 	assert!(txhashset::zip_read(db_root.clone(), &BlockHeader::default(), Some(rand)).is_ok());
 	let zip_path = Path::new(&db_root).join(format!("txhashset_snapshot_{}.zip", rand));

servers/src/common/adapters.rs

@@ -330,17 +330,16 @@ impl p2p::ChainAdapter for NetToChainAdapter {
 				update_time: old_update_time,
 				downloaded_size: old_downloaded_size,
 				..
-			} => {
-				self.sync_state
-					.update_txhashset_download(SyncStatus::TxHashsetDownload {
-						start_time,
-						prev_update_time: old_update_time,
-						update_time: Utc::now(),
-						prev_downloaded_size: old_downloaded_size,
-						downloaded_size,
-						total_size,
-					})
-			}
+			} => self
+				.sync_state
+				.update_txhashset_download(SyncStatus::TxHashsetDownload {
+					start_time,
+					prev_update_time: old_update_time,
+					update_time: Utc::now(),
+					prev_downloaded_size: old_downloaded_size,
+					downloaded_size,
+					total_size,
+				}),
 			_ => false,
 		}
 	}
@@ -360,6 +359,7 @@ impl p2p::ChainAdapter for NetToChainAdapter {
 			.chain()
 			.txhashset_write(h, txhashset_data, self.sync_state.as_ref())
 		{
+			self.chain().clean_txhashset_sandbox();
 			error!("Failed to save txhashset archive: {}", e);
 			let is_good_data = !e.is_bad_data();
 			self.sync_state.set_sync_error(types::Error::Chain(e));