Skip to content

Commit

Permalink
statedb: allow longer state pruning history (paritytech#11980)
Browse files Browse the repository at this point in the history 
* introduce DbBackedQueue for the state pruning window

Signed-off-by: linning <linningde25@gmail.com>

* avoid cloning for next_hash

Signed-off-by: linning <linningde25@gmail.com>

* add tests

Signed-off-by: linning <linningde25@gmail.com>

* make clippy happy

Signed-off-by: linning <linningde25@gmail.com>

* impl have_block by checking block number

Signed-off-by: linning <linningde25@gmail.com>

* refactor

Signed-off-by: linning <linningde25@gmail.com>

* fix tests & add test for init db-backed queue

Signed-off-by: linning <linningde25@gmail.com>

* update comment

Signed-off-by: linning <linningde25@gmail.com>

* add check for have_state_at

Signed-off-by: linning <linningde25@gmail.com>

* address comment

Signed-off-by: linning <linningde25@gmail.com>

* renanme unload_blocks to uncached_blocks

Signed-off-by: linning <linningde25@gmail.com>

* address comment

Signed-off-by: linning <linningde25@gmail.com>

* fix syncs_state test

Signed-off-by: linning <linningde25@gmail.com>

* address comment

Signed-off-by: linning <linningde25@gmail.com>

* revert change to make_test_db to add test cases

Signed-off-by: linning <linningde25@gmail.com>

* do not prune unavailable block & add tests

Signed-off-by: linning <linningde25@gmail.com>

* Update client/state-db/src/lib.rs

Signed-off-by: linning <linningde25@gmail.com>

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* Update client/state-db/src/pruning.rs

Signed-off-by: linning <linningde25@gmail.com>

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* address comment

Signed-off-by: linning <linningde25@gmail.com>

Signed-off-by: linning <linningde25@gmail.com>
Co-authored-by: cheme <emericchevalier.pro@gmail.com>
  • Loading branch information
@cheme @ark0f
2 people authored and ark0f committed Feb 27, 2023
1 parent 68953cc commit ad55a2a
Show file tree
Hide file tree
Showing 5 changed files with 1,051 additions and 318 deletions.
67 changes: 43 additions & 24 deletions client/db/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ use sc_client_api::{
utils::is_descendent_of,
IoInfo, MemoryInfo, MemorySize, UsageInfo,
};
use sc_state_db::StateDb;
use sc_state_db::{IsPruned, StateDb};
use sp_arithmetic::traits::Saturating;
use sp_blockchain::{
well_known_cache_keys, Backend as _, CachedHeaderMetadata, Error as ClientError, HeaderBackend,
Expand Down Expand Up @@ -442,9 +442,10 @@ struct PendingBlock<Block: BlockT> {
}

// wrapper that implements trait required for state_db
struct StateMetaDb<'a>(&'a dyn Database<DbHash>);
#[derive(Clone)]
struct StateMetaDb(Arc<dyn Database<DbHash>>);

impl<'a> sc_state_db::MetaDb for StateMetaDb<'a> {
impl sc_state_db::MetaDb for StateMetaDb {
type Error = sp_database::error::DatabaseError;

fn get_meta(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
Expand Down Expand Up @@ -915,7 +916,7 @@ impl<Block: BlockT> sc_client_api::backend::BlockImportOperation<Block>

struct StorageDb<Block: BlockT> {
pub db: Arc<dyn Database<DbHash>>,
pub state_db: StateDb<Block::Hash, Vec<u8>>,
pub state_db: StateDb<Block::Hash, Vec<u8>, StateMetaDb>,
prefix_keys: bool,
}

Expand Down Expand Up @@ -1104,11 +1105,11 @@ impl<Block: BlockT> Backend<Block> {
let mut db_init_transaction = Transaction::new();

let requested_state_pruning = config.state_pruning.clone();
let state_meta_db = StateMetaDb(db.as_ref());
let state_meta_db = StateMetaDb(db.clone());
let map_e = sp_blockchain::Error::from_state_db;

let (state_db_init_commit_set, state_db) = StateDb::open(
&state_meta_db,
state_meta_db,
requested_state_pruning,
!db.supports_ref_counting(),
should_init,
Expand Down Expand Up @@ -1317,10 +1318,11 @@ impl<Block: BlockT> Backend<Block> {
}

trace!(target: "db", "Canonicalize block #{} ({:?})", new_canonical, hash);
let commit =
self.storage.state_db.canonicalize_block(&hash).map_err(
sp_blockchain::Error::from_state_db::<sc_state_db::Error<io::Error>>,
)?;
let commit = self.storage.state_db.canonicalize_block(&hash).map_err(
sp_blockchain::Error::from_state_db::<
sc_state_db::Error<sp_database::error::DatabaseError>,
>,
)?;
apply_state_commit(transaction, commit);
}
Ok(())
Expand Down Expand Up @@ -1471,14 +1473,16 @@ impl<Block: BlockT> Backend<Block> {
.storage
.state_db
.insert_block(&hash, number_u64, pending_block.header.parent_hash(), changeset)
.map_err(|e: sc_state_db::Error<io::Error>| {
.map_err(|e: sc_state_db::Error<sp_database::error::DatabaseError>| {
sp_blockchain::Error::from_state_db(e)
})?;
apply_state_commit(&mut transaction, commit);
if number <= last_finalized_num {
// Canonicalize in the db when re-importing existing blocks with state.
let commit = self.storage.state_db.canonicalize_block(&hash).map_err(
sp_blockchain::Error::from_state_db::<sc_state_db::Error<io::Error>>,
sp_blockchain::Error::from_state_db::<
sc_state_db::Error<sp_database::error::DatabaseError>,
>,
)?;
apply_state_commit(&mut transaction, commit);
meta_updates.push(MetaUpdate {
Expand Down Expand Up @@ -1679,10 +1683,11 @@ impl<Block: BlockT> Backend<Block> {
.map(|c| f_num.saturated_into::<u64>() > c)
.unwrap_or(true)
{
let commit =
self.storage.state_db.canonicalize_block(&f_hash).map_err(
sp_blockchain::Error::from_state_db::<sc_state_db::Error<io::Error>>,
)?;
let commit = self.storage.state_db.canonicalize_block(&f_hash).map_err(
sp_blockchain::Error::from_state_db::<
sc_state_db::Error<sp_database::error::DatabaseError>,
>,
)?;
apply_state_commit(transaction, commit);
}

Expand Down Expand Up @@ -2294,13 +2299,14 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {

match self.blockchain.header_metadata(hash) {
Ok(ref hdr) => {
if !self.have_state_at(&hash, hdr.number) {
return Err(sp_blockchain::Error::UnknownBlock(format!(
"State already discarded for {:?}",
block
)))
}
if let Ok(()) = self.storage.state_db.pin(&hash) {
let hint = || {
sc_state_db::NodeDb::get(self.storage.as_ref(), hdr.state_root.as_ref())
.unwrap_or(None)
.is_some()
};
if let Ok(()) =
self.storage.state_db.pin(&hash, hdr.number.saturated_into::<u64>(), hint)
{
let root = hdr.state_root;
let db_state = DbStateBuilder::<Block>::new(self.storage.clone(), root)
.with_optional_cache(
Expand Down Expand Up @@ -2333,7 +2339,20 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
_ => false,
}
} else {
!self.storage.state_db.is_pruned(hash, number.saturated_into::<u64>())
match self.storage.state_db.is_pruned(hash, number.saturated_into::<u64>()) {
IsPruned::Pruned => false,
IsPruned::NotPruned => true,
IsPruned::MaybePruned => match self.blockchain.header_metadata(*hash) {
Ok(header) => sp_state_machine::Storage::get(
self.storage.as_ref(),
&header.state_root,
(&[], None),
)
.unwrap_or(None)
.is_some(),
_ => false,
},
}
}
}

Expand Down
Loading

0 comments on commit ad55a2a

Please sign in to comment.