Skip to content

Commit

Permalink
[CLI] Fixup export objects for indexer from statedb and rebuild index…
Browse files Browse the repository at this point in the history 
…er (#2378)

* fixup export indexer from statedb and rebuild indexer

* remove root object for export indexer
  • Loading branch information
@baichuan3
baichuan3 authored Aug 7, 2024
1 parent 3553aa0 commit 8b93857
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 66 deletions.
40 changes: 18 additions & 22 deletions crates/rooch/src/commands/indexer/commands/rebuild.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@ use rooch_types::rooch_network::RoochChainID;

use crate::commands::indexer::commands::init_indexer;
use crate::commands::statedb::commands::import::parse_csv_fields;
use crate::commands::statedb::commands::{
GLOBAL_STATE_TYPE_OBJECT, GLOBAL_STATE_TYPE_PREFIX, GLOBAL_STATE_TYPE_ROOT,
};

/// Rebuild indexer
#[derive(Debug, Parser)]
Expand Down Expand Up @@ -89,7 +86,7 @@ fn produce_updates(
batch_size: usize,
) -> Result<()> {
let mut csv_reader = BufReader::new(File::open(input).unwrap());
let mut last_state_type = None;
// let mut last_state_type = None;

// set genesis tx_order and state_index_generator for indexer rebuild
let tx_order: u64 = 0;
Expand All @@ -107,25 +104,24 @@ fn produce_updates(
for line in csv_reader.by_ref().lines().take(batch_size) {
let line = line?;

if line.starts_with(GLOBAL_STATE_TYPE_PREFIX) {
let (state_type, _) = parse_csv_fields(&line)?;
last_state_type = Some(state_type);
continue;
}

let (_c1, state_str) = parse_csv_fields(&line)?;
let state = ObjectState::from_str(&state_str)?;

let state_type = last_state_type
.clone()
.expect("Last state type should have value");

if state_type.eq(GLOBAL_STATE_TYPE_OBJECT) || state_type.eq(GLOBAL_STATE_TYPE_ROOT) {
let indexer_state =
IndexerObjectState::new(state.metadata, tx_order, state_index_generator);
state_index_generator += 1;
updates.object_states.push(indexer_state);
};
let state_result = ObjectState::from_str(&state_str);
match state_result {
Ok(state) => {
let indexer_state =
IndexerObjectState::new(state.metadata, tx_order, state_index_generator);
state_index_generator += 1;
updates.object_states.push(indexer_state);
}
Err(e) => {
println!(
"Parse object state error, state maybe not an object, line: {} , err: {:?}",
line,
e.to_string()
);
continue;
}
}
}
if updates.object_states.is_empty() {
break;
Expand Down
95 changes: 51 additions & 44 deletions crates/rooch/src/commands/statedb/commands/export.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,12 @@ use crate::commands::statedb::commands::{
#[serde(rename_all = "lowercase")]
pub enum ExportMode {
#[default]
Genesis = 0, // dump InscriptionStore, BitcoinUTXOStore, RoochToBitcoinAddressMapping for genesis start-up
Full = 1,
Snapshot = 2,
Indexer = 3, // dump InscriptionStore, BitcoinUTXOStore for rebuild indexer
Object = 4,
Genesis, // dump InscriptionStore, BitcoinUTXOStore, RoochToBitcoinAddressMapping for genesis start-up
Full,
Snapshot,
FullIndexer, // dump Full Objects, include InscriptionStore, BitcoinUTXOStore for rebuild indexer
Indexer, // dump InscriptionStore, BitcoinUTXOStore for rebuild indexer
Object,
}

impl Display for ExportMode {
Expand All @@ -44,6 +45,7 @@ impl Display for ExportMode {
ExportMode::Genesis => write!(f, "genesis"),
ExportMode::Full => write!(f, "full"),
ExportMode::Snapshot => write!(f, "snapshot"),
ExportMode::FullIndexer => write!(f, "fullindexer"),
ExportMode::Indexer => write!(f, "indexer"),
ExportMode::Object => write!(f, "object"),
}
Expand All @@ -58,19 +60,14 @@ impl FromStr for ExportMode {
"genesis" => Ok(ExportMode::Genesis),
"full" => Ok(ExportMode::Full),
"snapshot" => Ok(ExportMode::Snapshot),
"fullindexer" => Ok(ExportMode::FullIndexer),
"indexer" => Ok(ExportMode::Indexer),
"object" => Ok(ExportMode::Object),
_ => Err("export-mode no match"),
}
}
}

impl ExportMode {
pub fn to_num(self) -> u8 {
self as u8
}
}

#[derive(Debug, Serialize, Deserialize, Clone, Ord, Eq, PartialOrd, PartialEq)]
pub struct ExportID {
pub object_id: ObjectID,
Expand Down Expand Up @@ -162,6 +159,7 @@ impl FromStr for ExportObjectName {
}
}

#[allow(dead_code)]
#[derive(Debug, Parser)]
pub struct ExportCommand {
/// export state root, default latest state root
Expand Down Expand Up @@ -218,6 +216,9 @@ impl ExportCommand {
ExportMode::Snapshot => {
todo!()
}
ExportMode::FullIndexer => {
Self::export_full_indexer(&moveos_store, root_state_root, &mut writer)?;
}
ExportMode::Indexer => {
Self::export_indexer(&moveos_store, root_state_root, &mut writer)?;
}
Expand Down Expand Up @@ -263,49 +264,53 @@ impl ExportCommand {
)
}

fn export_full_indexer<W: std::io::Write>(
moveos_store: &MoveOSStore,
root_state_root: H256,
writer: &mut Writer<W>,
) -> Result<()> {
// export root object, utxo, inscription store object, exclude dynamic filed objects
let object_ids = vec![
ObjectID::root(),
BitcoinUTXOStore::object_id(),
InscriptionStore::object_id(),
];

Self::internal_export_indexer(moveos_store, root_state_root, writer, object_ids)?;
writer.flush()?;
Ok(())
}

fn export_indexer<W: std::io::Write>(
moveos_store: &MoveOSStore,
root_state_root: H256,
writer: &mut Writer<W>,
) -> Result<()> {
// export root's ExportID
export_root_export_id(root_state_root, writer)?;
// export utxo, inscription store object
let field_keys = vec![
BitcoinUTXOStore::object_id().field_key(),
InscriptionStore::object_id().field_key(),
];
export_fields(moveos_store, root_state_root, writer, field_keys)?;
// export utxo store fields
let utxo_store_state_root = get_state_root(
moveos_store,
root_state_root,
BitcoinUTXOStore::object_id().field_key(),
);
Self::export_top_level_fields(
moveos_store,
utxo_store_state_root,
BitcoinUTXOStore::object_id(),
Some(ExportObjectName::UtxoStore.to_string()),
writer,
)?;
// export inscription store fields
let inscription_store_state_root = get_state_root(
moveos_store,
root_state_root,
InscriptionStore::object_id().field_key(),
);
Self::export_top_level_fields(
moveos_store,
inscription_store_state_root,
BitcoinUTXOStore::object_id(),
Some(ExportObjectName::InscriptionStore.to_string()),
writer,
)?;
let object_ids = vec![BitcoinUTXOStore::object_id(), InscriptionStore::object_id()];

Self::internal_export_indexer(moveos_store, root_state_root, writer, object_ids)?;
writer.flush()?;
Ok(())
}

fn internal_export_indexer<W: std::io::Write>(
moveos_store: &MoveOSStore,
root_state_root: H256,
writer: &mut Writer<W>,
object_ids: Vec<ObjectID>,
) -> Result<()> {
for obj_id in object_ids.into_iter() {
let state_root = if obj_id == ObjectID::root() {
root_state_root
} else {
get_state_root(moveos_store, root_state_root, obj_id.field_key())
};
Self::export_top_level_fields(moveos_store, state_root, obj_id, None, writer)?;
}
Ok(())
}

// export top level fields of an object, no recursive export child field
fn export_top_level_fields<W: std::io::Write>(
moveos_store: &MoveOSStore,
Expand Down Expand Up @@ -477,6 +482,7 @@ impl ExportCommand {
}
}

#[allow(dead_code)]
// export root's export id for further checking in import job.
fn export_root_export_id<W: std::io::Write>(
root_state_root: H256,
Expand All @@ -487,6 +493,7 @@ fn export_root_export_id<W: std::io::Write>(
Ok(())
}

#[allow(dead_code)]
fn export_fields<W: std::io::Write>(
moveos_store: &MoveOSStore,
state_root: H256,
Expand Down

0 comments on commit 8b93857

Please sign in to comment.