Skip to content

Commit

Permalink
Refactor statediff to own crate (#779)
Browse files Browse the repository at this point in the history
* Refactor statediff out to be used elsewhere

* Lint

* Fix validation error
  • Loading branch information
austinabell authored Oct 22, 2020
1 parent 6fb284a commit 199d7fe
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 103 deletions.
22 changes: 18 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ members = [
"utils/commcid",
"utils/json_utils",
"utils/genesis",
"utils/statediff",
"types",
"key_management",
]
4 changes: 2 additions & 2 deletions blockchain/chain_sync/src/sync_worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -567,15 +567,15 @@ where
if &state_root != h.state_root() {
return Err(Error::Validation(format!(
"Parent state root did not match computed state: {} (header), {} (computed)",
h.state_root(),
state_root,
h.state_root()
)));
}
if &rec_root != h.message_receipts() {
return Err(Error::Validation(format!(
"Parent receipt root did not match computed root: {} (header), {} (computed)",
h.message_receipts(),
rec_root,
h.message_receipts()
)));
}
Ok(())
Expand Down
5 changes: 1 addition & 4 deletions tests/conformance_tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,7 @@ forest_car = { path = "../../ipld/car" }
flate2 = "1.0"
lazy_static = "1.4"
pretty_env_logger = "0.4.0"
difference = "2.0"
colored = "2.0"
ipld = { package = "forest_ipld", path = "../../ipld", features = ["json"] }
ipld_hamt = { path = "../../ipld/hamt", features = ["ignore-dead-links"] }
log = "0.4"
paramfetch = { path = "../../utils/paramfetch" }
statediff = { path = "../../utils/statediff" }
async-std = "1.6"
97 changes: 4 additions & 93 deletions tests/conformance_tests/tests/conformance_runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,32 +6,23 @@
#[macro_use]
extern crate lazy_static;

use address::Address;
use blockstore::resolve::resolve_cids_recursive;
use cid::{json::CidJson, Cid};
use cid::Cid;
use clock::ChainEpoch;
use colored::*;
use conformance_tests::*;
use difference::{Changeset, Difference};
use encoding::Cbor;
use fil_types::{HAMT_BIT_WIDTH, TOTAL_FILECOIN};
use fil_types::TOTAL_FILECOIN;
use flate2::read::GzDecoder;
use forest_message::{MessageReceipt, UnsignedMessage};
use interpreter::ApplyRet;
use ipld::json::{IpldJson, IpldJsonRef};
use ipld::Ipld;
use ipld_hamt::{BytesKey, Hamt};
use num_bigint::{BigInt, ToBigInt};
use paramfetch::{get_params_default, SectorSizeOpt};
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use statediff::print_state_diff;
use std::error::Error as StdError;
use std::fmt;
use std::fs::File;
use std::io::BufReader;
use std::sync::Arc;
use vm::ActorState;
use walkdir::{DirEntry, WalkDir};

lazy_static! {
Expand Down Expand Up @@ -120,56 +111,6 @@ fn check_msg_result(
Ok(())
}

#[derive(Serialize, Deserialize)]
struct ActorStateResolved {
code: CidJson,
sequence: u64,
balance: String,
state: IpldJson,
}

fn root_to_state_map(
bs: &db::MemoryDB,
root: &Cid,
) -> Result<BTreeMap<String, ActorStateResolved>, Box<dyn StdError>> {
let mut actors = BTreeMap::new();
let hamt: Hamt<_, _> = Hamt::load_with_bit_width(root, bs, HAMT_BIT_WIDTH)?;
hamt.for_each(|k: &BytesKey, actor: &ActorState| {
let addr = Address::from_bytes(&k.0)?;

let resolved =
resolve_cids_recursive(bs, &actor.state).unwrap_or(Ipld::Link(actor.state.clone()));
let resolved_state = ActorStateResolved {
state: IpldJson(resolved),
code: CidJson(actor.code.clone()),
balance: actor.balance.to_string(),
sequence: actor.sequence,
};

actors.insert(addr.to_string(), resolved_state);
Ok(())
})
.unwrap();

Ok(actors)
}

/// Tries to resolve state tree actors, if all data exists in store.
/// The actors hamt is hard to parse in a diff, so this attempts to remedy this.
fn try_resolve_actor_states(
bs: &db::MemoryDB,
root: &Cid,
expected_root: &Cid,
) -> Result<Changeset, Box<dyn StdError>> {
let e_state = root_to_state_map(bs, expected_root)?;
let c_state = root_to_state_map(bs, root)?;

let expected_json = serde_json::to_string_pretty(&e_state)?;
let actual_json = serde_json::to_string_pretty(&c_state)?;

Ok(Changeset::new(&expected_json, &actual_json, "\n"))
}

fn compare_state_roots(bs: &db::MemoryDB, root: &Cid, expected_root: &Cid) -> Result<(), String> {
if root != expected_root {
let error_msg = format!(
Expand All @@ -178,39 +119,9 @@ fn compare_state_roots(bs: &db::MemoryDB, root: &Cid, expected_root: &Cid) -> Re
);

if std::env::var("FOREST_DIFF") == Ok("1".to_owned()) {
let Changeset { diffs, .. } = try_resolve_actor_states(bs, root, expected_root)
.unwrap_or_else(|e| {
println!(
"Could not resolve actor states: {}\nUsing default resolution:",
e
);
let expected = resolve_cids_recursive(bs, &expected_root)
.expect("Failed to populate Ipld");
let actual =
resolve_cids_recursive(bs, &root).expect("Failed to populate Ipld");

let expected_json =
serde_json::to_string_pretty(&IpldJsonRef(&expected)).unwrap();
let actual_json = serde_json::to_string_pretty(&IpldJsonRef(&actual)).unwrap();

Changeset::new(&expected_json, &actual_json, "\n")
});

println!("{}:", error_msg);

for diff in diffs.iter() {
match diff {
Difference::Same(x) => {
println!(" {}", x);
}
Difference::Add(x) => {
println!("{}", format!("+{}", x).green());
}
Difference::Rem(x) => {
println!("{}", format!("-{}", x).red());
}
}
}
print_state_diff(bs, root, expected_root).unwrap();
}

return Err(error_msg.into());
Expand Down
25 changes: 25 additions & 0 deletions utils/statediff/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
[package]
name = "statediff"
version = "0.1.0"
authors = ["ChainSafe Systems <info@chainsafe.io>"]
edition = "2018"

[dependencies]
serde_json = "1.0"
blockstore = { package = "ipld_blockstore", path = "../../ipld/blockstore/", features = [
"resolve"
] }
cid = { package = "forest_cid", path = "../../ipld/cid", features = [
"cbor",
"json"
] }
difference = "2.0"
colored = "2.0"
ipld_hamt = { path = "../../ipld/hamt", features = ["ignore-dead-links"] }
address = { package = "forest_address", path = "../../vm/address", features = [
"json"
] }
serde = { version = "1.0", features = ["derive"] }
ipld = { package = "forest_ipld", path = "../../ipld", features = ["json"] }
vm = { package = "forest_vm", path = "../../vm" }
fil_types = { path = "../../types" }
111 changes: 111 additions & 0 deletions utils/statediff/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
// Copyright 2020 ChainSafe Systems
// SPDX-License-Identifier: Apache-2.0, MIT

use address::Address;
use blockstore::resolve::resolve_cids_recursive;
use blockstore::BlockStore;
use cid::{json::CidJson, Cid};
use colored::*;
use difference::{Changeset, Difference};
use fil_types::HAMT_BIT_WIDTH;
use ipld::json::{IpldJson, IpldJsonRef};
use ipld::Ipld;
use ipld_hamt::{BytesKey, Hamt};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::error::Error as StdError;
use vm::ActorState;

#[derive(Serialize, Deserialize)]
struct ActorStateResolved {
code: CidJson,
sequence: u64,
balance: String,
state: IpldJson,
}

fn root_to_state_map<BS: BlockStore>(
bs: &BS,
root: &Cid,
) -> Result<BTreeMap<String, ActorStateResolved>, Box<dyn StdError>> {
let mut actors = BTreeMap::new();
let hamt: Hamt<_, _> = Hamt::load_with_bit_width(root, bs, HAMT_BIT_WIDTH)?;
hamt.for_each(|k: &BytesKey, actor: &ActorState| {
let addr = Address::from_bytes(&k.0)?;

let resolved = resolve_cids_recursive(bs, &actor.state)
.unwrap_or_else(|_| Ipld::Link(actor.state.clone()));
let resolved_state = ActorStateResolved {
state: IpldJson(resolved),
code: CidJson(actor.code.clone()),
balance: actor.balance.to_string(),
sequence: actor.sequence,
};

actors.insert(addr.to_string(), resolved_state);
Ok(())
})
.unwrap();

Ok(actors)
}

/// Tries to resolve state tree actors, if all data exists in store.
/// The actors hamt is hard to parse in a diff, so this attempts to remedy this.
fn try_resolve_actor_states<BS: BlockStore>(
bs: &BS,
root: &Cid,
expected_root: &Cid,
) -> Result<Changeset, Box<dyn StdError>> {
let e_state = root_to_state_map(bs, expected_root)?;
let c_state = root_to_state_map(bs, root)?;

let expected_json = serde_json::to_string_pretty(&e_state)?;
let actual_json = serde_json::to_string_pretty(&c_state)?;

Ok(Changeset::new(&expected_json, &actual_json, "\n"))
}

/// Prints a diff of the resolved state tree.
/// If the actor's Hamt cannot be loaded, base ipld resolution is given.
pub fn print_state_diff<BS>(
bs: &BS,
root: &Cid,
expected_root: &Cid,
) -> Result<(), Box<dyn StdError>>
where
BS: BlockStore,
{
let Changeset { diffs, .. } = match try_resolve_actor_states(bs, root, expected_root) {
Ok(cs) => cs,
Err(e) => {
println!(
"Could not resolve actor states: {}\nUsing default resolution:",
e
);
let expected = resolve_cids_recursive(bs, &expected_root)?;
let actual = resolve_cids_recursive(bs, &root)?;

let expected_json = serde_json::to_string_pretty(&IpldJsonRef(&expected))?;
let actual_json = serde_json::to_string_pretty(&IpldJsonRef(&actual))?;

Changeset::new(&expected_json, &actual_json, "\n")
}
};

for diff in diffs.iter() {
match diff {
Difference::Same(x) => {
println!(" {}", x);
}
Difference::Add(x) => {
println!("{}", format!("+{}", x).green());
}
Difference::Rem(x) => {
println!("{}", format!("-{}", x).red());
}
}
}

Ok(())
}

0 comments on commit 199d7fe

Please sign in to comment.