Update conformance vectors (#768)

* setup versioning and upgrade testing logic

* Update vectors to handle circ supply param

* Update test skips

* reset feature usage

* lint

* header

* oops

* lint feature

* Refactor runtime, get rid of gas syscalls and default syscalls

* Remove commented out code

* Switch to full verifier and fetch params by default

* lock update

* nvm that was nightly

blockchain/state_manager/src/lib.rs

@@ -20,9 +20,7 @@ use flo_stream::Subscriber;
 use forest_blocks::{BlockHeader, Tipset, TipsetKeys};
 use futures::channel::oneshot;
 use futures::stream::{FuturesUnordered, StreamExt};
-use interpreter::{
-    resolve_to_key_addr, ApplyRet, BlockMessages, ChainRand, DefaultSyscalls, Rand, VM,
-};
+use interpreter::{resolve_to_key_addr, ApplyRet, BlockMessages, ChainRand, Rand, VM};
 use ipld_amt::Amt;
 use log::{trace, warn};
 use message::{message_receipt, unsigned_message};
@@ -196,14 +194,14 @@ where
     {
         let mut buf_store = BufferedBlockStore::new(self.bs.as_ref());
         // TODO change from statically using devnet params when needed
-        let mut vm = VM::<_, _, _, _>::new(
+        let mut vm = VM::<_, _, _, V>::new(
             p_state,
             &buf_store,
             epoch,
-            DefaultSyscalls::<_, V>::new(&buf_store),
             rand,
             base_fee,
             get_network_version_default,
+            None,
         )?;
 
         // Apply tipset messages
@@ -276,14 +274,14 @@ where
         span!("state_call_raw", {
             let block_store = self.blockstore();
             let buf_store = BufferedBlockStore::new(block_store);
-            let mut vm = VM::<_, _, _, _>::new(
+            let mut vm = VM::<_, _, _, V>::new(
                 bstate,
                 &buf_store,
                 *bheight,
-                DefaultSyscalls::<_, V>::new(&buf_store),
                 rand,
                 0.into(),
                 get_network_version_default,
+                None,
             )?;
 
             if msg.gas_limit() == 0 {
@@ -352,14 +350,14 @@ where
             .map_err(|_| Error::Other("Could not load tipset state".to_string()))?;
         let chain_rand = ChainRand::new(ts.key().to_owned());
 
-        let mut vm = VM::<_, _, _, _>::new(
+        let mut vm = VM::<_, _, _, V>::new(
             &st,
             self.bs.as_ref(),
             ts.epoch() + 1,
-            DefaultSyscalls::<_, V>::new(self.bs.as_ref()),
             &chain_rand,
             ts.blocks()[0].parent_base_fee().clone(),
             get_network_version_default,
+            None,
         )?;
 
         for msg in prior_messages {

crypto/src/randomness.rs

@@ -1,11 +1,12 @@
 // Copyright 2020 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
+use encoding::repr::*;
 use num_derive::FromPrimitive;
 use num_traits::FromPrimitive;
 
 /// Specifies a domain for randomness generation.
-#[derive(PartialEq, Eq, Copy, Clone, FromPrimitive, Debug, Hash)]
+#[derive(PartialEq, Eq, Copy, Clone, FromPrimitive, Debug, Hash, Deserialize_repr)]
 #[repr(i64)]
 pub enum DomainSeparationTag {
     TicketProduction = 1,

forest/Cargo.toml

@@ -33,14 +33,11 @@ rpc = { path = "../node/rpc" }
 rpc_client = { package = "rpc-client", path = "../node/rpc-client" }
 fil_types = { path = "../types" }
 serde_json = "1.0"
-blake2b_simd = "0.5.9"
-surf = "2.0"
-pbr = "1.0.3"
-pin-project-lite = "0.1"
 message_pool = { package = "message_pool", path = "../blockchain/message_pool" }
 wallet = { package = "key_management", path = "../key_management" }
 jsonrpc-v2 = { version = "0.5.2", git = "https://github.com/ChainSafe/jsonrpc-v2", features = ["easy-errors", "macros"], default-features = false }
 uuid = { version = "0.8.1", features = ["v4"] }
 auth = { path = "../utils/auth"}
 actor = { path = "../vm/actor/" }
 genesis = { path = "../utils/genesis" }
+paramfetch = { path = "../utils/paramfetch" }

forest/src/cli/fetch_params_cmd.rs

@@ -1,8 +1,8 @@
 // Copyright 2020 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
-use crate::paramfetch::{get_params_default, SectorSizeOpt};
 use fil_types::SectorSize;
+use paramfetch::{get_params_default, SectorSizeOpt};
 use structopt::StructOpt;
 
 #[allow(missing_docs)]

forest/src/daemon.rs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: Apache-2.0, MIT
 
 use super::cli::{block_until_sigint, Config};
-use super::paramfetch::{get_params_default, SectorSizeOpt};
 use actor::EPOCH_DURATION_SECONDS;
 use async_std::sync::RwLock;
 use async_std::task;
@@ -18,6 +17,7 @@ use genesis::initialize_genesis;
 use libp2p::identity::{ed25519, Keypair};
 use log::{debug, info, trace};
 use message_pool::{MessagePool, MpoolConfig, MpoolRpcProvider};
+use paramfetch::{get_params_default, SectorSizeOpt};
 use rpc::{start_rpc, RpcState};
 use state_manager::StateManager;
 use std::sync::Arc;

forest/src/main.rs

@@ -4,7 +4,6 @@
 mod cli;
 mod daemon;
 mod logger;
-pub(crate) mod paramfetch;
 mod subcommand;
 
 use cli::CLI;

tests/conformance_tests/Cargo.toml

@@ -5,6 +5,7 @@ authors = ["ChainSafe Systems <info@chainsafe.io>"]
 edition = "2018"
 
 [features]
+default = []
 submodule_tests = [
     "serde",
     "cid",
@@ -62,3 +63,5 @@ colored = "2.0"
 ipld = { package = "forest_ipld", path = "../../ipld", features = ["json"] }
 ipld_hamt = { path = "../../ipld/hamt", features = ["ignore-dead-links"] }
 log = "0.4"
+paramfetch = { path = "../../utils/paramfetch" }
+async-std = "1.6"

tests/conformance_tests/src/lib.rs

@@ -4,10 +4,12 @@
 #![cfg(feature = "submodule_tests")]
 
 mod message;
+mod rand_replay;
 mod stubs;
 mod tipset;
 
 pub use self::message::*;
+pub use self::rand_replay::*;
 pub use self::stubs::*;
 pub use self::tipset::*;
 use actor::CHAOS_ACTOR_CODE_ID;
@@ -16,7 +18,7 @@ use blockstore::BlockStore;
 use cid::Cid;
 use clock::ChainEpoch;
 use crypto::{DomainSeparationTag, Signature};
-use encoding::Cbor;
+use encoding::{tuple::*, Cbor};
 use fil_types::{SealVerifyInfo, WindowPoStVerifyInfo};
 use forest_message::{ChainMessage, Message, MessageReceipt, SignedMessage, UnsignedMessage};
 use interpreter::{ApplyRet, BlockMessages, Rand, VM};
@@ -86,15 +88,15 @@ pub struct StateTreeVector {
     pub root_cid: Cid,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
 pub struct GenerationData {
     #[serde(default)]
     pub source: String,
     #[serde(default)]
     pub version: String,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
 pub struct MetaData {
     pub id: String,
     #[serde(default)]
@@ -108,10 +110,13 @@ pub struct MetaData {
 
 #[derive(Debug, Deserialize)]
 pub struct PreConditions {
-    pub epoch: ChainEpoch,
     pub state_tree: StateTreeVector,
     #[serde(default)]
     pub basefee: Option<f64>,
+    #[serde(default)]
+    pub circ_supply: Option<f64>,
+    #[serde(default)]
+    pub variants: Vec<Variant>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -129,6 +134,43 @@ pub struct Selector {
     pub puppet_actor: Option<String>,
     #[serde(default)]
     pub chaos_actor: Option<String>,
+    #[serde(default)]
+    pub min_protocol_version: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Variant {
+    pub id: String,
+    pub epoch: ChainEpoch,
+    pub nv: u32,
+}
+
+/// Encoded VM randomness used to be replayed.
+pub type Randomness = Vec<RandomnessMatch>;
+
+/// One randomness entry.
+#[derive(Debug, Deserialize)]
+pub struct RandomnessMatch {
+    pub on: RandomnessRule,
+    #[serde(with = "base64_bytes")]
+    pub ret: Vec<u8>,
+}
+
+#[derive(Debug, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum RandomnessKind {
+    Beacon,
+    Chain,
+}
+
+/// Rule for matching when randomness is returned.
+#[derive(Debug, Deserialize_tuple, PartialEq)]
+pub struct RandomnessRule {
+    pub kind: RandomnessKind,
+    pub dst: DomainSeparationTag,
+    pub epoch: ChainEpoch,
+    #[serde(with = "base64_bytes")]
+    pub entropy: Vec<u8>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -145,12 +187,9 @@ pub enum TestVector {
         preconditions: PreConditions,
         apply_messages: Vec<MessageVector>,
         postconditions: PostConditions,
-    },
-    #[serde(rename = "block")]
-    Block {
-        selector: Option<Selector>,
-        #[serde(rename = "_meta")]
-        meta: Option<MetaData>,
+
+        #[serde(default)]
+        randomness: Randomness,
     },
     #[serde(rename = "tipset")]
     Tipset {
@@ -164,12 +203,6 @@ pub enum TestVector {
         apply_tipsets: Vec<TipsetVector>,
         postconditions: PostConditions,
     },
-    #[serde(rename = "chain")]
-    Chain {
-        selector: Option<Selector>,
-        #[serde(rename = "_meta")]
-        meta: Option<MetaData>,
-    },
 }
 
 // This might be changed to be encoded into vector, matching go runner for now

tests/conformance_tests/src/message.rs

@@ -10,25 +10,32 @@ pub struct MessageVector {
     #[serde(with = "base64_bytes")]
     pub bytes: Vec<u8>,
     #[serde(default)]
-    pub epoch: Option<ChainEpoch>,
+    pub epoch_offset: Option<ChainEpoch>,
+}
+
+pub struct ExecuteMessageParams<'a> {
+    pub pre_root: &'a Cid,
+    pub epoch: ChainEpoch,
+    pub msg: &'a ChainMessage,
+    pub circ_supply: TokenAmount,
+    pub basefee: TokenAmount,
+    pub randomness: ReplayingRand<'a>,
 }
 
 pub fn execute_message(
     bs: &db::MemoryDB,
-    msg: &ChainMessage,
-    pre_root: &Cid,
-    epoch: ChainEpoch,
-    basefee: TokenAmount,
     selector: &Option<Selector>,
+    params: ExecuteMessageParams,
 ) -> Result<(ApplyRet, Cid), Box<dyn StdError>> {
-    let mut vm = VM::<_, _, _, _>::new(
-        pre_root,
+    let circ_supply = params.circ_supply;
+    let mut vm = VM::<_, _, _>::new(
+        params.pre_root,
         bs,
-        epoch,
-        TestSyscalls,
-        &TestRand,
-        basefee,
+        params.epoch,
+        &params.randomness,
+        params.basefee,
         get_network_version_default,
+        Some(Box::new(move |_, _| Ok(circ_supply.clone()))),
     )?;
 
     if let Some(s) = &selector {
@@ -41,7 +48,7 @@ pub fn execute_message(
         }
     }
 
-    let ret = vm.apply_message(msg)?;
+    let ret = vm.apply_message(params.msg)?;
 
     let root = vm.flush()?;
     Ok((ret, root))