Update gas estimation code to use new data from NEAR protocol v53 (#517)

engine-standalone-storage/Cargo.toml

@@ -20,14 +20,14 @@ aurora-engine-sdk = { path = "../engine-sdk", default-features = false, features
 aurora-engine-transactions = { path = "../engine-transactions", default-features = false, features = ["std"] }
 borsh = { version = "0.8.2" }
 evm-core = { git = "https://github.com/aurora-is-near/sputnikvm.git", rev = "37448b6cacd98b06282cff5a559684505c29bd2b", default-features = false }
-rocksdb = { version = "0.16.0", default-features = false }
+rocksdb = { version = "0.18.0", default-features = false }
 postgres = "0.19.2"
 serde = "1.0.130"
 serde_json = "1.0.72"
 base64 = "0.13.0"
 
 [features]
-default = ["snappy", "lz4", "zstd", "zlib", "bzip2"]
+default = ["snappy", "lz4", "zstd", "zlib"]
 mainnet = []
 testnet = []
 snappy = ["rocksdb/snappy"]

engine-tests/Cargo.toml

@@ -35,13 +35,13 @@ ethabi = { git = "https://github.com/darwinia-network/ethabi", branch = "xavier-
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 hex = { version = "0.4.3", default-features = false }
-near-sdk = { git = "https://github.com/aurora-is-near/near-sdk-rs.git", rev = "5e58722bd61d9d24ae6293326146c751f0a814fb" }
-near-sdk-sim = { git = "https://github.com/aurora-is-near/near-sdk-rs.git", rev = "5e58722bd61d9d24ae6293326146c751f0a814fb" }
-near-crypto = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783" }
-near-vm-runner = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783", default-features = false, features = [ "wasmer2_vm" ] }
-near-vm-logic = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783" }
-near-primitives-core = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783" }
-near-primitives = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783" }
+near-sdk = { git = "https://github.com/aurora-is-near/near-sdk-rs.git", rev = "ba2eddbfbf4484ac3e44b4c8119bbac4907d6e07" }
+near-sdk-sim = { git = "https://github.com/aurora-is-near/near-sdk-rs.git", rev = "ba2eddbfbf4484ac3e44b4c8119bbac4907d6e07" }
+near-crypto = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda" }
+near-vm-runner = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda", default-features = false, features = [ "wasmer2_vm" ] }
+near-vm-logic = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda" }
+near-primitives-core = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda" }
+near-primitives = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda" }
 libsecp256k1 = "0.3.5"
 rand = "0.7.3"
 criterion = "0.3.4"

engine-tests/src/test_utils/mocked_external.rs

@@ -1,43 +1,66 @@
 use near_vm_logic::mocks::mock_external::MockedExternal;
 
-pub const MAINNET_AVERAGE_TRIE_DEPTH: u64 = 10;
+/// Derived from mainnet data reported here: https://hackmd.io/@birchmd/r1HRjr0P9
+/// Uses the formulas:
+/// n_T = (G_T / G_R) * (g_R / g_T)
+/// n_c = (G_c / G_R) * (g_R / g_c)
+/// Where n_T is the average number of new touched trie nodes per read,
+/// n_c is the average number of cached trie nodes read per read,
+/// G_T is the average gas cost of touching trie node per Aurora transaction,
+/// G_c is the average gas cost of reading cached trie node per Aurora transaction,
+/// G_R is the average gas cost of `STORAGE_READ_BASE`  per Aurora transaction,
+/// g_R is the `STORAGE_READ_BASE` cost (from the config),
+/// g_T is the `TOUCHING_TRIE_NODE` cost (from the config), and
+/// g_c is the `READ_CACHED_TRIE_NODE` cost (from the config).
+pub const MAINNET_AVERAGE_TOUCHED_TRIE_PER_READ: u64 = 2;
+pub const MAINNET_AVERAGE_READ_CACHED_TRIE_PER_READ: u64 = 11;
+/// This is still needed because writes will touch every node in the depth, unlike reads which take advantage of caching.
+pub const MAINNET_AVERAGE_TRIE_DEPTH: u64 = 13;
 
 #[derive(Clone)]
-pub(crate) struct MockedExternalWithTrie {
+pub struct MockedExternalWithTrie {
     pub underlying: MockedExternal,
-    trie_node_count: std::cell::Cell<u64>,
+    new_trie_node_count: std::cell::Cell<u64>,
+    cached_trie_node_count: std::cell::Cell<u64>,
 }
 
 impl MockedExternalWithTrie {
     pub fn new(ext: MockedExternal) -> Self {
         Self {
             underlying: ext,
-            trie_node_count: std::cell::Cell::new(0),
+            new_trie_node_count: std::cell::Cell::new(0),
+            cached_trie_node_count: std::cell::Cell::new(0),
         }
     }
 
-    fn increment_trie_node_count(&self, amount: u64) {
-        let cell_value = self.trie_node_count.get();
-        self.trie_node_count.set(cell_value + amount);
+    fn increment_new_trie_node_count(&self, amount: u64) {
+        let cell_value = self.new_trie_node_count.get();
+        self.new_trie_node_count.set(cell_value + amount);
+    }
+
+    fn increment_cached_trie_node_count(&self, amount: u64) {
+        let cell_value = self.cached_trie_node_count.get();
+        self.cached_trie_node_count.set(cell_value + amount);
     }
 }
 
 impl near_vm_logic::External for MockedExternalWithTrie {
     fn storage_set(&mut self, key: &[u8], value: &[u8]) -> Result<(), near_vm_logic::VMLogicError> {
-        self.increment_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
+        self.increment_new_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
         self.underlying.storage_set(key, value)
     }
 
     fn storage_get<'a>(
         &'a self,
         key: &[u8],
     ) -> Result<Option<Box<dyn near_vm_logic::ValuePtr + 'a>>, near_vm_logic::VMLogicError> {
-        self.increment_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
+        self.increment_new_trie_node_count(MAINNET_AVERAGE_TOUCHED_TRIE_PER_READ);
+        self.increment_cached_trie_node_count(MAINNET_AVERAGE_READ_CACHED_TRIE_PER_READ);
         self.underlying.storage_get(key)
     }
 
     fn storage_remove(&mut self, key: &[u8]) -> Result<(), near_vm_logic::VMLogicError> {
-        self.increment_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
+        self.increment_new_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
         self.underlying.storage_remove(key)
     }
 
@@ -49,117 +72,6 @@ impl near_vm_logic::External for MockedExternalWithTrie {
         self.underlying.storage_has_key(key)
     }
 
-    fn create_receipt(
-        &mut self,
-        receipt_indices: Vec<near_vm_logic::types::ReceiptIndex>,
-        receiver_id: near_primitives::types::AccountId,
-    ) -> Result<near_vm_logic::types::ReceiptIndex, near_vm_logic::VMLogicError> {
-        self.underlying.create_receipt(receipt_indices, receiver_id)
-    }
-
-    fn append_action_create_account(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying.append_action_create_account(receipt_index)
-    }
-
-    fn append_action_deploy_contract(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-        code: Vec<u8>,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying
-            .append_action_deploy_contract(receipt_index, code)
-    }
-
-    fn append_action_function_call(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-        method_name: Vec<u8>,
-        arguments: Vec<u8>,
-        attached_deposit: near_primitives::types::Balance,
-        prepaid_gas: near_primitives::types::Gas,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying.append_action_function_call(
-            receipt_index,
-            method_name,
-            arguments,
-            attached_deposit,
-            prepaid_gas,
-        )
-    }
-
-    fn append_action_transfer(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-        amount: near_primitives::types::Balance,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying
-            .append_action_transfer(receipt_index, amount)
-    }
-
-    fn append_action_stake(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-        stake: near_primitives::types::Balance,
-        public_key: near_vm_logic::types::PublicKey,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying
-            .append_action_stake(receipt_index, stake, public_key)
-    }
-
-    fn append_action_add_key_with_full_access(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-        public_key: near_vm_logic::types::PublicKey,
-        nonce: u64,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying
-            .append_action_add_key_with_full_access(receipt_index, public_key, nonce)
-    }
-
-    fn append_action_add_key_with_function_call(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-        public_key: near_vm_logic::types::PublicKey,
-        nonce: u64,
-        allowance: Option<near_primitives::types::Balance>,
-        receiver_id: near_primitives::types::AccountId,
-        method_names: Vec<Vec<u8>>,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying.append_action_add_key_with_function_call(
-            receipt_index,
-            public_key,
-            nonce,
-            allowance,
-            receiver_id,
-            method_names,
-        )
-    }
-
-    fn append_action_delete_key(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-        public_key: near_vm_logic::types::PublicKey,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying
-            .append_action_delete_key(receipt_index, public_key)
-    }
-
-    fn append_action_delete_account(
-        &mut self,
-        receipt_index: near_vm_logic::types::ReceiptIndex,
-        beneficiary_id: near_primitives::types::AccountId,
-    ) -> Result<(), near_vm_logic::VMLogicError> {
-        self.underlying
-            .append_action_delete_account(receipt_index, beneficiary_id)
-    }
-
-    fn get_touched_nodes_count(&self) -> u64 {
-        self.trie_node_count.get()
-    }
-
     fn validator_stake(
         &self,
         account_id: &near_primitives::types::AccountId,
@@ -172,4 +84,17 @@ impl near_vm_logic::External for MockedExternalWithTrie {
     ) -> Result<near_primitives::types::Balance, near_vm_logic::VMLogicError> {
         self.underlying.validator_total_stake()
     }
+
+    fn generate_data_id(&mut self) -> near_primitives::hash::CryptoHash {
+        self.underlying.generate_data_id()
+    }
+
+    fn get_trie_nodes_count(&self) -> near_primitives::types::TrieNodesCount {
+        let db_reads = self.new_trie_node_count.get();
+        let mem_reads = self.cached_trie_node_count.get();
+        near_primitives::types::TrieNodesCount {
+            db_reads,
+            mem_reads,
+        }
+    }
 }

engine-tests/src/test_utils/mod.rs

@@ -126,7 +126,7 @@ impl<'a> OneShotAuroraRunner<'a> {
             input,
         );
 
-        near_vm_runner::run(
+        match near_vm_runner::run(
             &self.base.code,
             method_name,
             &mut self.ext,
@@ -136,7 +136,10 @@ impl<'a> OneShotAuroraRunner<'a> {
             &[],
             self.base.current_protocol_version,
             Some(&self.base.cache),
-        )
+        ) {
+            near_vm_runner::VMResult::Aborted(outcome, error) => (Some(outcome), Some(error)),
+            near_vm_runner::VMResult::Ok(outcome) => (Some(outcome), None),
+        }
     }
 }
 
@@ -185,7 +188,7 @@ impl AuroraRunner {
             input,
         );
 
-        let (maybe_outcome, maybe_error) = near_vm_runner::run(
+        let (maybe_outcome, maybe_error) = match near_vm_runner::run(
             &self.code,
             method_name,
             &mut self.ext,
@@ -195,7 +198,10 @@ impl AuroraRunner {
             &[],
             self.current_protocol_version,
             Some(&self.cache),
-        );
+        ) {
+            near_vm_runner::VMResult::Aborted(outcome, error) => (Some(outcome), Some(error)),
+            near_vm_runner::VMResult::Ok(outcome) => (Some(outcome), None),
+        };
         if let Some(outcome) = &maybe_outcome {
             self.context.storage_usage = outcome.storage_usage;
             self.previous_logs = outcome.logs.clone();

engine-tests/src/tests/erc20.rs

@@ -108,12 +108,12 @@ fn profile_erc20_get_balance() {
     assert!(result.is_ok());
 
     // call costs less than 3 Tgas
-    test_utils::assert_gas_bound(profile.all_gas(), 3);
+    test_utils::assert_gas_bound(profile.all_gas(), 2);
     // at least 70% of the cost is spent on wasm computation (as opposed to host functions)
     let wasm_fraction = (100 * profile.wasm_gas()) / profile.all_gas();
     assert!(
-        15 <= wasm_fraction && wasm_fraction <= 20,
-        "{}% is not between 15% and 20%",
+        20 <= wasm_fraction && wasm_fraction <= 30,
+        "{}% is not between 20% and 30%",
         wasm_fraction
     );
 }

engine-tests/src/tests/erc20_connector.rs

@@ -410,7 +410,7 @@ mod sim_tests {
     const FT_TOTAL_SUPPLY: u128 = 1_000_000;
     const FT_TRANSFER_AMOUNT: u128 = 300_000;
     const FT_EXIT_AMOUNT: u128 = 100_000;
-    const FT_ACCOUNT: &str = "test_token";
+    const FT_ACCOUNT: &str = "test_token.root";
     const INITIAL_ETH_BALANCE: u64 = 777_777_777;
     const ETH_EXIT_AMOUNT: u64 = 111_111_111;
 

engine-tests/src/tests/eth_connector.rs

@@ -539,7 +539,7 @@ fn test_ft_transfer_call_without_message() {
     // Sending to external receiver with empty message should be success
     let dummy_ft_receiver = master_account.deploy(
         &dummy_ft_receiver_bytes(),
-        "ft-rec".parse().unwrap(),
+        "ft-rec.root".parse().unwrap(),
         near_sdk_sim::STORAGE_AMOUNT,
     );
     let res = recipient_account.call(

engine-tests/src/tests/one_inch.rs

@@ -23,11 +23,11 @@ fn test_1inch_liquidity_protocol() {
 
     let (result, profile, deployer_address) = helper.create_mooniswap_deployer();
     assert!(result.gas_used >= 5_100_000); // more than 5.1M EVM gas used
-    assert_gas_bound(profile.all_gas(), 12); // less than 12 NEAR Tgas used
+    assert_gas_bound(profile.all_gas(), 10); // less than 10 NEAR Tgas used
 
     let (result, profile, pool_factory) = helper.create_pool_factory(&deployer_address);
     assert!(result.gas_used >= 2_800_000); // more than 2.8M EVM gas used
-    assert_gas_bound(profile.all_gas(), 12); // less than 12 NEAR Tgas used
+    assert_gas_bound(profile.all_gas(), 9); // less than 9 NEAR Tgas used
 
     // create some ERC-20 tokens to have a liquidity pool for
     let signer_address = test_utils::address_from_secret_key(&helper.signer.secret_key);
@@ -39,7 +39,7 @@ fn test_1inch_liquidity_protocol() {
     let (result, profile, pool) =
         helper.create_pool(&pool_factory, token_a.0.address, token_b.0.address);
     assert!(result.gas_used >= 4_500_000); // more than 4.5M EVM gas used
-    assert_gas_bound(profile.all_gas(), 29);
+    assert_gas_bound(profile.all_gas(), 22);
 
     // Approve giving ERC-20 tokens to the pool
     helper.approve_erc20_tokens(&token_a, pool.address());
@@ -58,7 +58,7 @@ fn test_1inch_liquidity_protocol() {
         },
     );
     assert!(result.gas_used >= 302_000); // more than 302k EVM gas used
-    assert_gas_bound(profile.all_gas(), 35);
+    assert_gas_bound(profile.all_gas(), 25);
 
     // Same here
     helper.runner.context.block_timestamp += 10_000_000 * 1_000_000_000;
@@ -73,7 +73,7 @@ fn test_1inch_liquidity_protocol() {
         },
     );
     assert!(result.gas_used >= 210_000); // more than 210k EVM gas used
-    assert_gas_bound(profile.all_gas(), 37);
+    assert_gas_bound(profile.all_gas(), 27);
 
     let (result, profile) = helper.pool_withdraw(
         &pool,
@@ -84,7 +84,7 @@ fn test_1inch_liquidity_protocol() {
         },
     );
     assert!(result.gas_used >= 150_000); // more than 150k EVM gas used
-    assert_gas_bound(profile.all_gas(), 32);
+    assert_gas_bound(profile.all_gas(), 23);
 }
 
 #[test]
@@ -100,13 +100,13 @@ fn test_1_inch_limit_order_deploy() {
 
     // more than 3.5 million Ethereum gas used
     assert!(result.gas_used > 3_500_000);
-    // less than 12 NEAR Tgas used
-    assert_gas_bound(profile.all_gas(), 12);
+    // less than 10 NEAR Tgas used
+    assert_gas_bound(profile.all_gas(), 10);
     // at least 45% of which is from wasm execution
     let wasm_fraction = 100 * profile.wasm_gas() / profile.all_gas();
     assert!(
-        40 <= wasm_fraction && wasm_fraction <= 50,
-        "{}% is not between 40% and 50%",
+        50 <= wasm_fraction && wasm_fraction <= 60,
+        "{}% is not between 50% and 60%",
         wasm_fraction
     );
 }