Update gas estimation code to use new data from NEAR protocol v53 (#517)

engine-standalone-storage/Cargo.toml

@@ -20,14 +20,14 @@ aurora-engine-sdk = { path = "../engine-sdk", default-features = false, features
 aurora-engine-transactions = { path = "../engine-transactions", default-features = false, features = ["std"] }
 borsh = { version = "0.8.2" }
 evm-core = { git = "https://github.com/aurora-is-near/sputnikvm.git", rev = "37448b6cacd98b06282cff5a559684505c29bd2b", default-features = false }
-rocksdb = { version = "0.16.0", default-features = false }
+rocksdb = { version = "0.18.0", default-features = false }
 postgres = "0.19.2"
 serde = "1.0.130"
 serde_json = "1.0.72"
 base64 = "0.13.0"
 
 [features]
-default = ["snappy", "lz4", "zstd", "zlib", "bzip2"]
+default = ["snappy", "lz4", "zstd", "zlib"]
 mainnet = []
 testnet = []
 snappy = ["rocksdb/snappy"]

engine-tests/Cargo.toml

@@ -35,13 +35,13 @@ ethabi = { git = "https://github.com/darwinia-network/ethabi", branch = "xavier-
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 hex = { version = "0.4.3", default-features = false }
-near-sdk = { git = "https://github.com/aurora-is-near/near-sdk-rs.git", rev = "5e58722bd61d9d24ae6293326146c751f0a814fb" }
-near-sdk-sim = { git = "https://github.com/aurora-is-near/near-sdk-rs.git", rev = "5e58722bd61d9d24ae6293326146c751f0a814fb" }
-near-crypto = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783" }
-near-vm-runner = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783", default-features = false, features = [ "wasmer2_vm" ] }
-near-vm-logic = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783" }
-near-primitives-core = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783" }
-near-primitives = { git = "https://github.com/near/nearcore.git", rev = "83fc0f7d6b212bacc49f058e7400743de3e59783" }
+near-sdk = { git = "https://github.com/aurora-is-near/near-sdk-rs.git", rev = "ba2eddbfbf4484ac3e44b4c8119bbac4907d6e07" }
+near-sdk-sim = { git = "https://github.com/aurora-is-near/near-sdk-rs.git", rev = "ba2eddbfbf4484ac3e44b4c8119bbac4907d6e07" }
+near-crypto = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda" }
+near-vm-runner = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda", default-features = false, features = [ "wasmer2_vm" ] }
+near-vm-logic = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda" }
+near-primitives-core = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda" }
+near-primitives = { git = "https://github.com/birchmd/nearcore.git", rev = "980bc48dc02878fea1e0dbc5812ae7de49f12dda" }
 libsecp256k1 = "0.3.5"
 rand = "0.7.3"
 criterion = "0.3.4"

engine-tests/src/test_utils/mocked_external.rs

@@ -1,43 +1,66 @@
 use near_vm_logic::mocks::mock_external::MockedExternal;
 
-pub const MAINNET_AVERAGE_TRIE_DEPTH: u64 = 10;
+/// Derived from mainnet data reported here: https://hackmd.io/@birchmd/r1HRjr0P9
+/// Uses the formulas:
+/// n_T = (G_T / G_R) * (g_R / g_T)
+/// n_c = (G_c / G_R) * (g_R / g_c)
+/// Where n_T is the average number of new touched trie nodes per read,
+/// n_c is the average number of cached trie nodes read per read,
+/// G_T is the average gas cost of touching trie node per Aurora transaction,
+/// G_c is the average gas cost of reading cached trie node per Aurora transaction,
+/// G_R is the average gas cost of `STORAGE_READ_BASE` per Aurora transaction,
+/// g_R is the `STORAGE_READ_BASE` cost (from the config),
+/// g_T is the `TOUCHING_TRIE_NODE` cost (from the config), and
+/// g_c is the `READ_CACHED_TRIE_NODE` cost (from the config).
+pub const MAINNET_AVERAGE_TOUCHED_TRIE_PER_READ: u64 = 2;
+pub const MAINNET_AVERAGE_READ_CACHED_TRIE_PER_READ: u64 = 11;
+/// This is still needed because writes will touch every node in the depth, unlike reads which take advantage of caching.
+pub const MAINNET_AVERAGE_TRIE_DEPTH: u64 = 13;
 
 #[derive(Clone)]
-pub(crate) struct MockedExternalWithTrie {
+pub struct MockedExternalWithTrie {
  pub underlying: MockedExternal,
- trie_node_count: std::cell::Cell<u64>,
+ new_trie_node_count: std::cell::Cell<u64>,
+ cached_trie_node_count: std::cell::Cell<u64>,
 }
 
 impl MockedExternalWithTrie {
  pub fn new(ext: MockedExternal) -> Self {
  Self {
  underlying: ext,
- trie_node_count: std::cell::Cell::new(0),
+ new_trie_node_count: std::cell::Cell::new(0),
+ cached_trie_node_count: std::cell::Cell::new(0),
  }
  }
 
- fn increment_trie_node_count(&self, amount: u64) {
- let cell_value = self.trie_node_count.get();
- self.trie_node_count.set(cell_value + amount);
+ fn increment_new_trie_node_count(&self, amount: u64) {
+ let cell_value = self.new_trie_node_count.get();
+ self.new_trie_node_count.set(cell_value + amount);
+ }
+
+ fn increment_cached_trie_node_count(&self, amount: u64) {
+ let cell_value = self.cached_trie_node_count.get();
+ self.cached_trie_node_count.set(cell_value + amount);
  }
 }
 
 impl near_vm_logic::External for MockedExternalWithTrie {
  fn storage_set(&mut self, key: &[u8], value: &[u8]) -> Result<(), near_vm_logic::VMLogicError> {
- self.increment_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
+ self.increment_new_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
  self.underlying.storage_set(key, value)
  }
 
  fn storage_get<'a>(
  &'a self,
  key: &[u8],
  ) -> Result<Option<Box<dyn near_vm_logic::ValuePtr + 'a>>, near_vm_logic::VMLogicError> {
- self.increment_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
+ self.increment_new_trie_node_count(MAINNET_AVERAGE_TOUCHED_TRIE_PER_READ);
+ self.increment_cached_trie_node_count(MAINNET_AVERAGE_READ_CACHED_TRIE_PER_READ);
  self.underlying.storage_get(key)
  }
 
  fn storage_remove(&mut self, key: &[u8]) -> Result<(), near_vm_logic::VMLogicError> {
- self.increment_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
+ self.increment_new_trie_node_count(MAINNET_AVERAGE_TRIE_DEPTH);
  self.underlying.storage_remove(key)
  }
 
@@ -49,117 +72,6 @@ impl near_vm_logic::External for MockedExternalWithTrie {
  self.underlying.storage_has_key(key)
  }
 
- fn create_receipt(
- &mut self,
- receipt_indices: Vec<near_vm_logic::types::ReceiptIndex>,
- receiver_id: near_primitives::types::AccountId,
- ) -> Result<near_vm_logic::types::ReceiptIndex, near_vm_logic::VMLogicError> {
- self.underlying.create_receipt(receipt_indices, receiver_id)
- }
-
- fn append_action_create_account(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying.append_action_create_account(receipt_index)
- }
-
- fn append_action_deploy_contract(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- code: Vec<u8>,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying
- .append_action_deploy_contract(receipt_index, code)
- }
-
- fn append_action_function_call(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- method_name: Vec<u8>,
- arguments: Vec<u8>,
- attached_deposit: near_primitives::types::Balance,
- prepaid_gas: near_primitives::types::Gas,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying.append_action_function_call(
- receipt_index,
- method_name,
- arguments,
- attached_deposit,
- prepaid_gas,
- )
- }
-
- fn append_action_transfer(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- amount: near_primitives::types::Balance,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying
- .append_action_transfer(receipt_index, amount)
- }
-
- fn append_action_stake(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- stake: near_primitives::types::Balance,
- public_key: near_vm_logic::types::PublicKey,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying
- .append_action_stake(receipt_index, stake, public_key)
- }
-
- fn append_action_add_key_with_full_access(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- public_key: near_vm_logic::types::PublicKey,
- nonce: u64,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying
- .append_action_add_key_with_full_access(receipt_index, public_key, nonce)
- }
-
- fn append_action_add_key_with_function_call(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- public_key: near_vm_logic::types::PublicKey,
- nonce: u64,
- allowance: Option<near_primitives::types::Balance>,
- receiver_id: near_primitives::types::AccountId,
- method_names: Vec<Vec<u8>>,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying.append_action_add_key_with_function_call(
- receipt_index,
- public_key,
- nonce,
- allowance,
- receiver_id,
- method_names,
- )
- }
-
- fn append_action_delete_key(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- public_key: near_vm_logic::types::PublicKey,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying
- .append_action_delete_key(receipt_index, public_key)
- }
-
- fn append_action_delete_account(
- &mut self,
- receipt_index: near_vm_logic::types::ReceiptIndex,
- beneficiary_id: near_primitives::types::AccountId,
- ) -> Result<(), near_vm_logic::VMLogicError> {
- self.underlying
- .append_action_delete_account(receipt_index, beneficiary_id)
- }
-
- fn get_touched_nodes_count(&self) -> u64 {
- self.trie_node_count.get()
- }
-
  fn validator_stake(
  &self,
  account_id: &near_primitives::types::AccountId,
@@ -172,4 +84,17 @@ impl near_vm_logic::External for MockedExternalWithTrie {
  ) -> Result<near_primitives::types::Balance, near_vm_logic::VMLogicError> {
  self.underlying.validator_total_stake()
  }
+
+ fn generate_data_id(&mut self) -> near_primitives::hash::CryptoHash {
+ self.underlying.generate_data_id()
+ }
+
+ fn get_trie_nodes_count(&self) -> near_primitives::types::TrieNodesCount {
+ let db_reads = self.new_trie_node_count.get();
+ let mem_reads = self.cached_trie_node_count.get();
+ near_primitives::types::TrieNodesCount {
+ db_reads,
+ mem_reads,
+ }
+ }
 }

engine-tests/src/test_utils/mod.rs

@@ -126,7 +126,7 @@ impl<'a> OneShotAuroraRunner<'a> {
  input,
  );
 
- near_vm_runner::run(
+ match near_vm_runner::run(
  &self.base.code,
  method_name,
  &mut self.ext,
@@ -136,7 +136,10 @@ impl<'a> OneShotAuroraRunner<'a> {
  &[],
  self.base.current_protocol_version,
  Some(&self.base.cache),
- )
+ ) {
+ near_vm_runner::VMResult::Aborted(outcome, error) => (Some(outcome), Some(error)),
+ near_vm_runner::VMResult::Ok(outcome) => (Some(outcome), None),
+ }
  }
 }
 
@@ -185,7 +188,7 @@ impl AuroraRunner {
  input,
  );
 
- let (maybe_outcome, maybe_error) = near_vm_runner::run(
+ let (maybe_outcome, maybe_error) = match near_vm_runner::run(
  &self.code,
  method_name,
  &mut self.ext,
@@ -195,7 +198,10 @@ impl AuroraRunner {
  &[],
  self.current_protocol_version,
  Some(&self.cache),
- );
+ ) {
+ near_vm_runner::VMResult::Aborted(outcome, error) => (Some(outcome), Some(error)),
+ near_vm_runner::VMResult::Ok(outcome) => (Some(outcome), None),
+ };
  if let Some(outcome) = &maybe_outcome {
  self.context.storage_usage = outcome.storage_usage;
  self.previous_logs = outcome.logs.clone();

engine-tests/src/tests/erc20.rs

@@ -108,12 +108,12 @@ fn profile_erc20_get_balance() {
  assert!(result.is_ok());
 
  // call costs less than 3 Tgas
- test_utils::assert_gas_bound(profile.all_gas(), 3);
+ test_utils::assert_gas_bound(profile.all_gas(), 2);
  // at least 70% of the cost is spent on wasm computation (as opposed to host functions)
  let wasm_fraction = (100 * profile.wasm_gas()) / profile.all_gas();
  assert!(
- 15 <= wasm_fraction && wasm_fraction <= 20,
- "{}% is not between 15% and 20%",
+ 20 <= wasm_fraction && wasm_fraction <= 30,
+ "{}% is not between 20% and 30%",
  wasm_fraction
  );
 }

engine-tests/src/tests/erc20_connector.rs

@@ -410,7 +410,7 @@ mod sim_tests {
  const FT_TOTAL_SUPPLY: u128 = 1_000_000;
  const FT_TRANSFER_AMOUNT: u128 = 300_000;
  const FT_EXIT_AMOUNT: u128 = 100_000;
- const FT_ACCOUNT: &str = "test_token";
+ const FT_ACCOUNT: &str = "test_token.root";
  const INITIAL_ETH_BALANCE: u64 = 777_777_777;
  const ETH_EXIT_AMOUNT: u64 = 111_111_111;
 

engine-tests/src/tests/eth_connector.rs

@@ -539,7 +539,7 @@ fn test_ft_transfer_call_without_message() {
  // Sending to external receiver with empty message should be success
  let dummy_ft_receiver = master_account.deploy(
  &dummy_ft_receiver_bytes(),
- "ft-rec".parse().unwrap(),
+ "ft-rec.root".parse().unwrap(),
  near_sdk_sim::STORAGE_AMOUNT,
  );
  let res = recipient_account.call(

engine-tests/src/tests/one_inch.rs

@@ -23,11 +23,11 @@ fn test_1inch_liquidity_protocol() {
 
  let (result, profile, deployer_address) = helper.create_mooniswap_deployer();
  assert!(result.gas_used >= 5_100_000); // more than 5.1M EVM gas used
- assert_gas_bound(profile.all_gas(), 12); // less than 12 NEAR Tgas used
+ assert_gas_bound(profile.all_gas(), 10); // less than 10 NEAR Tgas used
 
  let (result, profile, pool_factory) = helper.create_pool_factory(&deployer_address);
  assert!(result.gas_used >= 2_800_000); // more than 2.8M EVM gas used
- assert_gas_bound(profile.all_gas(), 12); // less than 12 NEAR Tgas used
+ assert_gas_bound(profile.all_gas(), 9); // less than 9 NEAR Tgas used
 
  // create some ERC-20 tokens to have a liquidity pool for
  let signer_address = test_utils::address_from_secret_key(&helper.signer.secret_key);
@@ -39,7 +39,7 @@ fn test_1inch_liquidity_protocol() {
  let (result, profile, pool) =
  helper.create_pool(&pool_factory, token_a.0.address, token_b.0.address);
  assert!(result.gas_used >= 4_500_000); // more than 4.5M EVM gas used
- assert_gas_bound(profile.all_gas(), 29);
+ assert_gas_bound(profile.all_gas(), 22);
 
  // Approve giving ERC-20 tokens to the pool
  helper.approve_erc20_tokens(&token_a, pool.address());
@@ -58,7 +58,7 @@ fn test_1inch_liquidity_protocol() {
  },
  );
  assert!(result.gas_used >= 302_000); // more than 302k EVM gas used
- assert_gas_bound(profile.all_gas(), 35);
+ assert_gas_bound(profile.all_gas(), 25);
 
  // Same here
  helper.runner.context.block_timestamp += 10_000_000 * 1_000_000_000;
@@ -73,7 +73,7 @@ fn test_1inch_liquidity_protocol() {
  },
  );
  assert!(result.gas_used >= 210_000); // more than 210k EVM gas used
- assert_gas_bound(profile.all_gas(), 37);
+ assert_gas_bound(profile.all_gas(), 27);
 
  let (result, profile) = helper.pool_withdraw(
  &pool,
@@ -84,7 +84,7 @@ fn test_1inch_liquidity_protocol() {
  },
  );
  assert!(result.gas_used >= 150_000); // more than 150k EVM gas used
- assert_gas_bound(profile.all_gas(), 32);
+ assert_gas_bound(profile.all_gas(), 23);
 }
 
 #[test]
@@ -100,13 +100,13 @@ fn test_1_inch_limit_order_deploy() {
 
  // more than 3.5 million Ethereum gas used
  assert!(result.gas_used > 3_500_000);
- // less than 12 NEAR Tgas used
- assert_gas_bound(profile.all_gas(), 12);
+ // less than 10 NEAR Tgas used
+ assert_gas_bound(profile.all_gas(), 10);
  // at least 45% of which is from wasm execution
  let wasm_fraction = 100 * profile.wasm_gas() / profile.all_gas();
  assert!(
- 40 <= wasm_fraction && wasm_fraction <= 50,
- "{}% is not between 40% and 50%",
+ 50 <= wasm_fraction && wasm_fraction <= 60,
+ "{}% is not between 50% and 60%",
  wasm_fraction
  );
 }