WIP on chain heap

demo/cli/src/lib.rs

@@ -155,7 +155,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
  init_logger(log_pattern);
 
  // Create client
- let executor = NativeExecutor::with_heap_pages(8);
+ let executor = NativeExecutor::new();
 
  let god_key = hex!["3d866ec8a9190c8343c2fc593d21d8a6d0c5c4763aaab2349de3a6111d64d124"];
  let genesis_config = GenesisConfig {

demo/executor/src/lib.rs

@@ -98,7 +98,7 @@ mod tests {
  }
 
  fn executor() -> ::substrate_executor::NativeExecutor<Executor> {
- ::substrate_executor::NativeExecutor::with_heap_pages(8)
+ ::substrate_executor::NativeExecutor::new()
  }
 
  #[test]
@@ -114,9 +114,9 @@ mod tests {
  twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
  ];
 
- let r = executor().call(&mut t, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
+ let r = executor().call(&mut t, 8, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
  assert!(r.is_ok());
- let v = executor().call(&mut t, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap();
+ let v = executor().call(&mut t, 8, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap();
  let r = ApplyResult::decode(&mut &v[..]).unwrap();
  assert_eq!(r, Err(ApplyError::CantPay));
  }
@@ -134,9 +134,9 @@ mod tests {
  twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
  ];
 
- let r = executor().call(&mut t, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
+ let r = executor().call(&mut t, 8, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
  assert!(r.is_ok());
- let v = executor().call(&mut t, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap();
+ let v = executor().call(&mut t, 8, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap();
  let r = ApplyResult::decode(&mut &v[..]).unwrap();
  assert_eq!(r, Err(ApplyError::CantPay));
  }
@@ -154,9 +154,9 @@ mod tests {
  twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
  ];
 
- let r = executor().call(&mut t, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
+ let r = executor().call(&mut t, 8, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
  assert!(r.is_ok());
- let r = executor().call(&mut t, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0;
+ let r = executor().call(&mut t, 8, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0;
  assert!(r.is_ok());
 
  runtime_io::with_externalities(&mut t, || {
@@ -178,9 +178,9 @@ mod tests {
  twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
  ];
 
- let r = executor().call(&mut t, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
+ let r = executor().call(&mut t, 8, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
  assert!(r.is_ok());
- let r = executor().call(&mut t, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0;
+ let r = executor().call(&mut t, 8, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0;
  assert!(r.is_ok());
 
  runtime_io::with_externalities(&mut t, || {
@@ -312,7 +312,7 @@ mod tests {
  fn full_native_block_import_works() {
  let mut t = new_test_ext();
 
- executor().call(&mut t, COMPACT_CODE, "execute_block", &block1().0, true).0.unwrap();
+ executor().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1().0, true).0.unwrap();
 
  runtime_io::with_externalities(&mut t, || {
  assert_eq!(Balances::total_balance(&alice()), 41);
@@ -329,7 +329,7 @@ mod tests {
  ]);
  });
 
- executor().call(&mut t, COMPACT_CODE, "execute_block", &block2().0, true).0.unwrap();
+ executor().call(&mut t, 8, COMPACT_CODE, "execute_block", &block2().0, true).0.unwrap();
 
  runtime_io::with_externalities(&mut t, || {
  assert_eq!(Balances::total_balance(&alice()), 30);
@@ -359,14 +359,14 @@ mod tests {
  fn full_wasm_block_import_works() {
  let mut t = new_test_ext();
 
- WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block1().0).unwrap();
+ WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1().0).unwrap();
 
  runtime_io::with_externalities(&mut t, || {
  assert_eq!(Balances::total_balance(&alice()), 41);
  assert_eq!(Balances::total_balance(&bob()), 69);
  });
 
- WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block2().0).unwrap();
+ WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block2().0).unwrap();
 
  runtime_io::with_externalities(&mut t, || {
  assert_eq!(Balances::total_balance(&alice()), 30);
@@ -378,23 +378,23 @@ mod tests {
  fn wasm_big_block_import_fails() {
  let mut t = new_test_ext();
 
- let r = WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0);
+ let r = WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0);
  assert!(!r.is_ok());
  }
 
  #[test]
  fn native_big_block_import_succeeds() {
  let mut t = new_test_ext();
 
- let r = Executor::with_heap_pages(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, true).0;
+ let r = Executor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0, true).0;
  assert!(r.is_ok());
  }
 
  #[test]
  fn native_big_block_import_fails_on_fallback() {
  let mut t = new_test_ext();
 
- let r = Executor::with_heap_pages(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, false).0;
+ let r = Executor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0, false).0;
  assert!(!r.is_ok());
  }
 
@@ -412,9 +412,9 @@ mod tests {
  ];
 
  let foreign_code = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.wasm");
- let r = WasmExecutor::new(8).call(&mut t, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64)));
+ let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64)));
  assert!(r.is_ok());
- let r = WasmExecutor::new(8).call(&mut t, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap();
+ let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap();
  let r = ApplyResult::decode(&mut &r[..]).unwrap();
  assert_eq!(r, Err(ApplyError::CantPay));
  }
@@ -433,9 +433,9 @@ mod tests {
  ];
 
  let foreign_code = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.compact.wasm");
- let r = WasmExecutor::new(8).call(&mut t, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64)));
+ let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64)));
  assert!(r.is_ok());
- let r = WasmExecutor::new(8).call(&mut t, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap();
+ let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap();
  let r = ApplyResult::decode(&mut &r[..]).unwrap();
  assert_eq!(r, Ok(ApplyOutcome::Success));
 

substrate/cli/src/cli.yml

@@ -125,10 +125,6 @@ args:
  long: execution
  value_name: STRATEGY
  help: The means of execution used when calling into the runtime. Can be either wasm, native or both.
- - max-heap-pages:
- long: max-heap-pages
- value_name: COUNT
- help: The maximum number of 64KB pages to ever allocate for Wasm execution. Don't alter this unless you know what you're doing.
 subcommands:
  - build-spec:
  about: Build a spec.json file, outputing to stdout

substrate/cli/src/lib.rs

@@ -266,10 +266,6 @@ where
  service::Roles::FULL
  };
 
- if let Some(v) = matches.value_of("max-heap-pages") {
- config.max_heap_pages = v.parse().map_err(|_| "Invalid --max-heap-pages argument")?;
- }
-
  if let Some(s) = matches.value_of("execution") {
  config.execution_strategy = match s {
  "both" => service::ExecutionStrategy::Both,
@@ -398,10 +394,6 @@ fn import_blocks<F, E>(matches: &clap::ArgMatches, spec: ChainSpec<FactoryGenesi
  let mut config = service::Configuration::default_with_spec(spec);
  config.database_path = db_path(&base_path, config.chain_spec.id()).to_string_lossy().into();
 
- if let Some(v) = matches.value_of("max-heap-pages") {
- config.max_heap_pages = v.parse().map_err(|_| "Invalid --max-heap-pages argument")?;
- }
-
  if let Some(s) = matches.value_of("execution") {
  config.execution_strategy = match s {
  "both" => service::ExecutionStrategy::Both,

substrate/client/src/call_executor.rs

@@ -26,6 +26,7 @@ use patricia_trie::NodeCodec;
 use primitives::{KeccakHasher, RlpCodec};
 use hashdb::Hasher;
 use rlp::Encodable;
+use codec::Decode;
 
 use backend;
 use error;
@@ -145,8 +146,9 @@ where
  let mut externalities = Ext::new(&mut overlay, &state);
  let code = externalities.storage(b":code").ok_or(error::ErrorKind::VersionInvalid)?
  .to_vec();
+ let heap_pages = externalities.storage(b":heappages").and_then(|v| u64::decode(&mut &v[..])).unwrap_or(8) as usize;
 
- self.executor.runtime_version(&mut externalities, &code)
+ self.executor.runtime_version(&mut externalities, heap_pages, &code)
  .ok_or(error::ErrorKind::VersionInvalid.into())
  }
 

substrate/client/src/genesis.rs

@@ -56,7 +56,7 @@ mod tests {
  native_executor_instance!(Executor, test_client::runtime::api::dispatch, test_client::runtime::VERSION, include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm"));
 
  fn executor() -> ::executor::NativeExecutor<Executor> {
- NativeExecutionDispatch::with_heap_pages(8)
+ NativeExecutionDispatch::new()
  }
 
  fn construct_block(backend: &InMemory<KeccakHasher, RlpCodec>, number: BlockNumber, parent_hash: Hash, state_root: Hash, txs: Vec<Transfer>) -> (Vec<u8>, Hash) {
@@ -194,7 +194,7 @@ mod tests {
  let _ = execute(
  &backend,
  &mut overlay,
- &Executor::with_heap_pages(8),
+ &Executor::new(),
  "execute_block",
  &b1data,
  ExecutionStrategy::NativeWhenPossible,

substrate/client/src/light/backend.rs

@@ -270,7 +270,7 @@ pub mod tests {
  let remote_read_proof = remote_client.read_proof(&remote_block_id, b":auth:len").unwrap();
 
  // check remote read proof locally
- let local_executor = test_client::LocalExecutor::with_heap_pages(8);
+ let local_executor = test_client::LocalExecutor::new();
  let local_checker = new_fetch_checker::<_, KeccakHasher, RlpCodec>(local_executor);
  let request = RemoteReadRequest {
  block: remote_block_hash,

substrate/client/src/light/call_executor.rs

@@ -156,7 +156,7 @@ mod tests {
  let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap().1;
 
  // check remote execution proof locally
- let local_executor = test_client::LocalExecutor::with_heap_pages(8);
+ let local_executor = test_client::LocalExecutor::new();
  check_execution_proof::<_, _, _, RlpCodec>(&local_executor, &RemoteCallRequest {
  block: test_client::runtime::Hash::default(),
  header: test_client::runtime::Header {

substrate/executor/src/lib.rs

@@ -87,6 +87,7 @@ pub trait RuntimeInfo {
  fn runtime_version<E: Externalities<KeccakHasher>> (
  &self,
  ext: &mut E,
+ heap_pages: usize,
  code: &[u8]
  ) -> Option<RuntimeVersion>;
 }

substrate/executor/src/native_executor.rs

@@ -55,12 +55,13 @@ fn fetch_cached_runtime_version<'a, E: Externalities<KeccakHasher>>(
  wasm_executor: &WasmExecutor,
  cache: &'a mut MutexGuard<CacheType>,
  ext: &mut E,
+ heap_pages: usize,
  code: &[u8]
 ) -> Result<(&'a WasmModule, &'a Option<RuntimeVersion>)> {
  let maybe_runtime_preproc = cache.entry(gen_cache_key(code))
  .or_insert_with(|| match WasmModule::from_buffer(code) {
  Ok(module) => {
- let version = wasm_executor.call_in_wasm_module(ext, &module, "version", &[])
+ let version = wasm_executor.call_in_wasm_module(ext, heap_pages, &module, "version", &[])
  .ok()
  .and_then(|v| RuntimeVersion::decode(&mut v.as_slice()));
  RuntimePreproc::ValidCode(module, version)
@@ -108,9 +109,9 @@ pub trait NativeExecutionDispatch: Send + Sync {
  /// Get native runtime version.
  const VERSION: RuntimeVersion;
 
- /// Construct corresponding `NativeExecutor` with given `heap_pages`.
- fn with_heap_pages(max_heap_pages: usize) -> NativeExecutor<Self> where Self: Sized {
- NativeExecutor::with_heap_pages(max_heap_pages)
+ /// Construct corresponding `NativeExecutor`
+ fn new() -> NativeExecutor<Self> where Self: Sized {
+ NativeExecutor::new()
  }
 }
 
@@ -125,11 +126,11 @@ pub struct NativeExecutor<D: NativeExecutionDispatch> {
 }
 
 impl<D: NativeExecutionDispatch> NativeExecutor<D> {
- /// Create new instance with specific number of pages for wasm fallback's heap.
- pub fn with_heap_pages(max_heap_pages: usize) -> Self {
+ /// Create new instance.
+ pub fn new() -> Self {
  NativeExecutor {
  _dummy: Default::default(),
- fallback: WasmExecutor::new(max_heap_pages),
+ fallback: WasmExecutor::new(),
  }
  }
 }
@@ -149,9 +150,10 @@ impl<D: NativeExecutionDispatch> RuntimeInfo for NativeExecutor<D> {
  fn runtime_version<E: Externalities<KeccakHasher>>(
  &self,
  ext: &mut E,
+ heap_pages: usize,
  code: &[u8],
  ) -> Option<RuntimeVersion> {
- fetch_cached_runtime_version(&self.fallback, &mut RUNTIMES_CACHE.lock(), ext, code).ok()?.1.clone()
+ fetch_cached_runtime_version(&self.fallback, &mut RUNTIMES_CACHE.lock(), ext, heap_pages, code).ok()?.1.clone()
  }
 }
 
@@ -161,23 +163,24 @@ impl<D: NativeExecutionDispatch> CodeExecutor<KeccakHasher> for NativeExecutor<D
  fn call<E: Externalities<KeccakHasher>>(
  &self,
  ext: &mut E,
+ heap_pages: usize,
  code: &[u8],
  method: &str,
  data: &[u8],
  use_native: bool,
  ) -> (Result<Vec<u8>>, bool) {
  let mut c = RUNTIMES_CACHE.lock();
- let (module, onchain_version) = match fetch_cached_runtime_version(&self.fallback, &mut c, ext, code) {
+ let (module, onchain_version) = match fetch_cached_runtime_version(&self.fallback, &mut c, ext, heap_pages, code) {
  Ok((module, onchain_version)) => (module, onchain_version),
  Err(_) => return (Err(ErrorKind::InvalidCode(code.into()).into()), false),
  };
  match (use_native, onchain_version.as_ref().map_or(false, |v| v.can_call_with(&D::VERSION))) {
  (_, false) => {
  trace!(target: "executor", "Request for native execution failed (native: {}, chain: {})", D::VERSION, onchain_version.as_ref().map_or_else(||"<None>".into(), |v| format!("{}", v)));
- (self.fallback.call_in_wasm_module(ext, module, method, data), false)
+ (self.fallback.call_in_wasm_module(ext, heap_pages, module, method, data), false)
  }
  (false, _) => {
- (self.fallback.call_in_wasm_module(ext, module, method, data), false)
+ (self.fallback.call_in_wasm_module(ext, heap_pages, module, method, data), false)
  }
  _ => {
  trace!(target: "executor", "Request for native execution succeeded (native: {}, chain: {})", D::VERSION, onchain_version.as_ref().map_or_else(||"<None>".into(), |v| format!("{}", v)));
@@ -213,8 +216,8 @@ macro_rules! native_executor_instance {
  .ok_or_else(|| $crate::error::ErrorKind::MethodNotFound(method.to_owned()).into())
  }
 
- fn with_heap_pages(max_heap_pages: usize) -> $crate::NativeExecutor<$name> {
- $crate::NativeExecutor::with_heap_pages(max_heap_pages)
+ fn new() -> $crate::NativeExecutor<$name> {
+ $crate::NativeExecutor::new()
  }
  }
  }