Database backend (#133)

* DB backend

* DB backend

* Cleanup

* Clean build files after running tests

* Fixed comment

* add OOM lang item to runtime-io

.travis.yml

@@ -13,4 +13,5 @@ matrix:
 
 script:
  - cargo test --all
+ - cargo clean
  - ./publish-wasm.sh

Cargo.toml

@@ -27,6 +27,7 @@ members = [
  "polkadot/service",
  "substrate/bft",
  "substrate/client",
+ "substrate/client/db",
  "substrate/codec",
  "substrate/environmental",
  "substrate/executor",

polkadot/cli/Cargo.toml

@@ -21,6 +21,7 @@ app_dirs = "1.2"
 tokio-core = "0.1.12"
 futures = "0.1.17"
 ctrlc = { git = "https://github.com/paritytech/rust-ctrlc.git" }
+fdlimit = "0.1"
 substrate-client = { path = "../../substrate/client" }
 substrate-network = { path = "../../substrate/network" }
 substrate-codec = { path = "../../substrate/codec" }

polkadot/cli/src/lib.rs

@@ -27,6 +27,7 @@ extern crate time;
 extern crate futures;
 extern crate tokio_core;
 extern crate ctrlc;
+extern crate fdlimit;
 extern crate ed25519;
 extern crate triehash;
 extern crate substrate_codec as codec;
@@ -98,6 +99,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
  // TODO [ToDr] Split parameters parsing from actual execution.
  let log_pattern = matches.value_of("log").unwrap_or("");
  init_logger(log_pattern);
+ fdlimit::raise_fd_limit();
 
  let mut config = service::Configuration::default();
 
@@ -111,6 +113,8 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
  .to_string_lossy()
  .into();
 
+ config.database_path = db_path(&base_path).to_string_lossy().into();
+
  let mut role = service::Role::FULL;
  if matches.is_present("collator") {
  info!("Starting collator.");
@@ -204,6 +208,12 @@ fn keystore_path(base_path: &Path) -> PathBuf {
  path
 }
 
+fn db_path(base_path: &Path) -> PathBuf {
+ let mut path = base_path.to_owned();
+ path.push("db");
+ path
+}
+
 fn network_path(base_path: &Path) -> PathBuf {
  let mut path = base_path.to_owned();
  path.push("network");

polkadot/keystore/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 authors = ["Parity Technologies <robert@parity.io>"]
 
 [dependencies]
-ethcrypto = { git = "https://github.com/paritytech/parity", default_features = false }
+ethcore-crypto = { git = "https://github.com/paritytech/parity", default_features = false }
 ed25519 = { path = "../../substrate/ed25519" }
 error-chain = "0.11"
 hex = "0.3"

polkadot/keystore/src/lib.rs

@@ -16,7 +16,7 @@
 
 //! Keystore (and session key management) for polkadot.
 
-extern crate ethcrypto as crypto;
+extern crate ethcore_crypto as crypto;
 extern crate subtle;
 extern crate ed25519;
 extern crate rand;

polkadot/service/Cargo.toml

@@ -24,5 +24,6 @@ substrate-runtime-io = { path = "../../substrate/runtime-io" }
 substrate-primitives = { path = "../../substrate/primitives" }
 substrate-network = { path = "../../substrate/network" }
 substrate-client = { path = "../../substrate/client" }
+substrate-client-db = { path = "../../substrate/client/db" }
 substrate-codec = { path = "../../substrate/codec" }
 substrate-executor = { path = "../../substrate/executor" }

polkadot/service/src/config.rs

@@ -39,6 +39,8 @@ pub struct Configuration {
  pub network: NetworkConfiguration,
  /// Path to key files.
  pub keystore_path: String,
+ /// Path to the database.
+ pub database_path: String,
  /// Additional key seeds.
  pub keys: Vec<String>,
  /// Chain specification.
@@ -52,6 +54,7 @@ impl Default for Configuration {
  transaction_pool: Default::default(),
  network: Default::default(),
  keystore_path: Default::default(),
+ database_path: Default::default(),
  keys: Default::default(),
  chain_spec: ChainSpec::Development,
  }

polkadot/service/src/lib.rs

@@ -33,6 +33,7 @@ extern crate substrate_runtime_io as runtime_io;
 extern crate substrate_primitives as primitives;
 extern crate substrate_network as network;
 extern crate substrate_codec as codec;
+extern crate substrate_client_db as client_db;
 extern crate substrate_executor;
 
 extern crate exit_future;
@@ -64,14 +65,13 @@ use polkadot_api::PolkadotApi;
 use polkadot_runtime::{GenesisConfig, ConsensusConfig, CouncilConfig, DemocracyConfig,
  SessionConfig, StakingConfig, BuildExternalities};
 use client::{genesis, BlockchainEvents};
-use client::in_mem::Backend as InMemory;
 use network::ManageNetwork;
 use exit_future::Signal;
 
 pub use self::error::{ErrorKind, Error};
 pub use config::{Configuration, Role, ChainSpec};
 
-type Client = client::Client<InMemory, NativeExecutor<LocalDispatch>>;
+type Client = client::Client<client_db::Backend, NativeExecutor<LocalDispatch>>;
 
 /// Polkadot service.
 pub struct Service {
@@ -275,7 +275,12 @@ impl Service {
  (primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
  };
 
- let client = Arc::new(client::new_in_mem(executor, prepare_genesis)?);
+ let db_settings = client_db::DatabaseSettings {
+ cache_size: None,
+ path: config.database_path.into(),
+ };
+
+ let client = Arc::new(client_db::new_client(db_settings, executor, prepare_genesis)?);
  let best_header = client.best_block_header()?;
  info!("Starting Polkadot. Best block is #{}", best_header.number);
  let transaction_pool = Arc::new(Mutex::new(TransactionPool::new(config.transaction_pool)));

substrate/client/db/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "substrate-client-db"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+parking_lot = "0.4"
+log = "0.3"
+kvdb = { git = "https://github.com/paritytech/parity.git" }
+kvdb-rocksdb = { git = "https://github.com/paritytech/parity.git" }
+substrate-primitives = { path = "../../../substrate/primitives" }
+substrate-client = { path = "../../../substrate/client" }
+substrate-state-machine = { path = "../../../substrate/state-machine" }
+substrate-runtime-support = { path = "../../../substrate/runtime-support" }
+substrate-codec = { path = "../../../substrate/codec" }
+
+[dev-dependencies]