From 7d293f845c6db6c95bfd01bf694b1b71a853ddcb Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Tue, 5 Jan 2021 18:12:50 +0800 Subject: [PATCH 01/10] [config]Configuration optimization and synchronization for consistency --- Cargo.lock | 45 ++++++++--------- config/Cargo.toml | 1 + config/src/account_vault_config.rs | 5 +- config/src/lib.rs | 2 - config/src/logger_config.rs | 33 +++++++------ config/src/metrics_config.rs | 14 +++--- config/src/miner_config.rs | 29 +++++------ config/src/network_config.rs | 2 +- config/src/rpc_config.rs | 32 ++++++++++-- config/src/storage_config.rs | 19 +++++-- config/src/sync_config.rs | 55 +++++++-------------- config/src/txpool_config.rs | 6 ++- miner/src/generate_block_event_pacemaker.rs | 4 +- miner/src/lib.rs | 4 +- miner/tests/miner_test.rs | 4 +- node/src/node.rs | 6 +-- rpc/client/tests/client_server_test.rs | 6 +-- sync/tests/full_sync_test.rs | 6 +-- sync/tests/test_sync/mod.rs | 2 +- sync/tests/txn_sync_test.rs | 4 +- 20 files changed, 156 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d679abfba..7016344f67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,7 +630,7 @@ dependencies = [ "parking_lot 0.11.1", "proptest", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "starcoin-account-api", "starcoin-accumulator", "starcoin-chain", @@ -2192,7 +2192,7 @@ dependencies = [ "proptest", "proptest-derive", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "starcoin-canonical-serialization", "starcoin-crypto", @@ -5824,13 +5824,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76330fb486679b4ace3670f117bbc9e16204005c4bde9c4bd372f45bed34f12" +checksum = "c24fcd450d3fa2b592732565aa4f17a27a61c65ece4726353e000939b0edee34" dependencies = [ "libc", "rand_chacha 0.3.0", - "rand_core 0.6.0", + "rand_core 0.6.1", ] [[package]] @@ -5860,7 +5860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" dependencies = [ "ppv-lite86", - "rand_core 0.6.0", + "rand_core 0.6.1", ] [[package]] @@ -5889,9 +5889,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8b34ba8cfb21243bd8df91854c830ff0d785fff2e82ebd4434c2644cb9ada18" +checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" dependencies = [ "getrandom 0.2.0", ] @@ -5903,7 +5903,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da9e8f32ad24fb80d07d2323a9a2ce8b30d68a62b8cb4df88119ff49a698f038" dependencies = [ "num-traits", - "rand 0.8.0", + "rand 0.8.1", ] [[package]] @@ -6926,7 +6926,7 @@ dependencies = [ "futures 0.3.8", "hex", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "serde_bytes", "starcoin-crypto", @@ -6946,7 +6946,7 @@ dependencies = [ "hex", "parking_lot 0.11.1", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "starcoin-account-api", "starcoin-canonical-serialization", @@ -6997,7 +6997,7 @@ dependencies = [ "proptest", "proptest-derive", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "starcoin-canonical-serialization", "starcoin-crypto", @@ -7094,7 +7094,7 @@ dependencies = [ "futures 0.3.8", "network-api", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "starcoin-crypto", "starcoin-service-registry", @@ -7169,7 +7169,7 @@ dependencies = [ "dashmap 4.0.1", "futures 0.3.8", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "starcoin-chain", "starcoin-chain-api", @@ -7237,6 +7237,7 @@ name = "starcoin-config" version = "0.9.4" dependencies = [ "anyhow", + "clap 2.33.3", "dirs 3.0.1", "git-version", "hex", @@ -7246,7 +7247,7 @@ dependencies = [ "num_enum", "once_cell", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "serde_json", "starcoin-crypto", @@ -7272,7 +7273,7 @@ dependencies = [ "proptest", "proptest-derive", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "rust-argon2", "sha3", "starcoin-crypto", @@ -7296,7 +7297,7 @@ dependencies = [ "libra-crypto-derive", "once_cell", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "serde-name", "serde_bytes", @@ -7337,7 +7338,7 @@ dependencies = [ "hmac 0.7.1", "pbkdf2", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "sha2 0.8.2", ] @@ -7637,7 +7638,7 @@ dependencies = [ "jsonrpc-tcp-server", "libloading 0.6.5", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "rust-argon2", "serde", "serde_json", @@ -8386,7 +8387,7 @@ dependencies = [ "proptest", "proptest-derive", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "serde", "serde_derive", "starcoin-config", @@ -8456,7 +8457,7 @@ dependencies = [ "proptest-derive", "prost 0.7.0", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "ref-cast", "serde", "starcoin-accumulator", @@ -8491,7 +8492,7 @@ dependencies = [ "once_cell", "prometheus", "rand 0.7.3", - "rand_core 0.6.0", + "rand_core 0.6.1", "starcoin-canonical-serialization", "starcoin-config", "starcoin-crypto", diff --git a/config/Cargo.toml b/config/Cargo.toml index c4997700bf..210c4e6c75 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -7,6 +7,7 @@ publish = false edition = "2018" [dependencies] +clap = "2.33.3" dirs = "3" anyhow = "1.0.37" thiserror = "1.0" diff --git a/config/src/account_vault_config.rs b/config/src/account_vault_config.rs index f9ffea7f0c..2ae0b46dee 100644 --- a/config/src/account_vault_config.rs +++ b/config/src/account_vault_config.rs @@ -5,10 +5,13 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use structopt::StructOpt; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct AccountVaultConfig { + #[structopt(long = "vault-dir", parse(from_os_str), conflicts_with("vault-dir"))] + /// Account vault dir config. dir: PathBuf, #[serde(skip)] absolute_dir: Option, diff --git a/config/src/lib.rs b/config/src/lib.rs index 8c991c8097..723af6b42b 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -444,9 +444,7 @@ impl NodeConfig { } } }; - config.after_load(opt, &base)?; - save_config(&config, &config_file_path)?; Ok(config) } diff --git a/config/src/logger_config.rs b/config/src/logger_config.rs index fcb26e049b..7bb797489f 100644 --- a/config/src/logger_config.rs +++ b/config/src/logger_config.rs @@ -5,15 +5,20 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use structopt::StructOpt; static LOGGER_FILE_NAME: &str = "starcoin.log"; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct LoggerConfig { - pub enable_stderr: bool, - pub enable_file: bool, + #[structopt(name = "disable-stderr", long, help = "disable stderr logger")] + pub disable_stderr: bool, + #[structopt(name = "disable-file", long, help = "disable file logger")] + pub disable_file: bool, + #[structopt(name = "max-file-size", long, default_value = "1073741824")] pub max_file_size: u64, + #[structopt(name = "max-backup", long, default_value = "7")] pub max_backup: u32, #[serde(skip)] log_path: Option, @@ -28,35 +33,35 @@ impl LoggerConfig { } pub fn enable_file(&self) -> bool { - self.enable_file && self.log_path.is_some() + (!self.disable_file) && self.log_path.is_some() } } impl ConfigModule for LoggerConfig { fn default_with_opt(opt: &StarcoinOpt, base: &BaseConfig) -> Result { - let enable_stderr = !opt.disable_std_log; - let enable_file = !opt.disable_file_log; + let disable_stderr = opt.disable_std_log; + let disable_file = opt.disable_file_log; Ok(if base.net.is_test() { Self { - enable_stderr, - enable_file, + disable_stderr, + disable_file, max_file_size: 10 * 1024 * 1024, max_backup: 1, log_path: None, } } else if base.net.is_dev() { Self { - enable_stderr, - enable_file, + disable_stderr, + disable_file, max_file_size: 10 * 1024 * 1024, max_backup: 2, log_path: None, } } else { Self { - enable_stderr, - enable_file, + disable_stderr, + disable_file, max_file_size: 1024 * 1024 * 1024, max_backup: 7, log_path: None, @@ -66,8 +71,8 @@ impl ConfigModule for LoggerConfig { fn after_load(&mut self, opt: &StarcoinOpt, base: &BaseConfig) -> Result<()> { self.log_path = Some(base.data_dir.join(LOGGER_FILE_NAME)); - self.enable_stderr = !opt.disable_std_log; - self.enable_file = !opt.disable_file_log; + self.disable_stderr = opt.disable_std_log; + self.disable_file = opt.disable_file_log; Ok(()) } } diff --git a/config/src/metrics_config.rs b/config/src/metrics_config.rs index a3bdf01b6a..254cfeb6ec 100644 --- a/config/src/metrics_config.rs +++ b/config/src/metrics_config.rs @@ -6,12 +6,16 @@ use crate::{ }; use anyhow::Result; use serde::{Deserialize, Serialize}; +use structopt::StructOpt; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct MetricsConfig { - pub enable_metrics: bool, + #[structopt(name = "disable-metrics", long, help = "disable metrics")] + pub disable_metrics: bool, + #[structopt(name = "address", long, help = "address", default_value = "0.0.0.0")] pub address: String, + #[structopt(name = "metrics-port", long, default_value = "9101")] pub port: u16, } @@ -27,16 +31,14 @@ impl ConfigModule for MetricsConfig { DEFAULT_METRIC_SERVER_PORT }; Ok(Self { - enable_metrics: !opt.disable_metrics, + disable_metrics: opt.disable_metrics, address: "0.0.0.0".to_string(), port, }) } fn after_load(&mut self, opt: &StarcoinOpt, _base: &BaseConfig) -> Result<()> { - if opt.disable_metrics { - self.enable_metrics = false; - } + self.disable_metrics = opt.disable_metrics; Ok(()) } } diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index 3d311ed58d..33e0e86929 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -8,9 +8,9 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct MinerConfig { - pub enable_mint_empty_block: bool, + pub disable_mint_empty_block: bool, pub block_gas_limit: Option, - pub enable_miner_client: bool, + pub disable_miner_client: bool, pub client_config: MinerClientConfig, } @@ -33,9 +33,9 @@ impl ConfigModule for MinerConfig { .cloned() .unwrap_or_else(|| base.net.is_dev()); Ok(Self { - enable_mint_empty_block: !disable_mint_empty_block, + disable_mint_empty_block, block_gas_limit: None, - enable_miner_client: !opt.disable_miner_client, + disable_miner_client: opt.disable_miner_client, client_config: MinerClientConfig { server: None, plugin_path: None, @@ -47,17 +47,18 @@ impl ConfigModule for MinerConfig { fn after_load(&mut self, opt: &StarcoinOpt, base: &BaseConfig) -> Result<()> { // only dev network is on demand mine at default. - let disable_mint_empty_block = opt - .disable_mint_empty_block - .as_ref() - .cloned() - .unwrap_or_else(|| base.net.is_dev()); - self.enable_mint_empty_block = !disable_mint_empty_block; - if let Some(thread) = opt.miner_thread { - self.client_config.thread_num = thread; - } if opt.disable_miner_client { - self.enable_miner_client = false; + self.disable_miner_client = true; + } else { + let disable_mint_empty_block = opt + .disable_mint_empty_block + .as_ref() + .cloned() + .unwrap_or_else(|| base.net.is_dev()); + self.disable_mint_empty_block = disable_mint_empty_block; + if let Some(thread) = opt.miner_thread { + self.client_config.thread_num = thread; + } } Ok(()) } diff --git a/config/src/network_config.rs b/config/src/network_config.rs index 705726d28e..1480c029fa 100644 --- a/config/src/network_config.rs +++ b/config/src/network_config.rs @@ -85,7 +85,7 @@ impl Default for NetworkRpcQuotaConfiguration { #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct NetworkConfig { - // The address that this node is listening on for new connections. + /// The address that this node is listening on for new connections. pub listen: Multiaddr, #[serde(default)] pub seeds: Vec, diff --git a/config/src/rpc_config.rs b/config/src/rpc_config.rs index 08cf8f421a..48cbe05f0d 100644 --- a/config/src/rpc_config.rs +++ b/config/src/rpc_config.rs @@ -302,12 +302,38 @@ impl ConfigModule for RpcConfig { } fn after_load(&mut self, opt: &StarcoinOpt, _base: &BaseConfig) -> Result<()> { - if self.http.ip_headers.is_none() { - self.http.ip_headers = opt.http.ip_headers.clone(); + if !opt.http.disable { + self.http.disable = false; + self.http.apis = opt.http.apis.clone(); + self.http.port = opt.http.port; + self.http.max_request_body_size = opt.http.max_request_body_size; + if opt.http.threads.is_some() { + self.http.threads = opt.http.threads.clone(); + } + if opt.http.ip_headers.is_some() { + self.http.ip_headers = opt.http.ip_headers.clone(); + } } - info!("Ipc file path: {:?}", self.ipc.ipc_file_path); info!("Http rpc address: {:?}", self.get_http_address()); + if !opt.tcp.disable { + self.tcp.disable = false; + self.tcp.apis = opt.tcp.apis.clone(); + self.tcp.port = opt.tcp.port; + } info!("TCP rpc address: {:?}", self.get_tcp_address()); + if !opt.ipc.disable { + self.ipc.apis = opt.ipc.apis.clone(); + if opt.ipc.ipc_file_path.is_some() { + self.ipc.ipc_file_path = opt.ipc.ipc_file_path.clone(); + } + } + info!("Ipc file path: {:?}", self.ipc.ipc_file_path); + if !opt.ws.disable { + self.ws.disable = false; + self.ws.apis = opt.ws.apis.clone(); + self.ws.port = opt.ws.port; + self.ws.max_request_body_size = opt.ws.max_request_body_size; + } info!("Websocket rpc address: {:?}", self.get_ws_address()); Ok(()) } diff --git a/config/src/storage_config.rs b/config/src/storage_config.rs index 677b4dc8dd..e0225b0e6c 100644 --- a/config/src/storage_config.rs +++ b/config/src/storage_config.rs @@ -5,14 +5,21 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use structopt::StructOpt; /// Port selected RocksDB options for tuning underlying rocksdb instance of DiemDB. /// see https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h /// for detailed explanations. -#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(default, deny_unknown_fields)] pub struct RocksdbConfig { + #[structopt(name = "rocksdb-max-open-files", long, help = "rocksdb max open files")] pub max_open_files: i32, + #[structopt( + name = "rocksdb-max-total-wal-sizes", + long, + help = "rocksdb max total WAL sizes" + )] pub max_total_wal_size: u64, } @@ -52,14 +59,20 @@ impl Default for RocksdbConfig { } } -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct StorageConfig { + #[structopt(long = "dir", parse(from_os_str), conflicts_with("dir"))] dir: PathBuf, + #[structopt( + long = "absolute-dir", + parse(from_os_str), + conflicts_with("absolute-dir") + )] #[serde(skip)] absolute_dir: Option, + #[structopt(flatten)] /// Rocksdb-specific configurations - #[serde(default)] pub rocksdb_config: RocksdbConfig, } diff --git a/config/src/sync_config.rs b/config/src/sync_config.rs index 8741476677..f19a246950 100644 --- a/config/src/sync_config.rs +++ b/config/src/sync_config.rs @@ -2,15 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{BaseConfig, ConfigModule, StarcoinOpt}; -use anyhow::{bail, format_err, Result}; +use anyhow::{bail, Result}; +use clap::arg_enum; use serde::{Deserialize, Serialize}; use starcoin_logger::prelude::*; -use std::fmt::{Display, Formatter}; -use std::str::FromStr; +use structopt::StructOpt; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct SyncConfig { + #[structopt(long, possible_values = &SyncMode::variants(), case_insensitive = true)] + /// Sync mode: Light, Fast, Full, eg. sync_mode: SyncMode, } @@ -24,25 +26,25 @@ impl SyncConfig { } pub fn is_state_sync(&self) -> bool { - self.sync_mode == SyncMode::FAST + self.sync_mode == SyncMode::Fast } pub fn is_light(&self) -> bool { - self.sync_mode == SyncMode::LIGHT + self.sync_mode == SyncMode::Lignt } } impl ConfigModule for SyncConfig { fn default_with_opt(opt: &StarcoinOpt, _base: &BaseConfig) -> Result { - let sync_mode = opt.sync_mode.unwrap_or(SyncMode::FULL); + let sync_mode = opt.sync_mode.clone().unwrap_or(SyncMode::Full); Ok(SyncConfig { sync_mode }) } fn after_load(&mut self, opt: &StarcoinOpt, _base: &BaseConfig) -> Result<()> { - if let Some(sync_mode) = opt.sync_mode { + if let Some(sync_mode) = opt.sync_mode.clone() { self.sync_mode = sync_mode; } - if self.sync_mode == SyncMode::LIGHT || self.sync_mode == SyncMode::FAST { + if self.sync_mode == SyncMode::Lignt || self.sync_mode == SyncMode::Fast { bail!("{} is not supported yet.", self.sync_mode); } info!("Sync mode : {:?} : {:?}", opt.sync_mode, self.sync_mode); @@ -50,40 +52,17 @@ impl ConfigModule for SyncConfig { } } //TODO remove SyncMode. -#[allow(non_camel_case_types)] -#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] -#[serde(tag = "type")] +arg_enum! { +#[derive(Debug,Clone, Deserialize, PartialEq, Serialize)] pub enum SyncMode { - LIGHT, - FAST, - FULL, -} - -impl FromStr for SyncMode { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s { - "light" => Ok(SyncMode::LIGHT), - "fast" => Ok(SyncMode::FAST), - "full" => Ok(SyncMode::FULL), - _ => Err(format_err!("")), - } - } -} - -impl Display for SyncMode { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - SyncMode::LIGHT => write!(f, "light"), - SyncMode::FAST => write!(f, "fast"), - SyncMode::FULL => write!(f, "full"), - } + Lignt, + Fast, + Full, } } impl Default for SyncMode { fn default() -> Self { - SyncMode::FULL + SyncMode::Full } } diff --git a/config/src/txpool_config.rs b/config/src/txpool_config.rs index 8ba04afe67..aa4a2db04b 100644 --- a/config/src/txpool_config.rs +++ b/config/src/txpool_config.rs @@ -4,14 +4,18 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; +use structopt::StructOpt; -#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct TxPoolConfig { + #[structopt(name = "max-count", long, default_value = "4096")] /// Maximal number of transactions in the pool. pub max_count: u64, + #[structopt(name = "max-per-sender", long, default_value = "128")] /// Maximal number of transactions from single sender. pub max_per_sender: u64, + #[structopt(name = "max-mem-usage", long, default_value = "134217728")] /// Maximal memory usage. pub max_mem_usage: u64, } diff --git a/miner/src/generate_block_event_pacemaker.rs b/miner/src/generate_block_event_pacemaker.rs index 14e35d0bf8..1f52036a67 100644 --- a/miner/src/generate_block_event_pacemaker.rs +++ b/miner/src/generate_block_event_pacemaker.rs @@ -45,7 +45,7 @@ impl ActorService for GenerateBlockEventPacemaker { ctx.subscribe::(); ctx.subscribe::(); //if mint empty block is disabled, trigger mint event for on demand mint (Dev) - if !self.config.miner.enable_mint_empty_block { + if self.config.miner.disable_mint_empty_block { ctx.subscribe::(); } Ok(()) @@ -54,7 +54,7 @@ impl ActorService for GenerateBlockEventPacemaker { fn stopped(&mut self, ctx: &mut ServiceContext) -> Result<()> { ctx.unsubscribe::(); ctx.unsubscribe::(); - if !self.config.miner.enable_mint_empty_block { + if self.config.miner.disable_mint_empty_block { ctx.unsubscribe::(); } Ok(()) diff --git a/miner/src/lib.rs b/miner/src/lib.rs index 559656ccac..361e106285 100644 --- a/miner/src/lib.rs +++ b/miner/src/lib.rs @@ -72,9 +72,9 @@ impl MinerService { .send(CreateBlockTemplateRequest) .await? })?; - if block_template.body.transactions.is_empty() && !self.config.miner.enable_mint_empty_block + if block_template.body.transactions.is_empty() && self.config.miner.disable_mint_empty_block { - debug!("The flag enable_mint_empty_block is false and no txn in pool, so skip mint empty block."); + debug!("The flag disable_mint_empty_block is true and no txn in pool, so skip mint empty block."); Ok(()) } else { debug!("Mint block template: {:?}", block_template); diff --git a/miner/tests/miner_test.rs b/miner/tests/miner_test.rs index cbf70d4230..0b34abf123 100644 --- a/miner/tests/miner_test.rs +++ b/miner/tests/miner_test.rs @@ -22,7 +22,7 @@ use types::{ #[stest::test] fn test_miner() { let mut config = NodeConfig::random_for_test(); - config.miner.enable_miner_client = false; + config.miner.disable_miner_client = false; let config = Arc::new(config); let handle = test_helper::run_node_by_config(config.clone()).unwrap(); let bus = handle.bus().unwrap(); @@ -60,7 +60,7 @@ fn test_miner() { #[stest::test] async fn test_miner_service() { let mut config = NodeConfig::random_for_test(); - config.miner.enable_mint_empty_block = true; + config.miner.disable_mint_empty_block = false; let registry = RegistryService::launch(); let node_config = Arc::new(config.clone()); registry.put_shared(node_config.clone()).await.unwrap(); diff --git a/node/src/node.rs b/node/src/node.rs index 54daedd1c8..ceeeaf7ae2 100644 --- a/node/src/node.rs +++ b/node/src/node.rs @@ -129,14 +129,14 @@ impl NodeService { config.logger.max_backup, ); } - if config.logger.enable_stderr { + if config.logger.disable_stderr { logger_handle.enable_stderr(); } else { logger_handle.disable_stderr(); } // start metric server - if config.metrics.enable_metrics { + if !config.metrics.disable_metrics { starcoin_metrics::metric_server::start_server( config.metrics.address.clone(), config.metrics.port, @@ -246,7 +246,7 @@ impl NodeService { registry.register::().await?; registry.register::().await?; - if config.miner.enable_miner_client { + if config.miner.disable_miner_client { let miner_client_config = config.miner.client_config.clone(); registry.put_shared(miner_client_config).await?; let job_client = JobBusClient::new(bus.clone(), config.net().time_service()); diff --git a/rpc/client/tests/client_server_test.rs b/rpc/client/tests/client_server_test.rs index cbd9ace363..c3006db2de 100644 --- a/rpc/client/tests/client_server_test.rs +++ b/rpc/client/tests/client_server_test.rs @@ -19,7 +19,7 @@ async fn test_in_async() -> Result<()> { fn do_client_test() -> Result<()> { let mut node_config = NodeConfig::random_for_test(); - node_config.miner.enable_miner_client = false; + node_config.miner.disable_miner_client = false; let config = Arc::new(node_config); let ipc_file = config.rpc.get_ipc_file().to_path_buf(); let url = config.rpc.get_ws_address().unwrap(); @@ -60,7 +60,7 @@ fn test_multi_client() -> Result<()> { #[stest::test(timeout = 120)] fn test_client_reconnect() -> Result<()> { let mut node_config = NodeConfig::random_for_test(); - node_config.miner.enable_miner_client = false; + node_config.miner.disable_miner_client = false; let config = Arc::new(node_config); let url = config.rpc.get_ws_address().unwrap(); debug!("url:{}", url); @@ -93,7 +93,7 @@ fn test_client_reconnect() -> Result<()> { #[stest::test(timeout = 120)] fn test_client_reconnect_subscribe() -> Result<()> { let mut node_config = NodeConfig::random_for_test(); - node_config.miner.enable_miner_client = true; + node_config.miner.disable_miner_client = true; let config = Arc::new(node_config); let url = config.rpc.get_ws_address().unwrap(); debug!("url:{}", url); diff --git a/sync/tests/full_sync_test.rs b/sync/tests/full_sync_test.rs index b2ef58a23f..c46cb15131 100644 --- a/sync/tests/full_sync_test.rs +++ b/sync/tests/full_sync_test.rs @@ -13,7 +13,7 @@ use traits::ChainAsyncService; #[stest::test(timeout = 120)] fn test_full_sync() { - test_sync::test_sync(SyncMode::FULL) + test_sync::test_sync(SyncMode::Full) } #[ignore] @@ -30,7 +30,7 @@ fn test_sync_by_notification() { let mut second_config = NodeConfig::random_for_test(); info!("second peer : {:?}", second_config.network.self_peer_id()); second_config.network.seeds = vec![first_config.network.self_address()]; - second_config.miner.enable_miner_client = false; + second_config.miner.disable_miner_client = false; let second_node = run_node_by_config(Arc::new(second_config)).unwrap(); // stop sync service and just use notification message to sync. @@ -88,7 +88,7 @@ fn test_broadcast_with_difficulty() { info!("second peer : {:?}", second_config.network.self_peer_id()); second_config.network.seeds = vec![first_config.network.self_address()]; //second_config.miner.enable_miner_client = false; - second_config.sync.set_mode(SyncMode::FULL); + second_config.sync.set_mode(SyncMode::Full); let second_node = run_node_by_config(Arc::new(second_config)).unwrap(); let second_chain = second_node.chain_service().unwrap(); diff --git a/sync/tests/test_sync/mod.rs b/sync/tests/test_sync/mod.rs index 283c5ba5bf..f0c2273ae1 100644 --- a/sync/tests/test_sync/mod.rs +++ b/sync/tests/test_sync/mod.rs @@ -25,7 +25,7 @@ pub fn test_sync(sync_mode: SyncMode) { let mut second_config = NodeConfig::random_for_test(); info!("second peer : {:?}", second_config.network.self_peer_id()); second_config.network.seeds = vec![first_config.network.self_address()]; - second_config.miner.enable_miner_client = false; + second_config.miner.disable_miner_client = false; second_config.sync.set_mode(sync_mode); let second_node = run_node_by_config(Arc::new(second_config)).unwrap(); diff --git a/sync/tests/txn_sync_test.rs b/sync/tests/txn_sync_test.rs index 1f787338ac..4cb177c41e 100644 --- a/sync/tests/txn_sync_test.rs +++ b/sync/tests/txn_sync_test.rs @@ -15,7 +15,7 @@ use txpool::TxPoolService; #[stest::test] fn test_txn_sync_actor() { let mut first_config = NodeConfig::random_for_test(); - first_config.miner.enable_miner_client = false; + first_config.miner.disable_miner_client = false; let first_network_address = first_config.network.self_address(); let first_config = Arc::new(first_config); let first_node = run_node_by_config(first_config.clone()).unwrap(); @@ -31,7 +31,7 @@ fn test_txn_sync_actor() { let mut second_config = NodeConfig::random_for_test(); second_config.network.seeds = vec![first_network_address]; - second_config.miner.enable_miner_client = false; + second_config.miner.disable_miner_client = false; let second_config = Arc::new(second_config); let second_node = run_node_by_config(second_config.clone()).unwrap(); From eba6bfcb0cf3caeaf2ffe8f94614b86c320e47d0 Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Wed, 6 Jan 2021 10:35:37 +0800 Subject: [PATCH 02/10] [config]update txpool config for system free memory --- Cargo.lock | 97 +++++++++++++++++++++++++++++++++++-- commons/system/Cargo.toml | 11 +++++ commons/system/src/lib.rs | 14 ++++++ config/Cargo.toml | 1 + config/src/lib.rs | 2 +- config/src/miner_config.rs | 19 ++++++-- config/src/rpc_config.rs | 1 - config/src/sync_config.rs | 2 +- config/src/txpool_config.rs | 14 +++++- node/src/node.rs | 2 +- 10 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 commons/system/Cargo.toml create mode 100644 commons/system/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 7016344f67..ce66d44c60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -721,7 +721,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41262f11d771fd4a61aa3ce019fca363b4b6c282fca9da2a31186d3965a47a5c" dependencies = [ "either", - "radium", + "radium 0.3.0", +] + +[[package]] +name = "bitvec" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ba35e9565969edb811639dbebfe34edc0368e472c5018474c8eb2543397f81" +dependencies = [ + "funty", + "radium 0.5.3", + "tap", + "wyz", ] [[package]] @@ -959,6 +971,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad1f8e949d755f9d79112b5bb46938e0ef9d3804a0b16dfab13aafcaa5f0fa72" +[[package]] +name = "bytesize" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da" + [[package]] name = "cache-padded" version = "1.1.1" @@ -1001,7 +1019,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" dependencies = [ - "nom", + "nom 5.1.2", ] [[package]] @@ -2248,6 +2266,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + [[package]] name = "futures" version = "0.1.30" @@ -3433,6 +3457,19 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lexical-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616" +dependencies = [ + "arrayvec 0.5.2", + "bitflags", + "cfg-if 0.1.10", + "ryu", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.80" @@ -4746,6 +4783,18 @@ dependencies = [ "version_check 0.9.2", ] +[[package]] +name = "nom" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88034cfd6b4a0d54dd14f4a507eceee36c0b70e5a02236c4e4df571102be17f0" +dependencies = [ + "bitvec 0.19.4", + "lexical-core", + "memchr", + "version_check 0.9.2", +] + [[package]] name = "nonzero_ext" version = "0.2.0" @@ -5015,7 +5064,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c740e5fbcb6847058b40ac7e5574766c6388f585e184d769910fe0d3a2ca861" dependencies = [ "arrayvec 0.5.2", - "bitvec", + "bitvec 0.17.4", "byte-slice-cast", "parity-scale-codec-derive", "serde", @@ -5744,6 +5793,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "def50a86306165861203e7f84ecffbbdfdea79f0e51039b33de1e952358c47ac" +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + [[package]] name = "radix_trie" version = "0.2.1" @@ -7252,6 +7307,7 @@ dependencies = [ "serde_json", "starcoin-crypto", "starcoin-logger", + "starcoin-system", "starcoin-types", "starcoin-vm-types", "structopt 0.3.21", @@ -8318,6 +8374,14 @@ dependencies = [ "stream-task", ] +[[package]] +name = "starcoin-system" +version = "0.9.4" +dependencies = [ + "anyhow", + "systemstat", +] + [[package]] name = "starcoin-traits" version = "0.9.4" @@ -8794,6 +8858,27 @@ dependencies = [ "unicode-xid 0.2.1", ] +[[package]] +name = "systemstat" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1f726379f0c746ad672b7e932bd390b5faae073f502e37d314950ced761ac3e" +dependencies = [ + "bytesize", + "chrono", + "lazy_static", + "libc", + "nom 6.0.1", + "time", + "winapi 0.3.9", +] + +[[package]] +name = "tap" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36474e732d1affd3a6ed582781b3683df3d0563714c59c39591e8ff707cf078e" + [[package]] name = "target-spec" version = "0.4.1" @@ -10187,6 +10272,12 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + [[package]] name = "x" version = "0.1.0" diff --git a/commons/system/Cargo.toml b/commons/system/Cargo.toml new file mode 100644 index 0000000000..0855e84801 --- /dev/null +++ b/commons/system/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "starcoin-system" +version = "0.9.4" +authors = ["Starcoin Core Dev "] +license = "Apache-2.0" +publish = false +edition = "2018" + +[dependencies] +systemstat ="0.1.6" +anyhow = "1.0.37" \ No newline at end of file diff --git a/commons/system/src/lib.rs b/commons/system/src/lib.rs new file mode 100644 index 0000000000..8cc36cc43a --- /dev/null +++ b/commons/system/src/lib.rs @@ -0,0 +1,14 @@ +// Copyright (c) The Starcoin Core Contributors +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use systemstat::{Platform, System}; + +pub fn get_free_mem_size() -> Result { + let sys = System::new(); + let free = match sys.memory() { + Ok(mem) => mem.free.as_u64(), + Err(_x) => 0u64, + }; + Ok(free) +} diff --git a/config/Cargo.toml b/config/Cargo.toml index 210c4e6c75..719e72905a 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -27,4 +27,5 @@ starcoin-types = { path = "../types" } starcoin-vm-types = { path = "../vm/types" } network-p2p-types = { path = "../network-p2p/types"} starcoin-logger = {path = "../commons/logger", package="starcoin-logger"} +starcoin-system = {path = "../commons/system", package="starcoin-system"} libra-temppath = { git = "https://github.com/starcoinorg/libra", rev="bf037026b5842354a2892e0b58f527e80658ace5" } diff --git a/config/src/lib.rs b/config/src/lib.rs index 723af6b42b..96388c6ac8 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -190,7 +190,7 @@ pub struct StarcoinOpt { /// Node network private key file, only work for first init. pub node_key_file: Option, - #[structopt(long = "sync-mode", short = "s")] + #[structopt(long = "sync-mode", short = "s", possible_values = &SyncMode::variants(), case_insensitive = false)] /// Sync mode. Included value(full, fast, light). pub sync_mode: Option, diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index 33e0e86929..fce71241c9 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -4,22 +4,29 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; +use structopt::StructOpt; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct MinerConfig { + #[structopt(long = "disable-mint-empty-block")] pub disable_mint_empty_block: bool, + #[structopt(long = "block-gas-limit")] pub block_gas_limit: Option, + #[structopt(long = "disable-miner-client")] pub disable_miner_client: bool, + #[structopt(flatten)] pub client_config: MinerClientConfig, } -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct MinerClientConfig { pub server: Option, pub plugin_path: Option, + #[structopt(long = "thread-num")] pub thread_num: u16, + #[structopt(long = "enable-stderr")] #[serde(skip)] pub enable_stderr: bool, } @@ -54,7 +61,13 @@ impl ConfigModule for MinerConfig { .disable_mint_empty_block .as_ref() .cloned() - .unwrap_or_else(|| base.net.is_dev()); + .unwrap_or_else(|| { + if base.net.is_dev() || base.net.is_test() { + false + } else { + true + } + }); self.disable_mint_empty_block = disable_mint_empty_block; if let Some(thread) = opt.miner_thread { self.client_config.thread_num = thread; diff --git a/config/src/rpc_config.rs b/config/src/rpc_config.rs index 48cbe05f0d..e4dbef1f7d 100644 --- a/config/src/rpc_config.rs +++ b/config/src/rpc_config.rs @@ -289,7 +289,6 @@ impl ConfigModule for RpcConfig { config.ws.port = ports[2]; } else if base.net.is_dev() { config.http.port = get_available_port_from(DEFAULT_HTTP_PORT); - config.tcp.port = get_available_port_from(DEFAULT_TCP_PORT); config.ws.port = get_available_port_from(DEFAULT_WEB_SOCKET_PORT); } diff --git a/config/src/sync_config.rs b/config/src/sync_config.rs index f19a246950..6140826cbd 100644 --- a/config/src/sync_config.rs +++ b/config/src/sync_config.rs @@ -11,7 +11,7 @@ use structopt::StructOpt; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct SyncConfig { - #[structopt(long, possible_values = &SyncMode::variants(), case_insensitive = true)] + #[structopt(long, possible_values = &SyncMode::variants(), case_insensitive = false)] /// Sync mode: Light, Fast, Full, eg. sync_mode: SyncMode, } diff --git a/config/src/txpool_config.rs b/config/src/txpool_config.rs index aa4a2db04b..74a756e28d 100644 --- a/config/src/txpool_config.rs +++ b/config/src/txpool_config.rs @@ -4,7 +4,9 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; +use starcoin_system::get_free_mem_size; use structopt::StructOpt; +pub const DEFAULT_MEM_SIZE: u64 = 128 * 1024 * 1024; // 128M #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, StructOpt)] #[serde(deny_unknown_fields)] @@ -22,10 +24,20 @@ pub struct TxPoolConfig { impl ConfigModule for TxPoolConfig { fn default_with_opt(_opt: &StarcoinOpt, _base: &BaseConfig) -> Result { + let free_mem = match get_free_mem_size() { + Ok(free) => { + if free > 0 { + free / 2 + } else { + DEFAULT_MEM_SIZE + } + } + Err(_) => DEFAULT_MEM_SIZE, + }; Ok(Self { max_count: 4096, max_per_sender: 128, - max_mem_usage: 128 * 1024 * 1024, // 128M + max_mem_usage: free_mem, }) } } diff --git a/node/src/node.rs b/node/src/node.rs index ceeeaf7ae2..d789abc5f8 100644 --- a/node/src/node.rs +++ b/node/src/node.rs @@ -246,7 +246,7 @@ impl NodeService { registry.register::().await?; registry.register::().await?; - if config.miner.disable_miner_client { + if !config.miner.disable_miner_client { let miner_client_config = config.miner.client_config.clone(); registry.put_shared(miner_client_config).await?; let job_client = JobBusClient::new(bus.clone(), config.net().time_service()); From 96de3b6eafe76883032c0be49ab4daa239acbe23 Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Wed, 6 Jan 2021 10:58:37 +0800 Subject: [PATCH 03/10] [config]fix clippy check --- config/src/miner_config.rs | 8 +------- config/src/rpc_config.rs | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index fce71241c9..28e3ca71d6 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -61,13 +61,7 @@ impl ConfigModule for MinerConfig { .disable_mint_empty_block .as_ref() .cloned() - .unwrap_or_else(|| { - if base.net.is_dev() || base.net.is_test() { - false - } else { - true - } - }); + .unwrap_or_else(|| !(base.net.is_dev() || base.net.is_test())); self.disable_mint_empty_block = disable_mint_empty_block; if let Some(thread) = opt.miner_thread { self.client_config.thread_num = thread; diff --git a/config/src/rpc_config.rs b/config/src/rpc_config.rs index e4dbef1f7d..776393cf13 100644 --- a/config/src/rpc_config.rs +++ b/config/src/rpc_config.rs @@ -307,7 +307,7 @@ impl ConfigModule for RpcConfig { self.http.port = opt.http.port; self.http.max_request_body_size = opt.http.max_request_body_size; if opt.http.threads.is_some() { - self.http.threads = opt.http.threads.clone(); + self.http.threads = opt.http.threads; } if opt.http.ip_headers.is_some() { self.http.ip_headers = opt.http.ip_headers.clone(); From 78eaeb59609102502162459003de922a6df2e176 Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Wed, 6 Jan 2021 12:27:42 +0800 Subject: [PATCH 04/10] [config]fix test failure --- rpc/client/tests/client_server_test.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/rpc/client/tests/client_server_test.rs b/rpc/client/tests/client_server_test.rs index c3006db2de..a667ca0f92 100644 --- a/rpc/client/tests/client_server_test.rs +++ b/rpc/client/tests/client_server_test.rs @@ -18,8 +18,7 @@ async fn test_in_async() -> Result<()> { } fn do_client_test() -> Result<()> { - let mut node_config = NodeConfig::random_for_test(); - node_config.miner.disable_miner_client = false; + let node_config = NodeConfig::random_for_test(); let config = Arc::new(node_config); let ipc_file = config.rpc.get_ipc_file().to_path_buf(); let url = config.rpc.get_ws_address().unwrap(); @@ -92,8 +91,7 @@ fn test_client_reconnect() -> Result<()> { #[stest::test(timeout = 120)] fn test_client_reconnect_subscribe() -> Result<()> { - let mut node_config = NodeConfig::random_for_test(); - node_config.miner.disable_miner_client = true; + let node_config = NodeConfig::random_for_test(); let config = Arc::new(node_config); let url = config.rpc.get_ws_address().unwrap(); debug!("url:{}", url); From a343123674bb267a546523399fad5d25e6ceac93 Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Wed, 6 Jan 2021 15:07:06 +0800 Subject: [PATCH 05/10] [config]update disable_mint_empty_block to option --- config/src/miner_config.rs | 33 +++++++++++++-------- miner/src/generate_block_event_pacemaker.rs | 4 +-- miner/src/lib.rs | 3 +- miner/tests/miner_test.rs | 2 +- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index 28e3ca71d6..6556e6b6eb 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -10,7 +10,7 @@ use structopt::StructOpt; #[serde(deny_unknown_fields)] pub struct MinerConfig { #[structopt(long = "disable-mint-empty-block")] - pub disable_mint_empty_block: bool, + pub disable_mint_empty_block: Option, #[structopt(long = "block-gas-limit")] pub block_gas_limit: Option, #[structopt(long = "disable-miner-client")] @@ -19,6 +19,16 @@ pub struct MinerConfig { pub client_config: MinerClientConfig, } +impl MinerConfig { + pub fn is_disable_mint_empty_block(&self) -> bool { + if let Some(disable) = self.disable_mint_empty_block { + disable + } else { + false + } + } +} + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct MinerClientConfig { @@ -40,7 +50,7 @@ impl ConfigModule for MinerConfig { .cloned() .unwrap_or_else(|| base.net.is_dev()); Ok(Self { - disable_mint_empty_block, + disable_mint_empty_block: Some(disable_mint_empty_block), block_gas_limit: None, disable_miner_client: opt.disable_miner_client, client_config: MinerClientConfig { @@ -54,18 +64,17 @@ impl ConfigModule for MinerConfig { fn after_load(&mut self, opt: &StarcoinOpt, base: &BaseConfig) -> Result<()> { // only dev network is on demand mine at default. + let disable_mint_empty_block = opt + .disable_mint_empty_block + .as_ref() + .cloned() + .unwrap_or_else(|| base.net.is_dev()); + self.disable_mint_empty_block = Some(disable_mint_empty_block); + if let Some(thread) = opt.miner_thread { + self.client_config.thread_num = thread; + } if opt.disable_miner_client { self.disable_miner_client = true; - } else { - let disable_mint_empty_block = opt - .disable_mint_empty_block - .as_ref() - .cloned() - .unwrap_or_else(|| !(base.net.is_dev() || base.net.is_test())); - self.disable_mint_empty_block = disable_mint_empty_block; - if let Some(thread) = opt.miner_thread { - self.client_config.thread_num = thread; - } } Ok(()) } diff --git a/miner/src/generate_block_event_pacemaker.rs b/miner/src/generate_block_event_pacemaker.rs index 1f52036a67..c9fe79e744 100644 --- a/miner/src/generate_block_event_pacemaker.rs +++ b/miner/src/generate_block_event_pacemaker.rs @@ -45,7 +45,7 @@ impl ActorService for GenerateBlockEventPacemaker { ctx.subscribe::(); ctx.subscribe::(); //if mint empty block is disabled, trigger mint event for on demand mint (Dev) - if self.config.miner.disable_mint_empty_block { + if self.config.miner.is_disable_mint_empty_block() { ctx.subscribe::(); } Ok(()) @@ -54,7 +54,7 @@ impl ActorService for GenerateBlockEventPacemaker { fn stopped(&mut self, ctx: &mut ServiceContext) -> Result<()> { ctx.unsubscribe::(); ctx.unsubscribe::(); - if self.config.miner.disable_mint_empty_block { + if self.config.miner.is_disable_mint_empty_block() { ctx.unsubscribe::(); } Ok(()) diff --git a/miner/src/lib.rs b/miner/src/lib.rs index 361e106285..11256ca699 100644 --- a/miner/src/lib.rs +++ b/miner/src/lib.rs @@ -72,7 +72,8 @@ impl MinerService { .send(CreateBlockTemplateRequest) .await? })?; - if block_template.body.transactions.is_empty() && self.config.miner.disable_mint_empty_block + if block_template.body.transactions.is_empty() + && self.config.miner.is_disable_mint_empty_block() { debug!("The flag disable_mint_empty_block is true and no txn in pool, so skip mint empty block."); Ok(()) diff --git a/miner/tests/miner_test.rs b/miner/tests/miner_test.rs index 0b34abf123..5ba3d9d3db 100644 --- a/miner/tests/miner_test.rs +++ b/miner/tests/miner_test.rs @@ -60,7 +60,7 @@ fn test_miner() { #[stest::test] async fn test_miner_service() { let mut config = NodeConfig::random_for_test(); - config.miner.disable_mint_empty_block = false; + config.miner.disable_mint_empty_block = Some(false); let registry = RegistryService::launch(); let node_config = Arc::new(config.clone()); registry.put_shared(node_config.clone()).await.unwrap(); From f7da4532e5441d7be77ec8d29a05cd78472af1c1 Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Tue, 5 Jan 2021 18:12:50 +0800 Subject: [PATCH 06/10] [config]Configuration optimization and synchronization for consistency --- config/Cargo.toml | 1 + config/src/account_vault_config.rs | 5 +- config/src/lib.rs | 2 - config/src/logger_config.rs | 33 +++++++------ config/src/metrics_config.rs | 14 +++--- config/src/miner_config.rs | 29 +++++------ config/src/network_config.rs | 2 +- config/src/rpc_config.rs | 32 ++++++++++-- config/src/storage_config.rs | 19 +++++-- config/src/sync_config.rs | 55 +++++++-------------- config/src/txpool_config.rs | 6 ++- miner/src/generate_block_event_pacemaker.rs | 4 +- miner/src/lib.rs | 4 +- miner/tests/miner_test.rs | 4 +- node/src/node.rs | 6 +-- rpc/client/tests/client_server_test.rs | 6 +-- sync/tests/full_sync_test.rs | 6 +-- sync/tests/test_sync/mod.rs | 2 +- sync/tests/txn_sync_test.rs | 4 +- 19 files changed, 133 insertions(+), 101 deletions(-) diff --git a/config/Cargo.toml b/config/Cargo.toml index d23e534b06..ff0253cb2c 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -7,6 +7,7 @@ publish = false edition = "2018" [dependencies] +clap = "2.33.3" dirs = "3" anyhow = "1.0.37" thiserror = "1.0" diff --git a/config/src/account_vault_config.rs b/config/src/account_vault_config.rs index f9ffea7f0c..2ae0b46dee 100644 --- a/config/src/account_vault_config.rs +++ b/config/src/account_vault_config.rs @@ -5,10 +5,13 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use structopt::StructOpt; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct AccountVaultConfig { + #[structopt(long = "vault-dir", parse(from_os_str), conflicts_with("vault-dir"))] + /// Account vault dir config. dir: PathBuf, #[serde(skip)] absolute_dir: Option, diff --git a/config/src/lib.rs b/config/src/lib.rs index ef33738f03..d8636dae1c 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -444,9 +444,7 @@ impl NodeConfig { } } }; - config.after_load(opt, &base)?; - save_config(&config, &config_file_path)?; Ok(config) } diff --git a/config/src/logger_config.rs b/config/src/logger_config.rs index fcb26e049b..7bb797489f 100644 --- a/config/src/logger_config.rs +++ b/config/src/logger_config.rs @@ -5,15 +5,20 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use structopt::StructOpt; static LOGGER_FILE_NAME: &str = "starcoin.log"; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct LoggerConfig { - pub enable_stderr: bool, - pub enable_file: bool, + #[structopt(name = "disable-stderr", long, help = "disable stderr logger")] + pub disable_stderr: bool, + #[structopt(name = "disable-file", long, help = "disable file logger")] + pub disable_file: bool, + #[structopt(name = "max-file-size", long, default_value = "1073741824")] pub max_file_size: u64, + #[structopt(name = "max-backup", long, default_value = "7")] pub max_backup: u32, #[serde(skip)] log_path: Option, @@ -28,35 +33,35 @@ impl LoggerConfig { } pub fn enable_file(&self) -> bool { - self.enable_file && self.log_path.is_some() + (!self.disable_file) && self.log_path.is_some() } } impl ConfigModule for LoggerConfig { fn default_with_opt(opt: &StarcoinOpt, base: &BaseConfig) -> Result { - let enable_stderr = !opt.disable_std_log; - let enable_file = !opt.disable_file_log; + let disable_stderr = opt.disable_std_log; + let disable_file = opt.disable_file_log; Ok(if base.net.is_test() { Self { - enable_stderr, - enable_file, + disable_stderr, + disable_file, max_file_size: 10 * 1024 * 1024, max_backup: 1, log_path: None, } } else if base.net.is_dev() { Self { - enable_stderr, - enable_file, + disable_stderr, + disable_file, max_file_size: 10 * 1024 * 1024, max_backup: 2, log_path: None, } } else { Self { - enable_stderr, - enable_file, + disable_stderr, + disable_file, max_file_size: 1024 * 1024 * 1024, max_backup: 7, log_path: None, @@ -66,8 +71,8 @@ impl ConfigModule for LoggerConfig { fn after_load(&mut self, opt: &StarcoinOpt, base: &BaseConfig) -> Result<()> { self.log_path = Some(base.data_dir.join(LOGGER_FILE_NAME)); - self.enable_stderr = !opt.disable_std_log; - self.enable_file = !opt.disable_file_log; + self.disable_stderr = opt.disable_std_log; + self.disable_file = opt.disable_file_log; Ok(()) } } diff --git a/config/src/metrics_config.rs b/config/src/metrics_config.rs index a3bdf01b6a..254cfeb6ec 100644 --- a/config/src/metrics_config.rs +++ b/config/src/metrics_config.rs @@ -6,12 +6,16 @@ use crate::{ }; use anyhow::Result; use serde::{Deserialize, Serialize}; +use structopt::StructOpt; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct MetricsConfig { - pub enable_metrics: bool, + #[structopt(name = "disable-metrics", long, help = "disable metrics")] + pub disable_metrics: bool, + #[structopt(name = "address", long, help = "address", default_value = "0.0.0.0")] pub address: String, + #[structopt(name = "metrics-port", long, default_value = "9101")] pub port: u16, } @@ -27,16 +31,14 @@ impl ConfigModule for MetricsConfig { DEFAULT_METRIC_SERVER_PORT }; Ok(Self { - enable_metrics: !opt.disable_metrics, + disable_metrics: opt.disable_metrics, address: "0.0.0.0".to_string(), port, }) } fn after_load(&mut self, opt: &StarcoinOpt, _base: &BaseConfig) -> Result<()> { - if opt.disable_metrics { - self.enable_metrics = false; - } + self.disable_metrics = opt.disable_metrics; Ok(()) } } diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index 3d311ed58d..33e0e86929 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -8,9 +8,9 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct MinerConfig { - pub enable_mint_empty_block: bool, + pub disable_mint_empty_block: bool, pub block_gas_limit: Option, - pub enable_miner_client: bool, + pub disable_miner_client: bool, pub client_config: MinerClientConfig, } @@ -33,9 +33,9 @@ impl ConfigModule for MinerConfig { .cloned() .unwrap_or_else(|| base.net.is_dev()); Ok(Self { - enable_mint_empty_block: !disable_mint_empty_block, + disable_mint_empty_block, block_gas_limit: None, - enable_miner_client: !opt.disable_miner_client, + disable_miner_client: opt.disable_miner_client, client_config: MinerClientConfig { server: None, plugin_path: None, @@ -47,17 +47,18 @@ impl ConfigModule for MinerConfig { fn after_load(&mut self, opt: &StarcoinOpt, base: &BaseConfig) -> Result<()> { // only dev network is on demand mine at default. - let disable_mint_empty_block = opt - .disable_mint_empty_block - .as_ref() - .cloned() - .unwrap_or_else(|| base.net.is_dev()); - self.enable_mint_empty_block = !disable_mint_empty_block; - if let Some(thread) = opt.miner_thread { - self.client_config.thread_num = thread; - } if opt.disable_miner_client { - self.enable_miner_client = false; + self.disable_miner_client = true; + } else { + let disable_mint_empty_block = opt + .disable_mint_empty_block + .as_ref() + .cloned() + .unwrap_or_else(|| base.net.is_dev()); + self.disable_mint_empty_block = disable_mint_empty_block; + if let Some(thread) = opt.miner_thread { + self.client_config.thread_num = thread; + } } Ok(()) } diff --git a/config/src/network_config.rs b/config/src/network_config.rs index 705726d28e..1480c029fa 100644 --- a/config/src/network_config.rs +++ b/config/src/network_config.rs @@ -85,7 +85,7 @@ impl Default for NetworkRpcQuotaConfiguration { #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct NetworkConfig { - // The address that this node is listening on for new connections. + /// The address that this node is listening on for new connections. pub listen: Multiaddr, #[serde(default)] pub seeds: Vec, diff --git a/config/src/rpc_config.rs b/config/src/rpc_config.rs index 08cf8f421a..48cbe05f0d 100644 --- a/config/src/rpc_config.rs +++ b/config/src/rpc_config.rs @@ -302,12 +302,38 @@ impl ConfigModule for RpcConfig { } fn after_load(&mut self, opt: &StarcoinOpt, _base: &BaseConfig) -> Result<()> { - if self.http.ip_headers.is_none() { - self.http.ip_headers = opt.http.ip_headers.clone(); + if !opt.http.disable { + self.http.disable = false; + self.http.apis = opt.http.apis.clone(); + self.http.port = opt.http.port; + self.http.max_request_body_size = opt.http.max_request_body_size; + if opt.http.threads.is_some() { + self.http.threads = opt.http.threads.clone(); + } + if opt.http.ip_headers.is_some() { + self.http.ip_headers = opt.http.ip_headers.clone(); + } } - info!("Ipc file path: {:?}", self.ipc.ipc_file_path); info!("Http rpc address: {:?}", self.get_http_address()); + if !opt.tcp.disable { + self.tcp.disable = false; + self.tcp.apis = opt.tcp.apis.clone(); + self.tcp.port = opt.tcp.port; + } info!("TCP rpc address: {:?}", self.get_tcp_address()); + if !opt.ipc.disable { + self.ipc.apis = opt.ipc.apis.clone(); + if opt.ipc.ipc_file_path.is_some() { + self.ipc.ipc_file_path = opt.ipc.ipc_file_path.clone(); + } + } + info!("Ipc file path: {:?}", self.ipc.ipc_file_path); + if !opt.ws.disable { + self.ws.disable = false; + self.ws.apis = opt.ws.apis.clone(); + self.ws.port = opt.ws.port; + self.ws.max_request_body_size = opt.ws.max_request_body_size; + } info!("Websocket rpc address: {:?}", self.get_ws_address()); Ok(()) } diff --git a/config/src/storage_config.rs b/config/src/storage_config.rs index 677b4dc8dd..e0225b0e6c 100644 --- a/config/src/storage_config.rs +++ b/config/src/storage_config.rs @@ -5,14 +5,21 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use structopt::StructOpt; /// Port selected RocksDB options for tuning underlying rocksdb instance of DiemDB. /// see https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h /// for detailed explanations. -#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(default, deny_unknown_fields)] pub struct RocksdbConfig { + #[structopt(name = "rocksdb-max-open-files", long, help = "rocksdb max open files")] pub max_open_files: i32, + #[structopt( + name = "rocksdb-max-total-wal-sizes", + long, + help = "rocksdb max total WAL sizes" + )] pub max_total_wal_size: u64, } @@ -52,14 +59,20 @@ impl Default for RocksdbConfig { } } -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct StorageConfig { + #[structopt(long = "dir", parse(from_os_str), conflicts_with("dir"))] dir: PathBuf, + #[structopt( + long = "absolute-dir", + parse(from_os_str), + conflicts_with("absolute-dir") + )] #[serde(skip)] absolute_dir: Option, + #[structopt(flatten)] /// Rocksdb-specific configurations - #[serde(default)] pub rocksdb_config: RocksdbConfig, } diff --git a/config/src/sync_config.rs b/config/src/sync_config.rs index 8741476677..f19a246950 100644 --- a/config/src/sync_config.rs +++ b/config/src/sync_config.rs @@ -2,15 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{BaseConfig, ConfigModule, StarcoinOpt}; -use anyhow::{bail, format_err, Result}; +use anyhow::{bail, Result}; +use clap::arg_enum; use serde::{Deserialize, Serialize}; use starcoin_logger::prelude::*; -use std::fmt::{Display, Formatter}; -use std::str::FromStr; +use structopt::StructOpt; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct SyncConfig { + #[structopt(long, possible_values = &SyncMode::variants(), case_insensitive = true)] + /// Sync mode: Light, Fast, Full, eg. sync_mode: SyncMode, } @@ -24,25 +26,25 @@ impl SyncConfig { } pub fn is_state_sync(&self) -> bool { - self.sync_mode == SyncMode::FAST + self.sync_mode == SyncMode::Fast } pub fn is_light(&self) -> bool { - self.sync_mode == SyncMode::LIGHT + self.sync_mode == SyncMode::Lignt } } impl ConfigModule for SyncConfig { fn default_with_opt(opt: &StarcoinOpt, _base: &BaseConfig) -> Result { - let sync_mode = opt.sync_mode.unwrap_or(SyncMode::FULL); + let sync_mode = opt.sync_mode.clone().unwrap_or(SyncMode::Full); Ok(SyncConfig { sync_mode }) } fn after_load(&mut self, opt: &StarcoinOpt, _base: &BaseConfig) -> Result<()> { - if let Some(sync_mode) = opt.sync_mode { + if let Some(sync_mode) = opt.sync_mode.clone() { self.sync_mode = sync_mode; } - if self.sync_mode == SyncMode::LIGHT || self.sync_mode == SyncMode::FAST { + if self.sync_mode == SyncMode::Lignt || self.sync_mode == SyncMode::Fast { bail!("{} is not supported yet.", self.sync_mode); } info!("Sync mode : {:?} : {:?}", opt.sync_mode, self.sync_mode); @@ -50,40 +52,17 @@ impl ConfigModule for SyncConfig { } } //TODO remove SyncMode. -#[allow(non_camel_case_types)] -#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] -#[serde(tag = "type")] +arg_enum! { +#[derive(Debug,Clone, Deserialize, PartialEq, Serialize)] pub enum SyncMode { - LIGHT, - FAST, - FULL, -} - -impl FromStr for SyncMode { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s { - "light" => Ok(SyncMode::LIGHT), - "fast" => Ok(SyncMode::FAST), - "full" => Ok(SyncMode::FULL), - _ => Err(format_err!("")), - } - } -} - -impl Display for SyncMode { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - SyncMode::LIGHT => write!(f, "light"), - SyncMode::FAST => write!(f, "fast"), - SyncMode::FULL => write!(f, "full"), - } + Lignt, + Fast, + Full, } } impl Default for SyncMode { fn default() -> Self { - SyncMode::FULL + SyncMode::Full } } diff --git a/config/src/txpool_config.rs b/config/src/txpool_config.rs index 8ba04afe67..aa4a2db04b 100644 --- a/config/src/txpool_config.rs +++ b/config/src/txpool_config.rs @@ -4,14 +4,18 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; +use structopt::StructOpt; -#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct TxPoolConfig { + #[structopt(name = "max-count", long, default_value = "4096")] /// Maximal number of transactions in the pool. pub max_count: u64, + #[structopt(name = "max-per-sender", long, default_value = "128")] /// Maximal number of transactions from single sender. pub max_per_sender: u64, + #[structopt(name = "max-mem-usage", long, default_value = "134217728")] /// Maximal memory usage. pub max_mem_usage: u64, } diff --git a/miner/src/generate_block_event_pacemaker.rs b/miner/src/generate_block_event_pacemaker.rs index 14e35d0bf8..1f52036a67 100644 --- a/miner/src/generate_block_event_pacemaker.rs +++ b/miner/src/generate_block_event_pacemaker.rs @@ -45,7 +45,7 @@ impl ActorService for GenerateBlockEventPacemaker { ctx.subscribe::(); ctx.subscribe::(); //if mint empty block is disabled, trigger mint event for on demand mint (Dev) - if !self.config.miner.enable_mint_empty_block { + if self.config.miner.disable_mint_empty_block { ctx.subscribe::(); } Ok(()) @@ -54,7 +54,7 @@ impl ActorService for GenerateBlockEventPacemaker { fn stopped(&mut self, ctx: &mut ServiceContext) -> Result<()> { ctx.unsubscribe::(); ctx.unsubscribe::(); - if !self.config.miner.enable_mint_empty_block { + if self.config.miner.disable_mint_empty_block { ctx.unsubscribe::(); } Ok(()) diff --git a/miner/src/lib.rs b/miner/src/lib.rs index 559656ccac..361e106285 100644 --- a/miner/src/lib.rs +++ b/miner/src/lib.rs @@ -72,9 +72,9 @@ impl MinerService { .send(CreateBlockTemplateRequest) .await? })?; - if block_template.body.transactions.is_empty() && !self.config.miner.enable_mint_empty_block + if block_template.body.transactions.is_empty() && self.config.miner.disable_mint_empty_block { - debug!("The flag enable_mint_empty_block is false and no txn in pool, so skip mint empty block."); + debug!("The flag disable_mint_empty_block is true and no txn in pool, so skip mint empty block."); Ok(()) } else { debug!("Mint block template: {:?}", block_template); diff --git a/miner/tests/miner_test.rs b/miner/tests/miner_test.rs index cbf70d4230..0b34abf123 100644 --- a/miner/tests/miner_test.rs +++ b/miner/tests/miner_test.rs @@ -22,7 +22,7 @@ use types::{ #[stest::test] fn test_miner() { let mut config = NodeConfig::random_for_test(); - config.miner.enable_miner_client = false; + config.miner.disable_miner_client = false; let config = Arc::new(config); let handle = test_helper::run_node_by_config(config.clone()).unwrap(); let bus = handle.bus().unwrap(); @@ -60,7 +60,7 @@ fn test_miner() { #[stest::test] async fn test_miner_service() { let mut config = NodeConfig::random_for_test(); - config.miner.enable_mint_empty_block = true; + config.miner.disable_mint_empty_block = false; let registry = RegistryService::launch(); let node_config = Arc::new(config.clone()); registry.put_shared(node_config.clone()).await.unwrap(); diff --git a/node/src/node.rs b/node/src/node.rs index 54daedd1c8..ceeeaf7ae2 100644 --- a/node/src/node.rs +++ b/node/src/node.rs @@ -129,14 +129,14 @@ impl NodeService { config.logger.max_backup, ); } - if config.logger.enable_stderr { + if config.logger.disable_stderr { logger_handle.enable_stderr(); } else { logger_handle.disable_stderr(); } // start metric server - if config.metrics.enable_metrics { + if !config.metrics.disable_metrics { starcoin_metrics::metric_server::start_server( config.metrics.address.clone(), config.metrics.port, @@ -246,7 +246,7 @@ impl NodeService { registry.register::().await?; registry.register::().await?; - if config.miner.enable_miner_client { + if config.miner.disable_miner_client { let miner_client_config = config.miner.client_config.clone(); registry.put_shared(miner_client_config).await?; let job_client = JobBusClient::new(bus.clone(), config.net().time_service()); diff --git a/rpc/client/tests/client_server_test.rs b/rpc/client/tests/client_server_test.rs index cbd9ace363..c3006db2de 100644 --- a/rpc/client/tests/client_server_test.rs +++ b/rpc/client/tests/client_server_test.rs @@ -19,7 +19,7 @@ async fn test_in_async() -> Result<()> { fn do_client_test() -> Result<()> { let mut node_config = NodeConfig::random_for_test(); - node_config.miner.enable_miner_client = false; + node_config.miner.disable_miner_client = false; let config = Arc::new(node_config); let ipc_file = config.rpc.get_ipc_file().to_path_buf(); let url = config.rpc.get_ws_address().unwrap(); @@ -60,7 +60,7 @@ fn test_multi_client() -> Result<()> { #[stest::test(timeout = 120)] fn test_client_reconnect() -> Result<()> { let mut node_config = NodeConfig::random_for_test(); - node_config.miner.enable_miner_client = false; + node_config.miner.disable_miner_client = false; let config = Arc::new(node_config); let url = config.rpc.get_ws_address().unwrap(); debug!("url:{}", url); @@ -93,7 +93,7 @@ fn test_client_reconnect() -> Result<()> { #[stest::test(timeout = 120)] fn test_client_reconnect_subscribe() -> Result<()> { let mut node_config = NodeConfig::random_for_test(); - node_config.miner.enable_miner_client = true; + node_config.miner.disable_miner_client = true; let config = Arc::new(node_config); let url = config.rpc.get_ws_address().unwrap(); debug!("url:{}", url); diff --git a/sync/tests/full_sync_test.rs b/sync/tests/full_sync_test.rs index b2ef58a23f..c46cb15131 100644 --- a/sync/tests/full_sync_test.rs +++ b/sync/tests/full_sync_test.rs @@ -13,7 +13,7 @@ use traits::ChainAsyncService; #[stest::test(timeout = 120)] fn test_full_sync() { - test_sync::test_sync(SyncMode::FULL) + test_sync::test_sync(SyncMode::Full) } #[ignore] @@ -30,7 +30,7 @@ fn test_sync_by_notification() { let mut second_config = NodeConfig::random_for_test(); info!("second peer : {:?}", second_config.network.self_peer_id()); second_config.network.seeds = vec![first_config.network.self_address()]; - second_config.miner.enable_miner_client = false; + second_config.miner.disable_miner_client = false; let second_node = run_node_by_config(Arc::new(second_config)).unwrap(); // stop sync service and just use notification message to sync. @@ -88,7 +88,7 @@ fn test_broadcast_with_difficulty() { info!("second peer : {:?}", second_config.network.self_peer_id()); second_config.network.seeds = vec![first_config.network.self_address()]; //second_config.miner.enable_miner_client = false; - second_config.sync.set_mode(SyncMode::FULL); + second_config.sync.set_mode(SyncMode::Full); let second_node = run_node_by_config(Arc::new(second_config)).unwrap(); let second_chain = second_node.chain_service().unwrap(); diff --git a/sync/tests/test_sync/mod.rs b/sync/tests/test_sync/mod.rs index 283c5ba5bf..f0c2273ae1 100644 --- a/sync/tests/test_sync/mod.rs +++ b/sync/tests/test_sync/mod.rs @@ -25,7 +25,7 @@ pub fn test_sync(sync_mode: SyncMode) { let mut second_config = NodeConfig::random_for_test(); info!("second peer : {:?}", second_config.network.self_peer_id()); second_config.network.seeds = vec![first_config.network.self_address()]; - second_config.miner.enable_miner_client = false; + second_config.miner.disable_miner_client = false; second_config.sync.set_mode(sync_mode); let second_node = run_node_by_config(Arc::new(second_config)).unwrap(); diff --git a/sync/tests/txn_sync_test.rs b/sync/tests/txn_sync_test.rs index 1f787338ac..4cb177c41e 100644 --- a/sync/tests/txn_sync_test.rs +++ b/sync/tests/txn_sync_test.rs @@ -15,7 +15,7 @@ use txpool::TxPoolService; #[stest::test] fn test_txn_sync_actor() { let mut first_config = NodeConfig::random_for_test(); - first_config.miner.enable_miner_client = false; + first_config.miner.disable_miner_client = false; let first_network_address = first_config.network.self_address(); let first_config = Arc::new(first_config); let first_node = run_node_by_config(first_config.clone()).unwrap(); @@ -31,7 +31,7 @@ fn test_txn_sync_actor() { let mut second_config = NodeConfig::random_for_test(); second_config.network.seeds = vec![first_network_address]; - second_config.miner.enable_miner_client = false; + second_config.miner.disable_miner_client = false; let second_config = Arc::new(second_config); let second_node = run_node_by_config(second_config.clone()).unwrap(); From 8a14378332c0fe0f4b1ded39726ec5b3cb20836e Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Wed, 6 Jan 2021 10:35:37 +0800 Subject: [PATCH 07/10] [config]update txpool config for system free memory --- commons/system/Cargo.toml | 11 +++++++++++ commons/system/src/lib.rs | 14 ++++++++++++++ config/Cargo.toml | 1 + config/src/lib.rs | 2 +- config/src/miner_config.rs | 19 ++++++++++++++++--- config/src/rpc_config.rs | 1 - config/src/sync_config.rs | 2 +- config/src/txpool_config.rs | 14 +++++++++++++- node/src/node.rs | 2 +- 9 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 commons/system/Cargo.toml create mode 100644 commons/system/src/lib.rs diff --git a/commons/system/Cargo.toml b/commons/system/Cargo.toml new file mode 100644 index 0000000000..0855e84801 --- /dev/null +++ b/commons/system/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "starcoin-system" +version = "0.9.4" +authors = ["Starcoin Core Dev "] +license = "Apache-2.0" +publish = false +edition = "2018" + +[dependencies] +systemstat ="0.1.6" +anyhow = "1.0.37" \ No newline at end of file diff --git a/commons/system/src/lib.rs b/commons/system/src/lib.rs new file mode 100644 index 0000000000..8cc36cc43a --- /dev/null +++ b/commons/system/src/lib.rs @@ -0,0 +1,14 @@ +// Copyright (c) The Starcoin Core Contributors +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use systemstat::{Platform, System}; + +pub fn get_free_mem_size() -> Result { + let sys = System::new(); + let free = match sys.memory() { + Ok(mem) => mem.free.as_u64(), + Err(_x) => 0u64, + }; + Ok(free) +} diff --git a/config/Cargo.toml b/config/Cargo.toml index ff0253cb2c..3006f9f7c9 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -28,3 +28,4 @@ starcoin-vm-types = { path = "../vm/types" } network-p2p-types = { path = "../network-p2p/types"} starcoin-logger = {path = "../commons/logger", package="starcoin-logger"} diem-temppath = { git = "https://github.com/starcoinorg/diem", rev="a69729b2d54af44d2f779bcf167e3f6d681a9821" } +starcoin-system = {path = "../commons/system", package="starcoin-system"} diff --git a/config/src/lib.rs b/config/src/lib.rs index d8636dae1c..3ce9654b62 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -190,7 +190,7 @@ pub struct StarcoinOpt { /// Node network private key file, only work for first init. pub node_key_file: Option, - #[structopt(long = "sync-mode", short = "s")] + #[structopt(long = "sync-mode", short = "s", possible_values = &SyncMode::variants(), case_insensitive = false)] /// Sync mode. Included value(full, fast, light). pub sync_mode: Option, diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index 33e0e86929..fce71241c9 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -4,22 +4,29 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; +use structopt::StructOpt; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct MinerConfig { + #[structopt(long = "disable-mint-empty-block")] pub disable_mint_empty_block: bool, + #[structopt(long = "block-gas-limit")] pub block_gas_limit: Option, + #[structopt(long = "disable-miner-client")] pub disable_miner_client: bool, + #[structopt(flatten)] pub client_config: MinerClientConfig, } -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct MinerClientConfig { pub server: Option, pub plugin_path: Option, + #[structopt(long = "thread-num")] pub thread_num: u16, + #[structopt(long = "enable-stderr")] #[serde(skip)] pub enable_stderr: bool, } @@ -54,7 +61,13 @@ impl ConfigModule for MinerConfig { .disable_mint_empty_block .as_ref() .cloned() - .unwrap_or_else(|| base.net.is_dev()); + .unwrap_or_else(|| { + if base.net.is_dev() || base.net.is_test() { + false + } else { + true + } + }); self.disable_mint_empty_block = disable_mint_empty_block; if let Some(thread) = opt.miner_thread { self.client_config.thread_num = thread; diff --git a/config/src/rpc_config.rs b/config/src/rpc_config.rs index 48cbe05f0d..e4dbef1f7d 100644 --- a/config/src/rpc_config.rs +++ b/config/src/rpc_config.rs @@ -289,7 +289,6 @@ impl ConfigModule for RpcConfig { config.ws.port = ports[2]; } else if base.net.is_dev() { config.http.port = get_available_port_from(DEFAULT_HTTP_PORT); - config.tcp.port = get_available_port_from(DEFAULT_TCP_PORT); config.ws.port = get_available_port_from(DEFAULT_WEB_SOCKET_PORT); } diff --git a/config/src/sync_config.rs b/config/src/sync_config.rs index f19a246950..6140826cbd 100644 --- a/config/src/sync_config.rs +++ b/config/src/sync_config.rs @@ -11,7 +11,7 @@ use structopt::StructOpt; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct SyncConfig { - #[structopt(long, possible_values = &SyncMode::variants(), case_insensitive = true)] + #[structopt(long, possible_values = &SyncMode::variants(), case_insensitive = false)] /// Sync mode: Light, Fast, Full, eg. sync_mode: SyncMode, } diff --git a/config/src/txpool_config.rs b/config/src/txpool_config.rs index aa4a2db04b..74a756e28d 100644 --- a/config/src/txpool_config.rs +++ b/config/src/txpool_config.rs @@ -4,7 +4,9 @@ use crate::{BaseConfig, ConfigModule, StarcoinOpt}; use anyhow::Result; use serde::{Deserialize, Serialize}; +use starcoin_system::get_free_mem_size; use structopt::StructOpt; +pub const DEFAULT_MEM_SIZE: u64 = 128 * 1024 * 1024; // 128M #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, StructOpt)] #[serde(deny_unknown_fields)] @@ -22,10 +24,20 @@ pub struct TxPoolConfig { impl ConfigModule for TxPoolConfig { fn default_with_opt(_opt: &StarcoinOpt, _base: &BaseConfig) -> Result { + let free_mem = match get_free_mem_size() { + Ok(free) => { + if free > 0 { + free / 2 + } else { + DEFAULT_MEM_SIZE + } + } + Err(_) => DEFAULT_MEM_SIZE, + }; Ok(Self { max_count: 4096, max_per_sender: 128, - max_mem_usage: 128 * 1024 * 1024, // 128M + max_mem_usage: free_mem, }) } } diff --git a/node/src/node.rs b/node/src/node.rs index ceeeaf7ae2..d789abc5f8 100644 --- a/node/src/node.rs +++ b/node/src/node.rs @@ -246,7 +246,7 @@ impl NodeService { registry.register::().await?; registry.register::().await?; - if config.miner.disable_miner_client { + if !config.miner.disable_miner_client { let miner_client_config = config.miner.client_config.clone(); registry.put_shared(miner_client_config).await?; let job_client = JobBusClient::new(bus.clone(), config.net().time_service()); From 743ce5b7de92f4129b015470b799747cf6fb6123 Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Wed, 6 Jan 2021 10:58:37 +0800 Subject: [PATCH 08/10] [config]fix clippy check --- config/src/miner_config.rs | 8 +------- config/src/rpc_config.rs | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index fce71241c9..28e3ca71d6 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -61,13 +61,7 @@ impl ConfigModule for MinerConfig { .disable_mint_empty_block .as_ref() .cloned() - .unwrap_or_else(|| { - if base.net.is_dev() || base.net.is_test() { - false - } else { - true - } - }); + .unwrap_or_else(|| !(base.net.is_dev() || base.net.is_test())); self.disable_mint_empty_block = disable_mint_empty_block; if let Some(thread) = opt.miner_thread { self.client_config.thread_num = thread; diff --git a/config/src/rpc_config.rs b/config/src/rpc_config.rs index e4dbef1f7d..776393cf13 100644 --- a/config/src/rpc_config.rs +++ b/config/src/rpc_config.rs @@ -307,7 +307,7 @@ impl ConfigModule for RpcConfig { self.http.port = opt.http.port; self.http.max_request_body_size = opt.http.max_request_body_size; if opt.http.threads.is_some() { - self.http.threads = opt.http.threads.clone(); + self.http.threads = opt.http.threads; } if opt.http.ip_headers.is_some() { self.http.ip_headers = opt.http.ip_headers.clone(); From 33dd3472987b05e7133f8ea29342e3c3f101c678 Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Wed, 6 Jan 2021 12:27:42 +0800 Subject: [PATCH 09/10] [config]fix test failure --- rpc/client/tests/client_server_test.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/rpc/client/tests/client_server_test.rs b/rpc/client/tests/client_server_test.rs index c3006db2de..a667ca0f92 100644 --- a/rpc/client/tests/client_server_test.rs +++ b/rpc/client/tests/client_server_test.rs @@ -18,8 +18,7 @@ async fn test_in_async() -> Result<()> { } fn do_client_test() -> Result<()> { - let mut node_config = NodeConfig::random_for_test(); - node_config.miner.disable_miner_client = false; + let node_config = NodeConfig::random_for_test(); let config = Arc::new(node_config); let ipc_file = config.rpc.get_ipc_file().to_path_buf(); let url = config.rpc.get_ws_address().unwrap(); @@ -92,8 +91,7 @@ fn test_client_reconnect() -> Result<()> { #[stest::test(timeout = 120)] fn test_client_reconnect_subscribe() -> Result<()> { - let mut node_config = NodeConfig::random_for_test(); - node_config.miner.disable_miner_client = true; + let node_config = NodeConfig::random_for_test(); let config = Arc::new(node_config); let url = config.rpc.get_ws_address().unwrap(); debug!("url:{}", url); From 88306754f82225129eb6c42044b62c3950ce5a06 Mon Sep 17 00:00:00 2001 From: chengsuoyuan Date: Wed, 6 Jan 2021 15:07:06 +0800 Subject: [PATCH 10/10] [config]update disable_mint_empty_block to option --- config/src/miner_config.rs | 33 +++++++++++++-------- miner/src/generate_block_event_pacemaker.rs | 4 +-- miner/src/lib.rs | 3 +- miner/tests/miner_test.rs | 2 +- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index 28e3ca71d6..6556e6b6eb 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -10,7 +10,7 @@ use structopt::StructOpt; #[serde(deny_unknown_fields)] pub struct MinerConfig { #[structopt(long = "disable-mint-empty-block")] - pub disable_mint_empty_block: bool, + pub disable_mint_empty_block: Option, #[structopt(long = "block-gas-limit")] pub block_gas_limit: Option, #[structopt(long = "disable-miner-client")] @@ -19,6 +19,16 @@ pub struct MinerConfig { pub client_config: MinerClientConfig, } +impl MinerConfig { + pub fn is_disable_mint_empty_block(&self) -> bool { + if let Some(disable) = self.disable_mint_empty_block { + disable + } else { + false + } + } +} + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, StructOpt)] #[serde(deny_unknown_fields)] pub struct MinerClientConfig { @@ -40,7 +50,7 @@ impl ConfigModule for MinerConfig { .cloned() .unwrap_or_else(|| base.net.is_dev()); Ok(Self { - disable_mint_empty_block, + disable_mint_empty_block: Some(disable_mint_empty_block), block_gas_limit: None, disable_miner_client: opt.disable_miner_client, client_config: MinerClientConfig { @@ -54,18 +64,17 @@ impl ConfigModule for MinerConfig { fn after_load(&mut self, opt: &StarcoinOpt, base: &BaseConfig) -> Result<()> { // only dev network is on demand mine at default. + let disable_mint_empty_block = opt + .disable_mint_empty_block + .as_ref() + .cloned() + .unwrap_or_else(|| base.net.is_dev()); + self.disable_mint_empty_block = Some(disable_mint_empty_block); + if let Some(thread) = opt.miner_thread { + self.client_config.thread_num = thread; + } if opt.disable_miner_client { self.disable_miner_client = true; - } else { - let disable_mint_empty_block = opt - .disable_mint_empty_block - .as_ref() - .cloned() - .unwrap_or_else(|| !(base.net.is_dev() || base.net.is_test())); - self.disable_mint_empty_block = disable_mint_empty_block; - if let Some(thread) = opt.miner_thread { - self.client_config.thread_num = thread; - } } Ok(()) } diff --git a/miner/src/generate_block_event_pacemaker.rs b/miner/src/generate_block_event_pacemaker.rs index 1f52036a67..c9fe79e744 100644 --- a/miner/src/generate_block_event_pacemaker.rs +++ b/miner/src/generate_block_event_pacemaker.rs @@ -45,7 +45,7 @@ impl ActorService for GenerateBlockEventPacemaker { ctx.subscribe::(); ctx.subscribe::(); //if mint empty block is disabled, trigger mint event for on demand mint (Dev) - if self.config.miner.disable_mint_empty_block { + if self.config.miner.is_disable_mint_empty_block() { ctx.subscribe::(); } Ok(()) @@ -54,7 +54,7 @@ impl ActorService for GenerateBlockEventPacemaker { fn stopped(&mut self, ctx: &mut ServiceContext) -> Result<()> { ctx.unsubscribe::(); ctx.unsubscribe::(); - if self.config.miner.disable_mint_empty_block { + if self.config.miner.is_disable_mint_empty_block() { ctx.unsubscribe::(); } Ok(()) diff --git a/miner/src/lib.rs b/miner/src/lib.rs index 361e106285..11256ca699 100644 --- a/miner/src/lib.rs +++ b/miner/src/lib.rs @@ -72,7 +72,8 @@ impl MinerService { .send(CreateBlockTemplateRequest) .await? })?; - if block_template.body.transactions.is_empty() && self.config.miner.disable_mint_empty_block + if block_template.body.transactions.is_empty() + && self.config.miner.is_disable_mint_empty_block() { debug!("The flag disable_mint_empty_block is true and no txn in pool, so skip mint empty block."); Ok(()) diff --git a/miner/tests/miner_test.rs b/miner/tests/miner_test.rs index 0b34abf123..5ba3d9d3db 100644 --- a/miner/tests/miner_test.rs +++ b/miner/tests/miner_test.rs @@ -60,7 +60,7 @@ fn test_miner() { #[stest::test] async fn test_miner_service() { let mut config = NodeConfig::random_for_test(); - config.miner.disable_mint_empty_block = false; + config.miner.disable_mint_empty_block = Some(false); let registry = RegistryService::launch(); let node_config = Arc::new(config.clone()); registry.put_shared(node_config.clone()).await.unwrap();