service: storage monitor added

Cargo.toml

@@ -64,6 +64,7 @@ members = [
  "client/service",
  "client/service/test",
  "client/state-db",
+ "client/storage-monitor",
  "client/sysinfo",
  "client/sync-state-rpc",
  "client/telemetry",

bin/node/cli/Cargo.toml

@@ -81,6 +81,7 @@ sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" }
 sc-authority-discovery = { version = "0.10.0-dev", path = "../../../client/authority-discovery" }
 sc-sync-state-rpc = { version = "0.10.0-dev", path = "../../../client/sync-state-rpc" }
 sc-sysinfo = { version = "6.0.0-dev", path = "../../../client/sysinfo" }
+sc-storage-monitor = { version = "0.1.0", path = "../../../client/storage-monitor" }
 
 # frame dependencies
 frame-system = { version = "4.0.0-dev", path = "../../../frame/system" }
@@ -138,6 +139,7 @@ substrate-frame-cli = { version = "4.0.0-dev", optional = true, path = "../../..
 try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" }
 sc-cli = { version = "0.10.0-dev", path = "../../../client/cli", optional = true }
 pallet-balances = { version = "4.0.0-dev", path = "../../../frame/balances" }
+sc-storage-monitor = { version = "0.1.0", path = "../../../client/storage-monitor" }
 
 [features]
 default = ["cli"]

bin/node/cli/src/cli.rs

@@ -36,6 +36,10 @@ pub struct Cli {
  /// telemetry, if telemetry is enabled.
  #[arg(long)]
  pub no_hardware_benchmarks: bool,
+
+ #[allow(missing_docs)]
+ #[clap(flatten)]
+ pub storage_monitor: sc_storage_monitor::StorageMonitorParams,
 }
 
 /// Possible subcommands of the main binary.

bin/node/cli/src/command.rs

@@ -87,8 +87,7 @@ pub fn run() -> Result<()> {
  None => {
  let runner = cli.create_runner(&cli.run)?;
  runner.run_node_until_exit(|config| async move {
- service::new_full(config, cli.no_hardware_benchmarks)
- .map_err(sc_cli::Error::Service)
+ service::new_full(config, cli).map_err(sc_cli::Error::Service)
  })
  },
  Some(Subcommand::Inspect(cmd)) => {

bin/node/cli/src/service.rs

@@ -20,6 +20,7 @@
 
 //! Service implementation. Specialized wrapper over substrate service.
 
+use crate::Cli;
 use codec::Encode;
 use frame_benchmarking_cli::SUBSTRATE_REFERENCE_HARDWARE;
 use frame_system_rpc_runtime_api::AccountNonceApi;
@@ -556,12 +557,18 @@ pub fn new_full_base(
 }
 
 /// Builds a new service for a full client.
-pub fn new_full(
- config: Configuration,
- disable_hardware_benchmarks: bool,
-) -> Result<TaskManager, ServiceError> {
- new_full_base(config, disable_hardware_benchmarks, |_, _| ())
- .map(|NewFullBase { task_manager, .. }| task_manager)
+pub fn new_full(config: Configuration, cli: Cli) -> Result<TaskManager, ServiceError> {
+ let database_source = config.database.clone();
+ let task_manager = new_full_base(config, cli.no_hardware_benchmarks, |_, _| ())
+ .map(|NewFullBase { task_manager, .. }| task_manager)?;
+
+ sc_storage_monitor::StorageMonitorService::try_spawn(
+ cli.storage_monitor,
+ database_source,
+ &task_manager.spawn_essential_handle(),
+ )?;
+
+ Ok(task_manager)
 }
 
 #[cfg(test)]

client/cli/src/params/database_params.rs

@@ -19,7 +19,7 @@
 use crate::arg_enums::Database;
 use clap::Args;
 
-/// Parameters for block import.
+/// Parameters for database
 #[derive(Debug, Clone, PartialEq, Args)]
 pub struct DatabaseParams {
  /// Select database backend to use.
@@ -32,7 +32,7 @@ pub struct DatabaseParams {
 }
 
 impl DatabaseParams {
- /// Limit the memory the database cache can use.
+ /// Database backend
  pub fn database(&self) -> Option<Database> {
  self.database
  }

client/service/Cargo.toml

@@ -73,6 +73,7 @@ sc-offchain = { version = "4.0.0-dev", path = "../offchain" }
 prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev" }
 sc-tracing = { version = "4.0.0-dev", path = "../tracing" }
 sc-sysinfo = { version = "6.0.0-dev", path = "../sysinfo" }
+sc-storage-monitor = { version = "0.1.0", path = "../storage-monitor" }
 tracing = "0.1.29"
 tracing-futures = { version = "0.2.4" }
 async-trait = "0.1.57"

client/service/src/error.rs

@@ -48,6 +48,9 @@ pub enum Error {
  #[error(transparent)]
  Telemetry(#[from] sc_telemetry::Error),
 
+ #[error(transparent)]
+ Storage(#[from] sc_storage_monitor::Error),
+
  #[error("Best chain selection strategy (SelectChain) is not provided.")]
  SelectChainRequired,
 

client/storage-monitor/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "sc-storage-monitor"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2021"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository = "https://github.com/paritytech/substrate"
+description = "Storage monitor service for substrate"
+homepage = "https://substrate.io"
+
+[dependencies]
+clap = { version = "4.0.9", features = ["derive", "string"] }
+futures = "0.3.21"
+log = "0.4.17"
+nix = { version = "0.26.1", features = ["fs"] }
+sc-client-db = { version = "0.10.0-dev", default-features = false, path = "../db" }
+sc-utils = { version = "4.0.0-dev", path = "../utils" }
+sp-core = { version = "7.0.0", path = "../../primitives/core" }
+tokio = "1.22.0"
+thiserror = "1.0.30"

client/storage-monitor/src/lib.rs

@@ -0,0 +1,149 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use clap::Args;
+use nix::{errno::Errno, sys::statvfs::statvfs};
+use sc_client_db::DatabaseSource;
+use sp_core::traits::SpawnEssentialNamed;
+use std::{
+ path::{Path, PathBuf},
+ time::Duration,
+};
+
+const LOG_TARGET: &str = "storage-monitor";
+
+/// Error type used in this crate.
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("IO Error")]
+ IOError(#[from] Errno),
+ #[error("Out of storage space: available {0}MB, required {1}MB")]
+ StorageOutOfSpace(u64, u64),
+}
+
+/// Parameters used to create the storage monitor.
+#[derive(Default, Debug, Clone, Args)]
+pub struct StorageMonitorParams {
+ /// Required available space on database storage. If available space for DB storage drops below
+ /// the given threshold, node will be gracefully terminated. If `0` is given monitoring will be
+ /// disabled.
+ #[arg(long = "db-storage-threshold", value_name = "MB", default_value_t = 1000)]
+ pub threshold: u64,
+
+ /// How often available space is polled.
+ #[arg(long = "db-storage-polling-period", value_name = "SECONDS", default_value_t = 5, value_parser = clap::value_parser!(u32).range(1..))]
+ pub polling_period: u32,
+}
+
+/// Storage monitor service: checks the available space for the filesystem for fiven path.
+pub struct StorageMonitorService {
+ /// watched path
+ path: PathBuf,
+ /// number of megabytes that shall be free on the filesystem for watched path
+ threshold: u64,
+ /// storage space polling period (seconds)
+ polling_period: u32,
+}
+
+impl StorageMonitorService {
+ /// Creates new StorageMonitorService for given client config
+ pub fn try_spawn(
+ parameters: StorageMonitorParams,
+ database: DatabaseSource,
+ spawner: &impl SpawnEssentialNamed,
+ ) -> Result<(), Error> {
+ Ok(match (parameters.threshold, database.path()) {
+ (0, _) => {
+ log::info!(
+ target: LOG_TARGET,
+ "StorageMonitorService: threshold `0` given, storage monitoring disabled",
+ );
+ },
+ (_, None) => {
+ log::warn!(
+ target: LOG_TARGET,
+ "StorageMonitorService: no database path to observe",
+ );
+ },
+ (threshold, Some(path)) => {
+ log::debug!(
+ target: LOG_TARGET,
+ "Initializing StorageMonitorService for db path: {:?}",
+ path,
+ );
+
+ Self::check_free_space(&path, threshold)?;
+
+ let storage_monitor_service = StorageMonitorService {
+ path: path.to_path_buf(),
+ threshold,
+ polling_period: parameters.polling_period,
+ };
+
+ spawner.spawn_essential(
+ "storage-monitor",
+ None,
+ Box::pin(storage_monitor_service.run()),
+ );
+ },
+ })
+ }
+
+ /// Main monitoring loop, intended to be spawned as essential task. Quits if free space drop
+ /// below threshold.
+ async fn run(self) {
+ loop {
+ tokio::time::sleep(Duration::from_secs(self.polling_period.into())).await;
+ if Self::check_free_space(&self.path, self.threshold).is_err() {
+ break
+ };
+ }
+ }
+
+ /// Returns free space in MB, or error if statvfs failed.
+ fn free_space(path: &Path) -> Result<u64, Error> {
+ statvfs(path)
+ .map(|stats| stats.blocks_available() * stats.block_size() / 1_000_000)
+ .map_err(Error::from)
+ }
+
+ /// Checks if the amount of free space for given `path` is above given `threshold`.
+ /// If it dropped below, error is returned.
+ /// System errors are silently ignored.
+ fn check_free_space(path: &Path, threshold: u64) -> Result<(), Error> {
+ match StorageMonitorService::free_space(path) {
+ Ok(available_space) => {
+ log::trace!(
+ target: LOG_TARGET,
+ "free: {available_space} , threshold: {threshold}.",
+ );
+
+ if available_space < threshold {
+ log::error!(target: LOG_TARGET, "Available space {available_space}MB for path `{}` dropped below threshold: {threshold}MB , terminating...", path.display());
+ Err(Error::StorageOutOfSpace(available_space, threshold))
+ } else {
+ Ok(())
+ }
+ },
+ Err(e) => {
+ log::error!(target: LOG_TARGET, "Could not read available space: {:?}.", e);
+ Err(e)
+ },
+ }
+ }
+}