Companion for paritytech/substrate#12795 (#6374)

* Begin removing `parity-util-mem`; remove `collect_memory_stats` * Update some dependencies that were using `parity-util-mem` * Remove `trie-memory-tracker` feature * Update Cargo.lock * Update `kvdb-shared-tests` * Add back jemalloc * Add missing license header * update lockfile for {"substrate"} Co-authored-by: parity-processbot <> Co-authored-by: Andronik <write@reusable.software>
paritytech · Dec 6, 2022 · 9cf336a · 9cf336a
1 parent 1c8c54b
commit 9cf336a
Show file tree

Hide file tree

Showing 22 changed files with 291 additions and 395 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -21,7 +21,7 @@ version = "0.9.33"
 [dependencies]
 polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ]  }
 color-eyre = { version = "0.6.1", default-features = false }
-parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }
+tikv-jemallocator = "0.5.0"
 
 [dev-dependencies]
 assert_cmd = "2.0.4"

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
@@ -35,15 +35,11 @@ polkadot-node-metrics = { path = "../node/metrics" }
 sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-sysinfo = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
-# this crate is used only to enable `trie-memory-tracker` feature
-# see https://github.com/paritytech/substrate/pull/6745
-sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
-
 [build-dependencies]
 substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [features]
-default = ["db", "cli", "hostperfcheck", "full-node", "trie-memory-tracker", "polkadot-native"]
+default = ["db", "cli", "hostperfcheck", "full-node", "polkadot-native"]
 db = ["service/db"]
 cli = [
 	"clap",
@@ -60,7 +56,6 @@ runtime-benchmarks = [
 	"polkadot-node-metrics/runtime-benchmarks",
 	"polkadot-performance-test?/runtime-benchmarks"
 ]
-trie-memory-tracker = ["sp-trie/memory-tracker"]
 full-node = ["service/full-node"]
 try-runtime = ["service/try-runtime", "try-runtime-cli/try-runtime"]
 fast-runtime = ["service/fast-runtime"]

diff --git a/core-primitives/Cargo.toml b/core-primitives/Cargo.toml
@@ -10,7 +10,6 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", d
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
 parity-scale-codec = { version = "3.1.5", default-features = false, features = [ "derive" ] }
-parity-util-mem = { version = "0.12.0", default-features = false, optional = true }
 
 [features]
 default = [ "std" ]
@@ -21,5 +20,4 @@ std = [
 	"sp-std/std",
 	"scale-info/std",
 	"parity-scale-codec/std",
-	"parity-util-mem",
 ]
diff --git a/core-primitives/src/lib.rs b/core-primitives/src/lib.rs
@@ -21,8 +21,6 @@
 //! These core Polkadot types are used by the relay chain and the Parachains.
 
 use parity_scale_codec::{Decode, Encode};
-#[cfg(feature = "std")]
-use parity_util_mem::MallocSizeOf;
 use scale_info::TypeInfo;
 use sp_runtime::{
 	generic,
@@ -66,7 +64,6 @@ pub type Hash = sp_core::H256;
 ///
 /// This type makes it easy to enforce that a hash is a candidate hash on the type level.
 #[derive(Clone, Copy, Encode, Decode, Hash, Eq, PartialEq, Default, PartialOrd, Ord, TypeInfo)]
-#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct CandidateHash(pub Hash);
 
 #[cfg(feature = "std")]
@@ -127,7 +124,6 @@ pub type DownwardMessage = sp_std::vec::Vec<u8>;
 /// A wrapped version of `DownwardMessage`. The difference is that it has attached the block number when
 /// the message was sent.
 #[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq, TypeInfo)]
-#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct InboundDownwardMessage<BlockNumber = crate::BlockNumber> {
 	/// The block number at which these messages were put into the downward message queue.
 	pub sent_at: BlockNumber,
@@ -137,7 +133,6 @@ pub struct InboundDownwardMessage<BlockNumber = crate::BlockNumber> {
 
 /// An HRMP message seen from the perspective of a recipient.
 #[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq, TypeInfo)]
-#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct InboundHrmpMessage<BlockNumber = crate::BlockNumber> {
 	/// The block number at which this message was sent.
 	/// Specifically, it is the block number at which the candidate that sends this message was
@@ -149,7 +144,6 @@ pub struct InboundHrmpMessage<BlockNumber = crate::BlockNumber> {
 
 /// An HRMP message seen from the perspective of a sender.
 #[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq, Eq, Hash, TypeInfo)]
-#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct OutboundHrmpMessage<Id> {
 	/// The para that will get this message in its downward message queue.
 	pub recipient: Id,

diff --git a/node/core/approval-voting/Cargo.toml b/node/core/approval-voting/Cargo.toml
@@ -13,7 +13,7 @@ bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
 lru = "0.8"
 merlin = "2.0"
 schnorrkel = "0.9.1"
-kvdb = "0.12.0"
+kvdb = "0.13.0"
 derive_more = "0.99.17"
 thiserror = "1.0.31"
 
@@ -40,5 +40,5 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
diff --git a/node/core/av-store/Cargo.toml b/node/core/av-store/Cargo.toml
@@ -7,7 +7,7 @@ edition.workspace = true
 [dependencies]
 futures = "0.3.21"
 futures-timer = "3.0.2"
-kvdb = "0.12.0"
+kvdb = "0.13.0"
 thiserror = "1.0.31"
 gum = { package = "tracing-gum", path = "../../gum" }
 bitvec = "1.0.0"
@@ -24,7 +24,7 @@ polkadot-node-primitives = { path = "../../primitives" }
 log = "0.4.17"
 env_logger = "0.9.0"
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"
 
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }

diff --git a/node/core/chain-selection/Cargo.toml b/node/core/chain-selection/Cargo.toml
@@ -13,7 +13,7 @@ polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-primitives = { path = "../../primitives" }
 polkadot-node-subsystem = { path = "../../subsystem" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }
-kvdb = "0.12.0"
+kvdb = "0.13.0"
 thiserror = "1.0.31"
 parity-scale-codec = "3.1.5"
 
@@ -22,4 +22,4 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 parking_lot = "0.12.0"
 assert_matches = "1"
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"
diff --git a/node/core/dispute-coordinator/Cargo.toml b/node/core/dispute-coordinator/Cargo.toml
@@ -8,7 +8,7 @@ edition.workspace = true
 futures = "0.3.21"
 gum = { package = "tracing-gum", path = "../../gum" }
 parity-scale-codec = "3.1.5"
-kvdb = "0.12.0"
+kvdb = "0.13.0"
 thiserror = "1.0.31"
 lru = "0.8.0"
 fatality = "0.0.6"
@@ -22,7 +22,7 @@ sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste
 
 
 [dev-dependencies]
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }

diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml
@@ -23,7 +23,6 @@ polkadot-node-core-backing = { path = "../core/backing" }
 polkadot-node-primitives = { path = "../primitives" }
 polkadot-primitives = { path = "../../primitives" }
 polkadot-node-core-pvf = { path = "../core/pvf" }
-parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }
 color-eyre = { version = "0.6.1", default-features = false }
 assert_matches = "1.5"
 async-trait = "0.1.57"

diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml
@@ -18,9 +18,9 @@ polkadot-primitives = { path = "../../primitives" }
 orchestra = "0.0.2"
 gum = { package = "tracing-gum", path = "../gum" }
 lru = "0.8"
-parity-util-mem = { version = "0.12.0", default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 async-trait = "0.1.57"
+tikv-jemalloc-ctl = "0.5.0"
 
 [dev-dependencies]
 metered = { package = "prioritized-metered-channel", version = "0.2.0" }

diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs
@@ -101,8 +101,6 @@ pub use polkadot_node_metrics::{
 	Metronome,
 };
 
-use parity_util_mem::MemoryAllocationTracker;
-
 pub use orchestra as gen;
 pub use orchestra::{
 	contextbounds, orchestra, subsystem, FromOrchestra, MapSubsystem, MessagePacket,
@@ -118,11 +116,14 @@ pub const KNOWN_LEAVES_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(2 * 24
 	None => panic!("Known leaves cache size must be non-zero"),
 };
 
+mod memory_stats;
 #[cfg(test)]
 mod tests;
 
 use sp_core::traits::SpawnNamed;
 
+use memory_stats::MemoryAllocationTracker;
+
 /// Glue to connect `trait orchestra::Spawner` and `SpawnNamed` from `substrate`.
 pub struct SpawnGlue<S>(pub S);
 

diff --git a/node/overseer/src/memory_stats.rs b/node/overseer/src/memory_stats.rs
@@ -0,0 +1,53 @@
+// Copyright 2017-2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use tikv_jemalloc_ctl::{epoch, stats, Error};
+
+#[derive(Clone)]
+pub struct MemoryAllocationTracker {
+	epoch: tikv_jemalloc_ctl::epoch_mib,
+	allocated: stats::allocated_mib,
+	resident: stats::resident_mib,
+}
+
+impl MemoryAllocationTracker {
+	pub fn new() -> Result<Self, Error> {
+		Ok(Self {
+			epoch: epoch::mib()?,
+			allocated: stats::allocated::mib()?,
+			resident: stats::resident::mib()?,
+		})
+	}
+
+	pub fn snapshot(&self) -> Result<MemoryAllocationSnapshot, Error> {
+		// update stats by advancing the allocation epoch
+		self.epoch.advance()?;
+
+		let allocated: u64 = self.allocated.read()? as _;
+		let resident: u64 = self.resident.read()? as _;
+		Ok(MemoryAllocationSnapshot { allocated, resident })
+	}
+}
+
+/// Snapshot of collected memory metrics.
+#[non_exhaustive]
+#[derive(Debug, Clone)]
+pub struct MemoryAllocationSnapshot {
+	/// Total resident memory, in bytes.
+	pub resident: u64,
+	/// Total allocated memory, in bytes.
+	pub allocated: u64,
+}
diff --git a/node/overseer/src/metrics.rs b/node/overseer/src/metrics.rs
@@ -19,7 +19,7 @@
 use super::*;
 pub use polkadot_node_metrics::metrics::{self, prometheus, Metrics as MetricsTrait};
 
-use parity_util_mem::MemoryAllocationSnapshot;
+use memory_stats::MemoryAllocationSnapshot;
 
 /// Overseer Prometheus metrics.
 #[derive(Clone)]

diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml
@@ -71,8 +71,8 @@ gum = { package = "tracing-gum", path = "../gum/" }
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
 thiserror = "1.0.31"
-kvdb = "0.12.0"
-kvdb-rocksdb = { version = "0.16.0", optional = true }
+kvdb = "0.13.0"
+kvdb-rocksdb = { version = "0.17.0", optional = true }
 parity-db = { version = "0.4.2", optional = true }
 
 async-trait = "0.1.57"

diff --git a/node/subsystem-util/Cargo.toml b/node/subsystem-util/Cargo.toml
@@ -32,8 +32,7 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
-kvdb = "0.12.0"
-parity-util-mem = { version = "0.12.0", default-features = false }
+kvdb = "0.13.0"
 parity-db = { version = "0.4.2"}
 
 [dev-dependencies]
@@ -44,6 +43,6 @@ log = "0.4.17"
 polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }
 lazy_static = "1.4.0"
 polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" }
-kvdb-shared-tests = "0.10.0"
+kvdb-shared-tests = "0.11.0"
 tempfile = "3.1.0"
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"
diff --git a/node/subsystem-util/src/database.rs b/node/subsystem-util/src/database.rs
@@ -29,7 +29,6 @@ pub trait Database: KeyValueDB {
 pub mod kvdb_impl {
 	use super::{DBKeyValue, DBTransaction, DBValue, Database, KeyValueDB};
 	use kvdb::{DBOp, IoStats, IoStatsKind};
-	use parity_util_mem::{MallocSizeOf, MallocSizeOfOps};
 	use std::{collections::BTreeSet, io::Result};
 
 	/// Adapter implementing subsystem database
@@ -123,13 +122,6 @@ pub mod kvdb_impl {
 			self.db.has_prefix(col, prefix)
 		}
 	}
-
-	impl<D: KeyValueDB> MallocSizeOf for DbAdapter<D> {
-		fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize {
-			// ignore filter set
-			self.db.size_of(ops)
-		}
-	}
 }
 
 /// Utilities for using parity-db database.
@@ -160,12 +152,6 @@ pub mod paritydb_impl {
 		write_lock: Arc<Mutex<()>>,
 	}
 
-	impl parity_util_mem::MallocSizeOf for DbAdapter {
-		fn size_of(&self, _ops: &mut parity_util_mem::MallocSizeOfOps) -> usize {
-			unimplemented!("size_of is not supported for parity_db")
-		}
-	}
-
 	impl KeyValueDB for DbAdapter {
 		fn transaction(&self) -> DBTransaction {
 			DBTransaction::new()

diff --git a/parachain/Cargo.toml b/parachain/Cargo.toml
@@ -10,7 +10,6 @@ version.workspace = true
 # this crate for WASM. This is critical to avoid forcing all parachain WASM into implementing
 # various unnecessary Substrate-specific endpoints.
 parity-scale-codec = { version = "3.1.5", default-features = false, features = [ "derive" ] }
-parity-util-mem = { version = "0.12.0", default-features = false, optional = true }
 scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
 sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
@@ -32,7 +31,6 @@ std = [
 	"sp-std/std",
 	"sp-runtime/std",
 	"sp-core/std",
-	"parity-util-mem",
 	"polkadot-core-primitives/std",
 	"frame-support/std",
 ]

diff --git a/parachain/src/primitives.rs b/parachain/src/primitives.rs
@@ -31,9 +31,6 @@ use serde::{Deserialize, Serialize};
 #[cfg(feature = "std")]
 use sp_core::bytes;
 
-#[cfg(feature = "std")]
-use parity_util_mem::MallocSizeOf;
-
 use polkadot_core_primitives::{Hash, OutboundHrmpMessage};
 
 /// Block number type used by the relay chain.
@@ -43,7 +40,7 @@ pub use polkadot_core_primitives::BlockNumber as RelayChainBlockNumber;
 #[derive(
 	PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode, RuntimeDebug, derive_more::From, TypeInfo,
 )]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf, Default))]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, Default))]
 pub struct HeadData(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec<u8>);
 
 impl HeadData {
@@ -55,7 +52,7 @@ impl HeadData {
 
 /// Parachain validation code.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, derive_more::From, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash))]
 pub struct ValidationCode(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec<u8>);
 
 impl ValidationCode {
@@ -71,7 +68,6 @@ impl ValidationCode {
 ///
 /// This type makes it easy to enforce that a hash is a validation code hash on the type level.
 #[derive(Clone, Copy, Encode, Decode, Hash, Eq, PartialEq, PartialOrd, Ord, TypeInfo)]
-#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct ValidationCodeHash(Hash);
 
 impl sp_std::fmt::Display for ValidationCodeHash {
@@ -114,7 +110,7 @@ impl sp_std::fmt::LowerHex for ValidationCodeHash {
 ///
 /// Contains everything required to validate para-block, may contain block and witness data.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, derive_more::From, TypeInfo, RuntimeDebug)]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, MallocSizeOf))]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
 pub struct BlockData(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec<u8>);
 
 /// Unique identifier of a parachain.
@@ -134,10 +130,7 @@ pub struct BlockData(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec
 	RuntimeDebug,
 	TypeInfo,
 )]
-#[cfg_attr(
-	feature = "std",
-	derive(serde::Serialize, serde::Deserialize, derive_more::Display, MallocSizeOf)
-)]
+#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, derive_more::Display))]
 pub struct Id(u32);
 
 impl TypeId for Id {