Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

update kvdb & co #6111

Merged
merged 14 commits into from
Oct 5, 2022
Merged

update kvdb & co #6111

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
687068b
toml changes
ordian Sep 20, 2022
464b717
REVERTME: patch
ordian Sep 20, 2022
a76d9de
adapt parachains db interface
ordian Sep 20, 2022
3afcbf0
fix Cargo.toml patch after master rebase
ordian Oct 5, 2022
b560ef7
fix av-store
ordian Oct 5, 2022
353f9ac
fix chain-selection
ordian Oct 5, 2022
0e708b3
fix parachains-db?
ordian Oct 5, 2022
9507f41
Revert "fix Cargo.toml patch after master rebase"
ordian Oct 5, 2022
df7aa79
Revert "REVERTME: patch"
ordian Oct 5, 2022
31b1230
Use `Ok` imported from prelude
ordian Oct 5, 2022
fdc623d
update lockfile for {"substrate"}
Oct 5, 2022
203ffe5
Revert "update lockfile for {"substrate"}"
ordian Oct 5, 2022
109017e
Merge branch 'master' into ao-update-kvdb-and-co
ordian Oct 5, 2022
8082b63
cargo update -p sp-io
ordian Oct 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ readme = "README.md"
[dependencies]
polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ] }
color-eyre = { version = "0.6.1", default-features = false }
parity-util-mem = { version = "0.11.0", default-features = false, features = ["jemalloc-global"] }
parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }

[dev-dependencies]
assert_cmd = "2.0.4"
Expand Down
2 changes: 1 addition & 1 deletion core-primitives/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", d
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
parity-scale-codec = { version = "3.1.5", default-features = false, features = [ "derive" ] }
parity-util-mem = { version = "0.11.0", default-features = false, optional = true }
parity-util-mem = { version = "0.12.0", default-features = false, optional = true }

[features]
default = [ "std" ]
Expand Down
4 changes: 2 additions & 2 deletions node/core/approval-voting/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
lru = "0.8"
merlin = "2.0"
schnorrkel = "0.9.1"
kvdb = "0.11.0"
kvdb = "0.12.0"
derive_more = "0.99.17"
thiserror = "1.0.31"

Expand All @@ -40,5 +40,5 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
assert_matches = "1.4.0"
kvdb-memorydb = "0.11.0"
kvdb-memorydb = "0.12.0"
test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
4 changes: 2 additions & 2 deletions node/core/av-store/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ edition = "2021"
[dependencies]
futures = "0.3.21"
futures-timer = "3.0.2"
kvdb = "0.11.0"
kvdb = "0.12.0"
thiserror = "1.0.31"
gum = { package = "tracing-gum", path = "../../gum" }
bitvec = "1.0.0"
Expand All @@ -24,7 +24,7 @@ polkadot-node-primitives = { path = "../../primitives" }
log = "0.4.17"
env_logger = "0.9.0"
assert_matches = "1.4.0"
kvdb-memorydb = "0.11.0"
kvdb-memorydb = "0.12.0"

sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot-node-subsystem-util = { path = "../../subsystem-util" }
Expand Down
28 changes: 15 additions & 13 deletions node/core/av-store/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -792,8 +792,9 @@ fn note_block_included(
macro_rules! peek_num {
($iter:ident) => {
match $iter.peek() {
Some((k, _)) => decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b),
None => None,
Some(Ok((k, _))) => Ok(decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b)),
Some(Err(_)) => Err($iter.next().expect("peek returned Some(Err); qed").unwrap_err()),
None => Ok(None),
}
};
}
Expand All @@ -819,10 +820,10 @@ async fn process_block_finalized<Context>(
let mut iter = subsystem
.db
.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
.take_while(|(k, _)| &k[..] < &end_prefix[..])
.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &end_prefix[..]))
cheme marked this conversation as resolved.
Show resolved Hide resolved
.peekable();

match peek_num!(iter) {
match peek_num!(iter)? {
None => break, // end of iterator.
Some(n) => n,
}
Expand Down Expand Up @@ -867,10 +868,10 @@ async fn process_block_finalized<Context>(
let iter = subsystem
.db
.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
.take_while(|(k, _)| &k[..] < &end_prefix[..])
.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &end_prefix[..]))
cheme marked this conversation as resolved.
Show resolved Hide resolved
cheme marked this conversation as resolved.
Show resolved Hide resolved
.peekable();

let batch = load_all_at_finalized_height(iter, batch_num, batch_finalized_hash);
let batch = load_all_at_finalized_height(iter, batch_num, batch_finalized_hash)?;

// Now that we've iterated over the entire batch at this finalized height,
// update the meta.
Expand All @@ -890,22 +891,22 @@ async fn process_block_finalized<Context>(
// loads all candidates at the finalized height and maps them to `true` if finalized
// and `false` if unfinalized.
fn load_all_at_finalized_height(
mut iter: std::iter::Peekable<impl Iterator<Item = (Box<[u8]>, Box<[u8]>)>>,
mut iter: std::iter::Peekable<impl Iterator<Item = io::Result<util::database::DBKeyValue>>>,
block_number: BlockNumber,
finalized_hash: Hash,
) -> impl IntoIterator<Item = (CandidateHash, bool)> {
) -> io::Result<impl IntoIterator<Item = (CandidateHash, bool)>> {
// maps candidate hashes to true if finalized, false otherwise.
let mut candidates = HashMap::new();

// Load all candidates that were included at this height.
loop {
match peek_num!(iter) {
match peek_num!(iter)? {
None => break, // end of iterator.
Some(n) if n != block_number => break, // end of batch.
_ => {},
}

let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed");
let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed")?;
let (_, block_hash, candidate_hash) =
decode_unfinalized_key(&k[..]).expect("`peek_num` checks validity of key; qed");

Expand All @@ -916,7 +917,7 @@ fn load_all_at_finalized_height(
}
}

candidates
Ok(candidates)
}

fn update_blocks_at_finalized_height(
Expand Down Expand Up @@ -1214,9 +1215,10 @@ fn prune_all(db: &Arc<dyn Database>, config: &Config, clock: &dyn Clock) -> Resu
let mut tx = DBTransaction::new();
let iter = db
.iter_with_prefix(config.col_meta, &range_start[..])
.take_while(|(k, _)| &k[..] < &range_end[..]);
.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &range_end[..]));
cheme marked this conversation as resolved.
Show resolved Hide resolved

for (k, _v) in iter {
for r in iter {
let (k, _v) = r?;
tx.delete(config.col_meta, &k[..]);

let (_, candidate_hash) = match decode_pruning_key(&k[..]) {
Expand Down
4 changes: 2 additions & 2 deletions node/core/chain-selection/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ polkadot-primitives = { path = "../../../primitives" }
polkadot-node-primitives = { path = "../../primitives" }
polkadot-node-subsystem = { path = "../../subsystem" }
polkadot-node-subsystem-util = { path = "../../subsystem-util" }
kvdb = "0.11.0"
kvdb = "0.12.0"
thiserror = "1.0.31"
parity-scale-codec = "3.1.5"

Expand All @@ -22,4 +22,4 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
parking_lot = "0.12.0"
assert_matches = "1"
kvdb-memorydb = "0.11.0"
kvdb-memorydb = "0.12.0"
26 changes: 17 additions & 9 deletions node/core/chain-selection/src/db_backend/v1.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,16 +235,21 @@ impl Backend for DbBackend {
self.inner.iter_with_prefix(self.config.col_data, &STAGNANT_AT_PREFIX[..]);

let val = stagnant_at_iter
.filter_map(|(k, v)| {
match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok()) {
(Some(at), Some(stagnant_at)) => Some((at, stagnant_at)),
_ => None,
}
.filter_map(|r| match r {
Ok((k, v)) =>
match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok())
{
(Some(at), Some(stagnant_at)) => Some(Ok((at, stagnant_at))),
_ => None,
},
Err(e) => Some(Err(e)),
})
.enumerate()
.take_while(|(idx, (at, _))| *at <= up_to.into() && *idx < max_elements)
.take_while(|(idx, r)| {
r.as_ref().map_or(true, |(at, _)| *at <= up_to.into() && *idx < max_elements)
cheme marked this conversation as resolved.
Show resolved Hide resolved
})
.map(|(_, v)| v)
.collect::<Vec<_>>();
.collect::<Result<Vec<_>, _>>()?;

Ok(val)
}
Expand All @@ -254,10 +259,13 @@ impl Backend for DbBackend {
self.inner.iter_with_prefix(self.config.col_data, &BLOCK_HEIGHT_PREFIX[..]);

let val = blocks_at_height_iter
.filter_map(|(k, _)| decode_block_height_key(&k[..]))
.filter_map(|r| match r {
Ok((k, _)) => decode_block_height_key(&k[..]).map(Result::Ok),
ordian marked this conversation as resolved.
Show resolved Hide resolved
Err(e) => Some(Err(e)),
})
.next();

Ok(val)
val.transpose().map_err(Error::from)
}

fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error> {
Expand Down
4 changes: 2 additions & 2 deletions node/core/dispute-coordinator/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ edition = "2021"
futures = "0.3.21"
gum = { package = "tracing-gum", path = "../../gum" }
parity-scale-codec = "3.1.5"
kvdb = "0.11.0"
kvdb = "0.12.0"
thiserror = "1.0.31"
lru = "0.8.0"
fatality = "0.0.6"
Expand All @@ -22,7 +22,7 @@ sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste


[dev-dependencies]
kvdb-memorydb = "0.11.0"
kvdb-memorydb = "0.12.0"
polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
Expand Down
2 changes: 1 addition & 1 deletion node/core/runtime-api/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ edition = "2021"
futures = "0.3.21"
gum = { package = "tracing-gum", path = "../../gum" }
memory-lru = "0.1.1"
parity-util-mem = { version = "0.11.0", default-features = false }
parity-util-mem = { version = "0.12.0", default-features = false }

sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }

Expand Down
2 changes: 1 addition & 1 deletion node/malus/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ polkadot-node-core-backing = { path = "../core/backing" }
polkadot-node-primitives = { path = "../primitives" }
polkadot-primitives = { path = "../../primitives" }
polkadot-node-core-pvf = { path = "../core/pvf" }
parity-util-mem = { version = "0.11.0", default-features = false, features = ["jemalloc-global"] }
parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }
color-eyre = { version = "0.6.1", default-features = false }
assert_matches = "1.5"
async-trait = "0.1.57"
Expand Down
2 changes: 1 addition & 1 deletion node/overseer/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ polkadot-primitives = { path = "../../primitives" }
orchestra = "0.0.2"
gum = { package = "tracing-gum", path = "../gum" }
lru = "0.8"
parity-util-mem = { version = "0.11.0", default-features = false }
parity-util-mem = { version = "0.12.0", default-features = false }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
async-trait = "0.1.57"

Expand Down
4 changes: 2 additions & 2 deletions node/service/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ gum = { package = "tracing-gum", path = "../gum/" }
serde = { version = "1.0.137", features = ["derive"] }
serde_json = "1.0.81"
thiserror = "1.0.31"
kvdb = "0.11.0"
kvdb-rocksdb = { version = "0.15.2", optional = true }
kvdb = "0.12.0"
kvdb-rocksdb = { version = "0.16.0", optional = true }
parity-db = { version = "0.3.16", optional = true }
async-trait = "0.1.57"
lru = "0.8"
Expand Down
2 changes: 1 addition & 1 deletion node/service/src/parachains_db/upgrade.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> {
.to_str()
.ok_or_else(|| super::other_io_error("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(super::columns::v0::NUM_COLUMNS);
let db = Database::open(&db_cfg, db_path)?;
let mut db = Database::open(&db_cfg, db_path)?;

db.add_column()?;
db.add_column()?;
Expand Down
6 changes: 3 additions & 3 deletions node/subsystem-util/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }

kvdb = "0.11.0"
parity-util-mem = { version = "0.11", default-features = false }
kvdb = "0.12.0"
parity-util-mem = { version = "0.12.0", default-features = false }
parity-db = { version = "0.3.13" }

[dev-dependencies]
Expand All @@ -44,5 +44,5 @@ log = "0.4.17"
polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }
lazy_static = "1.4.0"
polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" }
kvdb-shared-tests = "0.9.0"
kvdb-shared-tests = "0.10.0"
tempfile = "3.1.0"
Loading