Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

[DNM] Feature/merge no gossip of approvals with assignments v2 #7522

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
110 commits
Select commit Hold shift + click to select a range
4adfc1f
Add compact vrf modulo assignment
sandreim Feb 7, 2023
71bf07b
compute all assignments from a single vrf output
sandreim Feb 7, 2023
b1aabc9
impl compute/check of `RelayVRFModuloCompact`
sandreim Feb 7, 2023
8b881ee
fmt and missed file
sandreim Feb 7, 2023
81e62ec
check_and_import_assignment: multiple candidates
sandreim Feb 8, 2023
82c2b5f
integration step 1: subsystem message changes
sandreim Feb 10, 2023
d9d5681
Approval distribution WIP
sandreim Feb 10, 2023
596a347
approval-dist: fixed compilation errors
sandreim Feb 10, 2023
c87502b
test fixups, WIP
sandreim Feb 10, 2023
cc9f63a
itertools
sandreim Feb 13, 2023
eb86e56
Allow claimed cores to be a subset of vrf output assignments
sandreim Feb 13, 2023
4d421d2
Revert approval distribution changes
sandreim Feb 13, 2023
7b6db76
WIP
sandreim Feb 15, 2023
46d76d4
Fix compile errors
sandreim Feb 16, 2023
b18295e
add todo
sandreim Feb 16, 2023
a41254c
Keep track of peer protocol version
sandreim Feb 17, 2023
3bb8823
network protocol bump and addition
sandreim Feb 20, 2023
b34788e
Approval distribution handing of v1 vs v2
sandreim Feb 20, 2023
48109ec
Add VStaging network protocol
sandreim Feb 23, 2023
b672d86
comment aggression metrics
sandreim Feb 23, 2023
2a300de
finish impl v2/v1 sending
sandreim Feb 24, 2023
7858ad5
fix view change handling typo bug
sandreim Feb 24, 2023
0cf73fd
Rename leftover V2 to vstaging
sandreim Feb 24, 2023
412282a
Disable assignments V2
sandreim Feb 24, 2023
af3b64b
update todo, docs
sandreim Feb 24, 2023
6612b3c
fmt
sandreim Feb 24, 2023
f8ccec9
Temporarly disable CI cargo test to get a build for burn-in
sandreim Feb 24, 2023
ef42e0c
debug -> trace
sandreim Feb 24, 2023
0c31d61
enable v2 assignments
sandreim Feb 24, 2023
e3598b1
Remove unimplemented!
sandreim Mar 10, 2023
24db36c
Metric updates
sandreim Mar 10, 2023
741e91d
approval-voting: simplify v2 assignments (#6802)
sandreim Mar 31, 2023
7bf83b1
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Mar 31, 2023
01b6ec0
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Apr 3, 2023
1a38026
fix merge damage
sandreim Apr 8, 2023
b0402a3
Include core bitfield in compact assignments
sandreim Apr 18, 2023
2ad5234
Fix existing approval voting tests
sandreim Apr 20, 2023
6a8f17a
Add bitfield certificate extra check
sandreim Apr 20, 2023
a8f952c
Approval dist test compilation fixes
sandreim Apr 20, 2023
31c0548
Uncomment aggression
sandreim Apr 20, 2023
ef0bca5
approval-dist: some tests fail, but it compiles
sandreim Apr 20, 2023
1a7d33b
Re-enable tests
sandreim Apr 21, 2023
d2fd321
Fix tests
sandreim Apr 21, 2023
2288086
Fix build
sandreim Apr 21, 2023
5755b17
clippy fixes
sandreim Apr 23, 2023
fc43f5b
fix more tests :(
sandreim Apr 24, 2023
33359cb
more network protocol test fixes
sandreim Apr 25, 2023
c8af81a
Fix network bridge tests using code from async branch
sandreim Apr 25, 2023
d0d1209
Merge holy master branch :church:
sandreim Apr 25, 2023
510c28a
heal merge damage :hospital:
sandreim Apr 27, 2023
32ab33a
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Apr 27, 2023
9f6f0e6
Bump test timeout to make CI happy when slow
sandreim May 2, 2023
274ea46
approval primitives v1/v2 refactor
sandreim May 11, 2023
9f66ed1
remove leftover sleep in CI test pipeline
sandreim May 11, 2023
6274c37
Fix tests and enable v2 assignments in tests
sandreim May 11, 2023
28dd203
add/modify approval distribution tests for v2
sandreim May 17, 2023
5c5b86b
Add another approval distribution test
sandreim May 17, 2023
7f51489
add zombienet test
sandreim May 17, 2023
f4a983b
fix tests build
sandreim May 17, 2023
180273a
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim May 18, 2023
2a65a98
fix build
sandreim May 18, 2023
88b4349
enable v2 assignemnts for testing
sandreim May 18, 2023
424ea57
Add back removed comment
sandreim May 18, 2023
229d1c1
review feedback 1/2
sandreim May 22, 2023
9152a9c
Full approval db migration to v2
sandreim May 24, 2023
0f9e1ec
fix comment
sandreim May 24, 2023
fe5b548
Fix AcceptedDuplicate, test and import log
sandreim May 25, 2023
a4fae3f
clippy
sandreim May 25, 2023
2be89ff
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim May 25, 2023
14c3643
fmt
sandreim May 25, 2023
ad3875d
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim May 25, 2023
0bc5aa8
Fix `get_approval_signatures`
sandreim May 26, 2023
574c92b
Add ApprovalEntry::get_approval
sandreim May 26, 2023
4c04699
disable v2 assignments
sandreim May 26, 2023
0be0201
review feedback and print remove
sandreim May 26, 2023
496f0c5
Fix logging
sandreim May 31, 2023
ba6078c
fix
sandreim May 31, 2023
bfce579
Get rid of old "legacy" protocols, not used anymore
sandreim Jun 8, 2023
d012e7b
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Jun 8, 2023
cd911e6
merge fixes
sandreim Jun 8, 2023
959e529
Refactor and always send on main protocol version
sandreim Jun 9, 2023
a8e3f28
Fix db upgrade after merge and fmt
sandreim Jun 9, 2023
78de04e
Fix zombienet merge damage
sandreim Jun 9, 2023
112b0be
Implement seq updates 0->4 now possible for parachainsDB
sandreim Jun 12, 2023
4b1743c
add/fix db upgrade tests
sandreim Jun 13, 2023
36f6b39
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Jun 14, 2023
4e0fac6
update colander image version
pepoviola Jun 14, 2023
7f97355
update image to polkadot-parachain
pepoviola Jun 14, 2023
d356081
review
sandreim Jul 4, 2023
a179646
fmt
sandreim Jul 4, 2023
de6ea33
Remove superfuous `assignment_bitfield` from `OurAssignment`
sandreim Jul 4, 2023
bbcf35d
node: approval-distribution/voting: lazy approvals signature checks
alexggh Jul 10, 2023
1245d18
fix warning in test-linux-stable
alexggh Jul 11, 2023
e28153f
Hack: Disable gossiping at a certain block number
alexggh Jul 13, 2023
7f9a314
HACK: Add versi block to start test
alexggh Jul 13, 2023
db7a840
Hack: update versi enablement block number
alexggh Jul 14, 2023
f0a2472
Fix
alexggh Jul 14, 2023
9ebee8c
Test block time
alexggh Jul 14, 2023
69b7e8e
Fixed on delayed approval processed
alexggh Jul 14, 2023
f24fc2f
Add more metrics regarding gossipped approvals
alexggh Jul 14, 2023
1c312ed
Add metric as well
alexggh Jul 14, 2023
953bc16
Add more logs to understand why gossiping is still happening
alexggh Jul 14, 2023
5f6edef
add more metrics: to understand versi behaviour
alexggh Jul 18, 2023
b09a619
fix: gossiped approvals received metric
alexggh Jul 18, 2023
2af3525
Add more logs to debug metrics
alexggh Jul 18, 2023
2af7d29
Merge remote-tracking branch 'origin/sandreim/vrf_modulo_comapct_assi…
alexggh Jul 19, 2023
5a4cec9
Merge branch 'assignment_v2' into approval_voting_lazy_checking_assig…
alexggh Jul 19, 2023
6ae7d1c
Adapt no gossip approval for assignments v2
alexggh Jul 19, 2023
e2358d0
Fix tests warnings
alexggh Jul 19, 2023
47212a8
Enable assignments v2
alexggh Jul 20, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions node/core/approval-voting/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ schnorrkel = "0.9.1"
kvdb = "0.13.0"
derive_more = "0.99.17"
thiserror = "1.0.31"
itertools = "0.10.5"

polkadot-node-subsystem = { path = "../../subsystem" }
polkadot-node-subsystem-util = { path = "../../subsystem-util" }
Expand All @@ -29,6 +30,9 @@ sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "mast
sp-consensus-slots = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, features = ["full_crypto"] }
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
# Needed for migration test helpers
test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
rand_core = "0.5.1"

[dev-dependencies]
async-trait = "0.1.57"
Expand Down
60 changes: 30 additions & 30 deletions node/core/approval-voting/src/approval_checking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
//! Utilities for checking whether a candidate has been approved under a given block.

use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice};
use polkadot_node_primitives::approval::DelayTranche;
use polkadot_node_primitives::approval::v1::DelayTranche;
use polkadot_primitives::ValidatorIndex;

use crate::{
Expand Down Expand Up @@ -472,9 +472,9 @@ mod tests {
}
.into();

let approval_entry = approval_db::v1::ApprovalEntry {
let approval_entry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: BitVec::default(),
assigned_validators: BitVec::default(),
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -509,22 +509,22 @@ mod tests {
candidate.mark_approval(ValidatorIndex(i));
}

let approval_entry = approval_db::v1::ApprovalEntry {
let approval_entry = approval_db::v2::ApprovalEntry {
tranches: vec![
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 0,
assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(),
},
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 1,
assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(),
},
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 2,
assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(),
},
],
assignments: bitvec![u8, BitOrderLsb0; 1; 10],
assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -581,22 +581,22 @@ mod tests {
candidate.mark_approval(ValidatorIndex(i));
}

let approval_entry = approval_db::v1::ApprovalEntry {
let approval_entry = approval_db::v2::ApprovalEntry {
tranches: vec![
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 0,
assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(),
},
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 1,
assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(),
},
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 2,
assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(),
},
],
assignments: bitvec![u8, BitOrderLsb0; 1; 10],
assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -647,9 +647,9 @@ mod tests {
let no_show_duration = 10;
let needed_approvals = 4;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; 5],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -691,9 +691,9 @@ mod tests {
let no_show_duration = 10;
let needed_approvals = 4;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; 10],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -731,9 +731,9 @@ mod tests {
let no_show_duration = 10;
let needed_approvals = 4;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; 10],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -776,9 +776,9 @@ mod tests {
let needed_approvals = 4;
let n_validators = 8;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -843,9 +843,9 @@ mod tests {
let needed_approvals = 4;
let n_validators = 8;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -934,9 +934,9 @@ mod tests {
let needed_approvals = 4;
let n_validators = 8;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -1041,15 +1041,15 @@ mod tests {
candidate.mark_approval(ValidatorIndex(i));
}

let approval_entry = approval_db::v1::ApprovalEntry {
let approval_entry = approval_db::v2::ApprovalEntry {
tranches: vec![
// Assignments with invalid validator indexes.
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 1,
assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(),
},
],
assignments: bitvec![u8, BitOrderLsb0; 1; 3],
assigned_validators: bitvec![u8, BitOrderLsb0; 1; 3],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -1094,12 +1094,12 @@ mod tests {
];

for test_tranche in test_tranches {
let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
backing_group: GroupIndex(0),
our_assignment: None,
our_approval_sig: None,
assignments: bitvec![u8, BitOrderLsb0; 0; 3],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; 3],
approved: false,
}
.into();
Expand Down
1 change: 1 addition & 0 deletions node/core/approval-voting/src/approval_db/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@
//! time being we share the same DB with the rest of Substrate.

pub mod v1;
pub mod v2;
Loading