Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Erasure encoding availability #345

Merged
merged 48 commits into from
Dec 3, 2019
Merged

Erasure encoding availability #345

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
f8a71c7
Erasure encoding availability initial commit
montekki Jul 29, 2019
42a97c7
Apply suggestions from code review
montekki Jul 31, 2019
66f8929
Merge remote-tracking branch 'upstream/master' into fs-availability-e…
montekki Aug 7, 2019
13339d3
Fix build after a merge
montekki Aug 7, 2019
ba601bd
Gossip erasure chunk messages under their own topic
montekki Aug 8, 2019
3c9f260
erasure_chunks should use the appropriate topic
montekki Aug 8, 2019
335df20
Merge remote-tracking branch 'upstream/master' into fs-availability-e…
montekki Aug 8, 2019
6ceb392
Updates Cargo.lock
montekki Aug 8, 2019
d8ee374
Merge branch 'master' into fs-availability-encoding-and-distribution
montekki Sep 25, 2019
71dbe6e
Fixes after merge
montekki Sep 25, 2019
0f70e2a
Removes a couple of leftover pieces of code
montekki Sep 27, 2019
05f3eb2
Merge remote-tracking branch 'upstream/master' into fs-availability-e…
montekki Sep 27, 2019
eb01f98
Fixes simple stuff from review
montekki Oct 1, 2019
0040b0b
Updates erasure and storage for more flexible logic
montekki Oct 3, 2019
f6ba7a6
Changes validation and candidate receipt production.
montekki Oct 4, 2019
28de50e
Adds add_erasure_chunks method
montekki Oct 11, 2019
0431cac
Fixes most of the nits
montekki Oct 15, 2019
3a348bb
Better validate_collation and validate_receipt functions
montekki Oct 16, 2019
5e19b3c
Fixes the tests
montekki Oct 16, 2019
0ddda19
Apply suggestions from code review
montekki Oct 25, 2019
13571b9
Removes unwrap() calls
montekki Oct 25, 2019
e6a57bf
Removes ErasureChunks primitive
montekki Oct 25, 2019
eee87bf
Removes redundant fields from ErasureChunk struct
montekki Oct 28, 2019
e6ca479
AvailabilityStore should store CandidateReceipt
montekki Oct 29, 2019
a463151
Changes the way chunk messages are imported and validated.
montekki Oct 30, 2019
701e36b
Correctly compute topics for erasure messages
montekki Oct 31, 2019
065c3bc
Removes an unused parameter
montekki Oct 31, 2019
3819843
Refactors availability db querying into a helper
montekki Nov 1, 2019
3e101ec
Adds the apis described in the writeup
montekki Nov 5, 2019
4c3faee
Merge remote-tracking branch 'upstream/master' into fs-availability-e…
montekki Nov 6, 2019
1d4f5f8
Adds a runtime api to extract erasure roots form raw extrinsics.
montekki Nov 8, 2019
5a665a0
Adds a barebone BlockImport impl for avalability store
montekki Nov 12, 2019
b2816d0
Adds the implementation of the availability worker
montekki Nov 22, 2019
06fd8bf
Merge branch 'master' into fs-availability-encoding-and-distribution
montekki Nov 26, 2019
6d09a53
Fix build after the merge with master.
montekki Nov 27, 2019
9c0e705
Make availability store API async
montekki Nov 27, 2019
fe8bdab
Bring back the default wasmtime feature
montekki Nov 27, 2019
9f7723c
Lines width
montekki Nov 27, 2019
acb6692
Bump runtime version
montekki Nov 27, 2019
1a406d9
Formatting and dead code elimination
montekki Nov 28, 2019
7465900
some style nits (#1)
rphmeier Dec 2, 2019
202bda2
More nits and api cleanup
montekki Dec 2, 2019
b4213d2
Merge branch 'master' into fs-availability-encoding-and-distribution
montekki Dec 3, 2019
381a32e
Disable wasm CI for availability-store
montekki Dec 3, 2019
b70fd19
Another nit
montekki Dec 3, 2019
257eea7
Formatting
montekki Dec 3, 2019
c08cbc2
Merge branch 'master' into fs-availability-encoding-and-distribution
montekki Dec 3, 2019
7e73a3e
Merge branch 'master' into fs-availability-encoding-and-distribution
gavofyork Dec 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 61 additions & 44 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 14 additions & 10 deletions availability-store/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,17 @@ futures = { package = "futures", version = "0.3.1", features = ["compat"] }
tokio = "0.1.7"
exit-future = "0.1"
codec = { package = "parity-scale-codec", version = "1.1.0", default-features = false, features = ["derive"] }
substrate-network = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
consensus_common = { package = "substrate-consensus-common", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
substrate-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
substrate-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
sr-primitives = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
sr-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
keystore = { package = "substrate-keystore", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
sc-network = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
consensus_common = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
client = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
sc-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
sp-runtime = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
keystore = { package = "sc-keystore", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
kvdb = "0.1.1"
kvdb-memorydb = "0.1.2"

[target.'cfg(not(target_os = "unknown"))'.dependencies]
kvdb-rocksdb = "0.2"
33 changes: 19 additions & 14 deletions availability-store/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,15 @@ use polkadot_primitives::{
Hash, Block,
parachain::{
Id as ParaId, BlockData, CandidateReceipt, Message, AvailableMessages, ErasureChunk,
ExtrinsicsQuerying, ParachainHost,
ParachainHost,
},
};
use sr_primitives::traits::{BlakeTwo256, Hash as HashT, ProvideRuntimeApi};
use substrate_client::{
error::Result as ClientResult,
use sp_runtime::traits::{BlakeTwo256, Hash as HashT, ProvideRuntimeApi};
use sp_blockchain::{Result as ClientResult};
use client::{
BlockchainEvents, BlockBody,
};
use sr_api::ApiExt;
use sp_api::ApiExt;

use log::warn;

Expand All @@ -52,7 +52,7 @@ mod store;
pub use worker::AvailabilityBlockImport;

use worker::{
Worker, WorkerHandle, Chunks, ParachainBlocks, WorkerMsg,
Worker, WorkerHandle, Chunks, ParachainBlocks, WorkerMsg, MakeAvailable,
};

use store::{Store as InnerStore};
Expand Down Expand Up @@ -123,6 +123,7 @@ pub trait ProvideGossipMessages {
}

/// Some data to keep available about a parachain block candidate.
#[derive(Debug)]
pub struct Data {
/// The relay chain parent hash this should be localized to.
pub relay_parent: Hash,
Expand Down Expand Up @@ -158,7 +159,7 @@ impl Store {
where PGM: ProvideGossipMessages + Send + Sync + Clone + 'static
{
let inner = InnerStore::new(config)?;
let worker = Arc::new(Worker::new(inner.clone(), gossip).start());
let worker = Arc::new(Worker::start(inner.clone(), gossip));
let to_worker = worker.to_worker().clone();

Ok(Self {
Expand All @@ -176,7 +177,7 @@ impl Store {
where PGM: ProvideGossipMessages + Send + Sync + Clone + 'static
{
let inner = InnerStore::new_in_memory();
let worker = Arc::new(Worker::new(inner.clone(), gossip).start());
let worker = Arc::new(Worker::start(inner.clone(), gossip));
let to_worker = worker.to_worker().clone();

Self {
Expand All @@ -202,8 +203,8 @@ impl Store {
) -> ClientResult<(AvailabilityBlockImport<I, P>)>
where
P: ProvideRuntimeApi + BlockchainEvents<Block> + BlockBody<Block> + Send + Sync + 'static,
P::Api: ExtrinsicsQuerying<Block> + ParachainHost<Block>,
P::Api: ApiExt<Block, Error=substrate_client::error::Error>,
P::Api: ParachainHost<Block>,
P::Api: ApiExt<Block, Error=sp_blockchain::Error>,
{
let to_worker = self.to_worker.clone();

Expand Down Expand Up @@ -235,9 +236,8 @@ impl Store {
/// asynchrounously wait for the result.
pub async fn make_available(&self, data: Data) -> io::Result<()> {
let (s, r) = oneshot::channel();
let msg = WorkerMsg::ParachainBlocks(ParachainBlocks {
relay_parent: data.relay_parent,
blocks: vec![],
let msg = WorkerMsg::MakeAvailable(MakeAvailable {
data,
result: s,
});

Expand All @@ -263,6 +263,11 @@ impl Store {
}

/// Make a validator's index and a number of validators at a relay parent available.
///
/// This information is needed before the `add_candidates_in_relay_block` is called
/// since that call forms the awaited frontier of chunks.
/// In the current implementation this function is called in the `get_or_instantiate` at
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I won't block the PR on this, but in general I'd prefer to avoid these kinds of long-distance expectations of how the code should be used. It's an assumption that can easily change without noticing, unlike things like "X is called before Y".

/// the start of the parachain agreement process on top of some parent hash.
pub fn add_validator_index_and_n_validators(
&self,
relay_parent: &Hash,
Expand Down Expand Up @@ -346,7 +351,7 @@ impl Store {
pub async fn add_candidate(
&self,
relay_parent: Hash,
receipt: CandidateReceipt
receipt: CandidateReceipt,
) -> io::Result<()> {
let (s, r) = oneshot::channel();

Expand Down
Loading