This repository has been archived by the owner on Nov 15, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 2.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduce Pallet
paged-list (#14120)
* Prototype StoragePagedList Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add drain Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Remove stale docs Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add fuzzer tests Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Update Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review Co-authored-by: Koute <koute@users.noreply.github.com> Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * fmt Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Docs and clippy Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Sum docs Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Cleanup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Undo WIP Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add pallet-paged-list Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Move code to pallet Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Move fuzzer Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Cleanup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * fmt Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * docs Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Rename Appendix -> Appender Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Rename clear -> delete Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Feature gate testing stuff Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Docs review Co-authored-by: Koute <koute@users.noreply.github.com> Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Cleanup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * doc review Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review renames Co-authored-by: Koute <koute@users.noreply.github.com> Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add docs Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fix fuzzer Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Docs + examples Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Remove hasher Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Remove empty Event and Call Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Remove MaxPages Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fix docs Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Test eager page removal Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Cleanup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Update frame/paged-list/src/paged_list.rs Co-authored-by: Koute <koute@users.noreply.github.com> * Fix docs Co-authored-by: Koute <koute@users.noreply.github.com> Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Remove as_*_vec Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Update versions Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Rename ValuesPerPage -> ValuesPerNewPage Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Update lockfile Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fix mock Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> --------- Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> Co-authored-by: Koute <koute@users.noreply.github.com> Co-authored-by: parity-processbot <>
- Loading branch information
Showing
12 changed files
with
1,222 additions
and
4 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| [package] | ||
| name = "pallet-paged-list" | ||
| version = "0.1.0" | ||
| description = "FRAME pallet that provides a paged list data structure." | ||
| authors = ["Parity Technologies <admin@parity.io>"] | ||
| homepage = "https://substrate.io" | ||
| edition = "2021" | ||
| license = "Apache-2.0" | ||
| repository = "https://github.com/paritytech/substrate" | ||
|
|
||
| [package.metadata.docs.rs] | ||
| targets = ["x86_64-unknown-linux-gnu"] | ||
|
|
||
| [dependencies] | ||
| codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive"] } | ||
| docify = "0.1.13" | ||
| scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } | ||
|
|
||
| frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../benchmarking" } | ||
| frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } | ||
| frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } | ||
|
|
||
| sp-runtime = { version = "24.0.0", default-features = false, path = "../../primitives/runtime" } | ||
| sp-std = { version = "8.0.0", default-features = false, path = "../../primitives/std" } | ||
| sp-core = { version = "21.0.0", path = "../../primitives/core", default-features = false } | ||
| sp-io = { version = "23.0.0", path = "../../primitives/io", default-features = false } | ||
|
|
||
| [features] | ||
| default = ["std"] | ||
|
|
||
| std = ["codec/std", "frame-benchmarking?/std", "frame-support/std", "frame-system/std", "scale-info/std", "sp-core/std", "sp-io/std", "sp-runtime/std", "sp-std/std"] | ||
|
|
||
| runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", "sp-runtime/runtime-benchmarks"] | ||
|
|
||
| try-runtime = ["frame-support/try-runtime"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| [package] | ||
| name = "pallet-paged-list-fuzzer" | ||
| version = "0.1.0" | ||
| authors = ["Parity Technologies <admin@parity.io>"] | ||
| edition = "2021" | ||
| license = "Apache-2.0" | ||
| homepage = "https://substrate.io" | ||
| repository = "https://github.com/paritytech/substrate/" | ||
| description = "Fuzz storage types of pallet-paged-list" | ||
| publish = false | ||
|
|
||
| [[bin]] | ||
| name = "pallet-paged-list" | ||
| path = "src/paged_list.rs" | ||
|
|
||
| [dependencies] | ||
| arbitrary = "1.3.0" | ||
| honggfuzz = "0.5.49" | ||
|
|
||
| frame-support = { version = "4.0.0-dev", default-features = false, features = [ "std" ], path = "../../support" } | ||
| sp-io = { path = "../../../primitives/io", default-features = false, features = [ "std" ] } | ||
| pallet-paged-list = { path = "../", default-features = false, features = [ "std" ] } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // This file is part of Substrate. | ||
|
|
||
| // Copyright (C) Parity Technologies (UK) Ltd. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| //! # Running | ||
| //! Running this fuzzer can be done with `cargo hfuzz run pallet-paged-list`. `honggfuzz` CLI | ||
| //! options can be used by setting `HFUZZ_RUN_ARGS`, such as `-n 4` to use 4 threads. | ||
| //! | ||
| //! # Debugging a panic | ||
| //! Once a panic is found, it can be debugged with | ||
| //! `cargo hfuzz run-debug pallet-paged-list hfuzz_workspace/pallet-paged-list/*.fuzz`. | ||
| //! | ||
| //! # More information | ||
| //! More information about `honggfuzz` can be found | ||
| //! [here](https://docs.rs/honggfuzz/). | ||
| use arbitrary::Arbitrary; | ||
| use honggfuzz::fuzz; | ||
|
|
||
| use frame_support::{storage::StorageList, StorageNoopGuard}; | ||
| use pallet_paged_list::mock::{PagedList as List, *}; | ||
| use sp_io::TestExternalities; | ||
| type Meta = MetaOf<Test, ()>; | ||
|
|
||
| fn main() { | ||
| loop { | ||
| fuzz!(|data: (Vec<Op>, u8)| { | ||
| drain_append_work(data.0, data.1); | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| /// Appends and drains random number of elements in random order and checks storage invariants. | ||
| /// | ||
| /// It also changes the maximal number of elements per page dynamically, hence the `page_size`. | ||
| fn drain_append_work(ops: Vec<Op>, page_size: u8) { | ||
| if page_size == 0 { | ||
| return | ||
| } | ||
|
|
||
| TestExternalities::default().execute_with(|| { | ||
| ValuesPerNewPage::set(&page_size.into()); | ||
| let _g = StorageNoopGuard::default(); | ||
| let mut total: i64 = 0; | ||
|
|
||
| for op in ops.into_iter() { | ||
| total += op.exec(); | ||
|
|
||
| assert!(total >= 0); | ||
| assert_eq!(List::iter().count(), total as usize); | ||
|
|
||
| // We have the assumption that the queue removes the metadata when empty. | ||
| if total == 0 { | ||
| assert_eq!(List::drain().count(), 0); | ||
| assert_eq!(Meta::from_storage().unwrap_or_default(), Default::default()); | ||
| } | ||
| } | ||
|
|
||
| assert_eq!(List::drain().count(), total as usize); | ||
| // `StorageNoopGuard` checks that there is no storage leaked. | ||
| }); | ||
| } | ||
|
|
||
| enum Op { | ||
| Append(Vec<u32>), | ||
| Drain(u8), | ||
| } | ||
|
|
||
| impl Arbitrary<'_> for Op { | ||
| fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<Self> { | ||
| if u.arbitrary::<bool>()? { | ||
| Ok(Op::Append(Vec::<u32>::arbitrary(u)?)) | ||
| } else { | ||
| Ok(Op::Drain(u.arbitrary::<u8>()?)) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl Op { | ||
| pub fn exec(self) -> i64 { | ||
| match self { | ||
| Op::Append(v) => { | ||
| let l = v.len(); | ||
| List::append_many(v); | ||
| l as i64 | ||
| }, | ||
| Op::Drain(v) => -(List::drain().take(v as usize).count() as i64), | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| // This file is part of Substrate. | ||
|
|
||
| // Copyright (C) Parity Technologies (UK) Ltd. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| //! > Made with *Substrate*, for *DotSama*. | ||
| //! | ||
| //! [![github]](https://github.com/paritytech/substrate/frame/fast-unstake) - | ||
| //! [![polkadot]](https://polkadot.network) | ||
| //! | ||
| //! [polkadot]: https://img.shields.io/badge/polkadot-E6007A?style=for-the-badge&logo=polkadot&logoColor=white | ||
| //! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github | ||
| //! | ||
| //! # Paged List Pallet | ||
| //! | ||
| //! A thin wrapper pallet around a [`paged_list::StoragePagedList`]. It provides an API for a single | ||
| //! paginated list. It can be instantiated multiple times to provide multiple lists. | ||
| //! | ||
| //! ## Overview | ||
| //! | ||
| //! The pallet is quite unique since it does not expose any `Call`s, `Error`s or `Event`s. All | ||
| //! interaction goes through the implemented [`StorageList`][frame_support::storage::StorageList] | ||
| //! trait. | ||
| //! | ||
| //! A fuzzer for testing is provided in crate `pallet-paged-list-fuzzer`. | ||
| //! | ||
| //! ## Examples | ||
| //! | ||
| //! 1. **Appending** some data to the list can happen either by [`Pallet::append_one`]: | ||
| #![doc = docify::embed!("frame/paged-list/src/tests.rs", append_one_works)] | ||
| //! 2. or by [`Pallet::append_many`]. This should always be preferred to repeated calls to | ||
| //! [`Pallet::append_one`]: | ||
| #![doc = docify::embed!("frame/paged-list/src/tests.rs", append_many_works)] | ||
| //! 3. If you want to append many values (ie. in a loop), then best use the [`Pallet::appender`]: | ||
| #![doc = docify::embed!("frame/paged-list/src/tests.rs", appender_works)] | ||
| //! 4. **Iterating** over the list can be done with [`Pallet::iter`]. It uses the standard | ||
| //! `Iterator` trait: | ||
| #![doc = docify::embed!("frame/paged-list/src/tests.rs", iter_works)] | ||
| //! 5. **Draining** elements happens through the [`Pallet::drain`] iterator. Note that even | ||
| //! *peeking* a value will already remove it. | ||
| #![doc = docify::embed!("frame/paged-list/src/tests.rs", drain_works)] | ||
| //! | ||
| //! ## Pallet API | ||
| //! | ||
| //! None. Only things to consider is the [`Config`] traits. | ||
| //! | ||
| //! ## Low Level / Implementation Details | ||
| //! | ||
| //! Implementation details are documented in [`paged_list::StoragePagedList`]. | ||
| //! All storage entries are prefixed with a unique prefix that is generated by [`ListPrefix`]. | ||
| #![cfg_attr(not(feature = "std"), no_std)] | ||
|
|
||
| pub use pallet::*; | ||
|
|
||
| pub mod mock; | ||
| mod paged_list; | ||
| mod tests; | ||
|
|
||
| use codec::FullCodec; | ||
| use frame_support::{ | ||
| pallet_prelude::StorageList, | ||
| traits::{PalletInfoAccess, StorageInstance}, | ||
| }; | ||
| pub use paged_list::StoragePagedList; | ||
|
|
||
| #[frame_support::pallet] | ||
| pub mod pallet { | ||
| use super::*; | ||
| use frame_support::pallet_prelude::*; | ||
|
|
||
| #[pallet::pallet] | ||
| pub struct Pallet<T, I = ()>(_); | ||
|
|
||
| #[pallet::config] | ||
| pub trait Config<I: 'static = ()>: frame_system::Config { | ||
| /// The value type that can be stored in the list. | ||
| type Value: FullCodec; | ||
|
|
||
| /// The number of values that can be put into newly created pages. | ||
| /// | ||
| /// Note that this does not retroactively affect already created pages. This value can be | ||
| /// changed at any time without requiring a runtime migration. | ||
| #[pallet::constant] | ||
| type ValuesPerNewPage: Get<u32>; | ||
| } | ||
|
|
||
| /// A storage paged list akin to what the FRAME macros would generate. | ||
| // Note that FRAME does natively support paged lists in storage. | ||
| pub type List<T, I> = StoragePagedList< | ||
| ListPrefix<T, I>, | ||
| <T as Config<I>>::Value, | ||
| <T as Config<I>>::ValuesPerNewPage, | ||
| >; | ||
| } | ||
|
|
||
| // This exposes the list functionality to other pallets. | ||
| impl<T: Config<I>, I: 'static> StorageList<T::Value> for Pallet<T, I> { | ||
| type Iterator = <List<T, I> as StorageList<T::Value>>::Iterator; | ||
| type Appender = <List<T, I> as StorageList<T::Value>>::Appender; | ||
|
|
||
| fn iter() -> Self::Iterator { | ||
| List::<T, I>::iter() | ||
| } | ||
|
|
||
| fn drain() -> Self::Iterator { | ||
| List::<T, I>::drain() | ||
| } | ||
|
|
||
| fn appender() -> Self::Appender { | ||
| List::<T, I>::appender() | ||
| } | ||
| } | ||
|
|
||
| /// Generates a unique storage prefix for each instance of the pallet. | ||
| pub struct ListPrefix<T, I>(core::marker::PhantomData<(T, I)>); | ||
|
|
||
| impl<T: Config<I>, I: 'static> StorageInstance for ListPrefix<T, I> { | ||
| fn pallet_prefix() -> &'static str { | ||
| crate::Pallet::<T, I>::name() | ||
| } | ||
|
|
||
| const STORAGE_PREFIX: &'static str = "paged_list"; | ||
| } |
Oops, something went wrong.