From bcd649ffca9efc93f8b4ac1506ec8117b71e1aac Mon Sep 17 00:00:00 2001 From: Keith Yeung Date: Fri, 14 May 2021 02:44:29 -0700 Subject: [PATCH] Implement StorageNMap (#8635) * Implement StorageNMap * Change copyright date to 2021 * Rewrite keys to use impl_for_tuples instead of recursion * Implement prefix iteration on StorageNMap * Implement EncodeLike for key arguments * Rename KeyGenerator::Arg to KeyGenerator::KArg * Support StorageNMap in decl_storage and #[pallet::storage] macros * Use StorageNMap in assets pallet * Support migrate_keys in StorageNMap * Reduce line characters on select files * Refactor crate imports in decl_storage macros * Some more line char reductions and doc comment update * Update UI test expectations * Revert whitespace changes to untouched files * Generate Key struct instead of a 1-tuple when only 1 pair of key and hasher is provided * Revert formatting changes to unrelated files * Introduce KeyGeneratorInner * Add tests for StorageNMap in FRAMEv2 pallet macro * Small fixes to unit tests for StorageNMap * Bump runtime metadata version * Remove unused import * Update tests to use runtime metadata v13 * Introduce and use EncodeLikeTuple as a trait bound for KArg * Add some rustdocs * Revert usage of StorageNMap in assets pallet * Make use of ext::PunctuatedTrailing * Add rustdoc for final_hash * Fix StorageNMap proc macro expansions for single key cases * Create associated const in KeyGenerator for hasher metadata * Refactor code according to comments from Basti * Add module docs for generator/nmap.rs * Re-export storage::Key as NMapKey in pallet prelude * Seal the EncodeLikeTuple trait * Extract sealing code out of key.rs Co-authored-by: Shawn Tabrizi --- frame/metadata/src/lib.rs | 17 +- .../procedural/src/pallet/expand/storage.rs | 47 + .../procedural/src/pallet/parse/storage.rs | 79 +- .../src/storage/genesis_config/builder_def.rs | 15 + .../genesis_config/genesis_config_def.rs | 4 + .../src/storage/genesis_config/mod.rs | 6 +- .../support/procedural/src/storage/getters.rs | 18 +- .../procedural/src/storage/instance_trait.rs | 3 +- .../procedural/src/storage/metadata.rs | 24 +- frame/support/procedural/src/storage/mod.rs | 87 +- frame/support/procedural/src/storage/parse.rs | 34 +- .../src/storage/print_pallet_upgrade.rs | 9 + .../procedural/src/storage/storage_struct.rs | 40 +- frame/support/src/lib.rs | 9 +- frame/support/src/storage/generator/mod.rs | 2 + frame/support/src/storage/generator/nmap.rs | 541 ++++++++++ frame/support/src/storage/mod.rs | 186 +++- frame/support/src/storage/types/key.rs | 957 +++++++++++++++++ frame/support/src/storage/types/mod.rs | 19 +- frame/support/src/storage/types/nmap.rs | 995 ++++++++++++++++++ frame/support/test/tests/construct_runtime.rs | 2 +- frame/support/test/tests/pallet.rs | 87 +- .../test/tests/pallet_compatibility.rs | 2 +- .../tests/pallet_compatibility_instance.rs | 2 +- frame/support/test/tests/pallet_instance.rs | 73 +- .../pallet_ui/storage_not_storage_type.stderr | 2 +- 26 files changed, 3210 insertions(+), 50 deletions(-) create mode 100755 frame/support/src/storage/generator/nmap.rs create mode 100755 frame/support/src/storage/types/key.rs create mode 100755 frame/support/src/storage/types/nmap.rs diff --git a/frame/metadata/src/lib.rs b/frame/metadata/src/lib.rs index a63da82ca00db..ba232a88f11c4 100644 --- a/frame/metadata/src/lib.rs +++ b/frame/metadata/src/lib.rs @@ -300,6 +300,11 @@ pub enum StorageEntryType { value: DecodeDifferentStr, key2_hasher: StorageHasher, }, + NMap { + keys: DecodeDifferentArray<&'static str, StringBuf>, + hashers: DecodeDifferentArray, + value: DecodeDifferentStr, + }, } /// A storage entry modifier. @@ -364,8 +369,10 @@ pub enum RuntimeMetadata { V10(RuntimeMetadataDeprecated), /// Version 11 for runtime metadata. No longer used. V11(RuntimeMetadataDeprecated), - /// Version 12 for runtime metadata. - V12(RuntimeMetadataV12), + /// Version 12 for runtime metadata. No longer used. + V12(RuntimeMetadataDeprecated), + /// Version 13 for runtime metadata. + V13(RuntimeMetadataV13), } /// Enum that should fail. @@ -389,7 +396,7 @@ impl Decode for RuntimeMetadataDeprecated { /// The metadata of a runtime. #[derive(Eq, Encode, PartialEq, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Decode, Serialize))] -pub struct RuntimeMetadataV12 { +pub struct RuntimeMetadataV13 { /// Metadata of all the modules. pub modules: DecodeDifferentArray, /// Metadata of the extrinsic. @@ -397,7 +404,7 @@ pub struct RuntimeMetadataV12 { } /// The latest version of the metadata. -pub type RuntimeMetadataLastVersion = RuntimeMetadataV12; +pub type RuntimeMetadataLastVersion = RuntimeMetadataV13; /// All metadata about an runtime module. #[derive(Clone, PartialEq, Eq, Encode, RuntimeDebug)] @@ -425,6 +432,6 @@ impl Into for RuntimeMetadataPrefixed { impl Into for RuntimeMetadataLastVersion { fn into(self) -> RuntimeMetadataPrefixed { - RuntimeMetadataPrefixed(META_RESERVED, RuntimeMetadata::V12(self)) + RuntimeMetadataPrefixed(META_RESERVED, RuntimeMetadata::V13(self)) } } diff --git a/frame/support/procedural/src/pallet/expand/storage.rs b/frame/support/procedural/src/pallet/expand/storage.rs index 86fb84b339b24..c78e93e1d6391 100644 --- a/frame/support/procedural/src/pallet/expand/storage.rs +++ b/frame/support/procedural/src/pallet/expand/storage.rs @@ -90,6 +90,9 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { Metadata::DoubleMap { .. } => quote::quote_spanned!(storage.attr_span => #frame_support::storage::types::StorageDoubleMapMetadata ), + Metadata::NMap { .. } => quote::quote_spanned!(storage.attr_span => + #frame_support::storage::types::StorageNMapMetadata + ), }; let ty = match &storage.metadata { @@ -126,6 +129,24 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { value: #frame_support::metadata::DecodeDifferent::Encode(#value), } ) + }, + Metadata::NMap { keys, value, .. } => { + let keys = keys + .iter() + .map(|key| clean_type_string("e::quote!(#key).to_string())) + .collect::>(); + let value = clean_type_string("e::quote!(#value).to_string()); + quote::quote_spanned!(storage.attr_span => + #frame_support::metadata::StorageEntryType::NMap { + keys: #frame_support::metadata::DecodeDifferent::Encode(&[ + #( #keys, )* + ]), + hashers: #frame_support::metadata::DecodeDifferent::Encode( + <#full_ident as #metadata_trait>::HASHERS, + ), + value: #frame_support::metadata::DecodeDifferent::Encode(#value), + } + ) } }; @@ -227,6 +248,32 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { } ) }, + Metadata::NMap { keygen, value, .. } => { + let query = match storage.query_kind.as_ref().expect("Checked by def") { + QueryKind::OptionQuery => quote::quote_spanned!(storage.attr_span => + Option<#value> + ), + QueryKind::ValueQuery => quote::quote!(#value), + }; + quote::quote_spanned!(storage.attr_span => + #(#cfg_attrs)* + impl<#type_impl_gen> #pallet_ident<#type_use_gen> #completed_where_clause { + #( #docs )* + pub fn #getter(key: KArg) -> #query + where + KArg: #frame_support::storage::types::EncodeLikeTuple< + <#keygen as #frame_support::storage::types::KeyGenerator>::KArg + > + + #frame_support::storage::types::TupleToEncodedIter, + { + < + #full_ident as + #frame_support::storage::StorageNMap<#keygen, #value> + >::get(key) + } + } + ) + } } } else { Default::default() diff --git a/frame/support/procedural/src/pallet/parse/storage.rs b/frame/support/procedural/src/pallet/parse/storage.rs index 41ef337b76615..80c2e10a25206 100644 --- a/frame/support/procedural/src/pallet/parse/storage.rs +++ b/frame/support/procedural/src/pallet/parse/storage.rs @@ -50,7 +50,7 @@ impl syn::parse::Parse for PalletStorageAttr { } /// The value and key types used by storages. Needed to expand metadata. -pub enum Metadata{ +pub enum Metadata { Value { value: syn::GenericArgument }, Map { value: syn::GenericArgument, key: syn::GenericArgument }, DoubleMap { @@ -58,6 +58,11 @@ pub enum Metadata{ key1: syn::GenericArgument, key2: syn::GenericArgument }, + NMap { + keys: Vec, + keygen: syn::GenericArgument, + value: syn::GenericArgument, + }, } pub enum QueryKind { @@ -115,6 +120,64 @@ fn retrieve_arg( } } +/// Parse the 2nd type argument to `StorageNMap` and return its keys. +fn collect_keys(keygen: &syn::GenericArgument) -> syn::Result> { + if let syn::GenericArgument::Type(syn::Type::Tuple(tup)) = keygen { + tup + .elems + .iter() + .map(extract_key) + .collect::>>() + } else if let syn::GenericArgument::Type(ty) = keygen { + Ok(vec![extract_key(ty)?]) + } else { + let msg = format!("Invalid pallet::storage, expected tuple of Key structs or Key struct"); + Err(syn::Error::new(keygen.span(), msg)) + } +} + +/// In `Key`, extract K and return it. +fn extract_key(ty: &syn::Type) -> syn::Result { + let typ = if let syn::Type::Path(typ) = ty { + typ + } else { + let msg = "Invalid pallet::storage, expected type path"; + return Err(syn::Error::new(ty.span(), msg)); + }; + + let key_struct = typ.path.segments.last().ok_or_else(|| { + let msg = "Invalid pallet::storage, expected type path with at least one segment"; + syn::Error::new(typ.path.span(), msg) + })?; + if key_struct.ident != "Key" && key_struct.ident != "NMapKey" { + let msg = "Invalid pallet::storage, expected Key or NMapKey struct"; + return Err(syn::Error::new(key_struct.ident.span(), msg)); + } + + let ty_params = if let syn::PathArguments::AngleBracketed(args) = &key_struct.arguments { + args + } else { + let msg = "Invalid pallet::storage, expected angle bracketed arguments"; + return Err(syn::Error::new(key_struct.arguments.span(), msg)); + }; + + if ty_params.args.len() != 2 { + let msg = format!("Invalid pallet::storage, unexpected number of generic arguments \ + for Key struct, expected 2 args, found {}", ty_params.args.len()); + return Err(syn::Error::new(ty_params.span(), msg)); + } + + let key = match &ty_params.args[1] { + syn::GenericArgument::Type(key_ty) => key_ty.clone(), + _ => { + let msg = "Invalid pallet::storage, expected type"; + return Err(syn::Error::new(ty_params.args[1].span(), msg)); + } + }; + + Ok(key) +} + impl StorageDef { pub fn try_from( attr_span: proc_macro2::Span, @@ -177,11 +240,21 @@ impl StorageDef { value: retrieve_arg(&typ.path.segments[0], 5)?, } } + "StorageNMap" => { + query_kind = retrieve_arg(&typ.path.segments[0], 3); + let keygen = retrieve_arg(&typ.path.segments[0], 1)?; + let keys = collect_keys(&keygen)?; + Metadata::NMap { + keys, + keygen, + value: retrieve_arg(&typ.path.segments[0], 2)?, + } + } found => { let msg = format!( "Invalid pallet::storage, expected ident: `StorageValue` or \ - `StorageMap` or `StorageDoubleMap` in order to expand metadata, found \ - `{}`", + `StorageMap` or `StorageDoubleMap` or `StorageNMap` in order \ + to expand metadata, found `{}`", found, ); return Err(syn::Error::new(item.ty.span(), msg)); diff --git a/frame/support/procedural/src/storage/genesis_config/builder_def.rs b/frame/support/procedural/src/storage/genesis_config/builder_def.rs index 0cbfa04787f78..5b73928951cfa 100644 --- a/frame/support/procedural/src/storage/genesis_config/builder_def.rs +++ b/frame/support/procedural/src/storage/genesis_config/builder_def.rs @@ -120,6 +120,21 @@ impl BuilderDef { }); }} }, + StorageLineTypeDef::NMap(map) => { + let key_tuple = map.to_key_tuple(); + let key_arg = if map.keys.len() == 1 { + quote!((k,)) + } else { + quote!(k) + }; + quote!{{ + #data + let data: &#scrate::sp_std::vec::Vec<(#key_tuple, #value_type)> = data; + data.iter().for_each(|(k, v)| { + <#storage_struct as #scrate::#storage_trait>::insert(#key_arg, v); + }); + }} + }, }); } } diff --git a/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs b/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs index 300e47bc850ee..c54349136cf05 100644 --- a/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs +++ b/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs @@ -104,6 +104,10 @@ impl GenesisConfigDef { parse_quote!( Vec<(#key1, #key2, #value_type)> ) }, + StorageLineTypeDef::NMap(map) => { + let key_tuple = map.to_key_tuple(); + parse_quote!( Vec<(#key_tuple, #value_type)> ) + } }; let default = line.default_value.as_ref() diff --git a/frame/support/procedural/src/storage/genesis_config/mod.rs b/frame/support/procedural/src/storage/genesis_config/mod.rs index 6dfa5a13fe5b2..abc7af729f064 100644 --- a/frame/support/procedural/src/storage/genesis_config/mod.rs +++ b/frame/support/procedural/src/storage/genesis_config/mod.rs @@ -177,10 +177,8 @@ fn impl_build_storage( } } -pub fn genesis_config_and_build_storage( - scrate: &TokenStream, - def: &DeclStorageDefExt, -) -> TokenStream { +pub fn genesis_config_and_build_storage(def: &DeclStorageDefExt) -> TokenStream { + let scrate = &def.hidden_crate; let builders = BuilderDef::from_def(scrate, def); if !builders.blocks.is_empty() { let genesis_config = match GenesisConfigDef::from_def(def) { diff --git a/frame/support/procedural/src/storage/getters.rs b/frame/support/procedural/src/storage/getters.rs index 65a3519033aa2..32155239acdc6 100644 --- a/frame/support/procedural/src/storage/getters.rs +++ b/frame/support/procedural/src/storage/getters.rs @@ -21,7 +21,8 @@ use proc_macro2::TokenStream; use quote::quote; use super::{DeclStorageDefExt, StorageLineTypeDef}; -pub fn impl_getters(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStream { +pub fn impl_getters(def: &DeclStorageDefExt) -> TokenStream { + let scrate = &def.hidden_crate; let mut getters = TokenStream::new(); for (get_fn, line) in def.storage_lines.iter() @@ -65,6 +66,21 @@ pub fn impl_getters(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStrea } } }, + StorageLineTypeDef::NMap(map) => { + let keygen = map.to_keygen_struct(&def.hidden_crate); + let value = &map.value; + quote!{ + pub fn #get_fn(key: KArg) -> #value + where + KArg: #scrate::storage::types::EncodeLikeTuple< + <#keygen as #scrate::storage::types::KeyGenerator>::KArg + > + + #scrate::storage::types::TupleToEncodedIter, + { + <#storage_struct as #scrate::#storage_trait>::get(key) + } + } + } }; getters.extend(getter); } diff --git a/frame/support/procedural/src/storage/instance_trait.rs b/frame/support/procedural/src/storage/instance_trait.rs index a9e06c6299041..55f6ef478054c 100644 --- a/frame/support/procedural/src/storage/instance_trait.rs +++ b/frame/support/procedural/src/storage/instance_trait.rs @@ -34,7 +34,8 @@ struct InstanceDef { index: u8, } -pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStream { +pub fn decl_and_impl(def: &DeclStorageDefExt) -> TokenStream { + let scrate = &def.hidden_crate; let mut impls = TokenStream::new(); impls.extend(reexport_instance_trait(scrate, def)); diff --git a/frame/support/procedural/src/storage/metadata.rs b/frame/support/procedural/src/storage/metadata.rs index c321386ae1dc4..8a42dd4308d12 100644 --- a/frame/support/procedural/src/storage/metadata.rs +++ b/frame/support/procedural/src/storage/metadata.rs @@ -63,6 +63,27 @@ fn storage_line_metadata_type(scrate: &TokenStream, line: &StorageLineDefExt) -> } } }, + StorageLineTypeDef::NMap(map) => { + let keys = map.keys + .iter() + .map(|key| clean_type_string("e!(#key).to_string())) + .collect::>(); + let hashers = map.hashers + .iter() + .map(|hasher| hasher.to_storage_hasher_struct()) + .collect::>(); + quote!{ + #scrate::metadata::StorageEntryType::NMap { + keys: #scrate::metadata::DecodeDifferent::Encode(&[ + #( #keys, )* + ]), + hashers: #scrate::metadata::DecodeDifferent::Encode(&[ + #( #scrate::metadata::StorageHasher::#hashers, )* + ]), + value: #scrate::metadata::DecodeDifferent::Encode(#value_type), + } + } + } } } @@ -140,7 +161,8 @@ fn default_byte_getter( (struct_def, struct_instance) } -pub fn impl_metadata(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStream { +pub fn impl_metadata(def: &DeclStorageDefExt) -> TokenStream { + let scrate = &def.hidden_crate; let mut entries = TokenStream::new(); let mut default_byte_getter_struct_defs = TokenStream::new(); diff --git a/frame/support/procedural/src/storage/mod.rs b/frame/support/procedural/src/storage/mod.rs index 2f9625d2c941e..71bcf704f0d73 100644 --- a/frame/support/procedural/src/storage/mod.rs +++ b/frame/support/procedural/src/storage/mod.rs @@ -70,7 +70,9 @@ impl syn::parse::Parse for DeclStorageDef { /// Extended version of `DeclStorageDef` with useful precomputed value. pub struct DeclStorageDefExt { /// Name of the module used to import hidden imports. - hidden_crate: Option, + hidden_crate: proc_macro2::TokenStream, + /// Hidden imports used by the module. + hidden_imports: proc_macro2::TokenStream, /// Visibility of store trait. visibility: syn::Visibility, /// Name of store trait: usually `Store`. @@ -108,9 +110,15 @@ pub struct DeclStorageDefExt { impl From for DeclStorageDefExt { fn from(mut def: DeclStorageDef) -> Self { + let hidden_crate_name = def.hidden_crate.as_ref().map(|i| i.to_string()) + .unwrap_or_else(|| "decl_storage".to_string()); + + let hidden_crate = generate_crate_access(&hidden_crate_name, "frame-support"); + let hidden_imports = generate_hidden_includes(&hidden_crate_name, "frame-support"); + let storage_lines = def.storage_lines.drain(..).collect::>(); let storage_lines = storage_lines.into_iter() - .map(|line| StorageLineDefExt::from_def(line, &def)) + .map(|line| StorageLineDefExt::from_def(line, &def, &hidden_crate)) .collect(); let ( @@ -144,7 +152,8 @@ impl From for DeclStorageDefExt { ); Self { - hidden_crate: def.hidden_crate, + hidden_crate, + hidden_imports, visibility: def.visibility, store_trait: def.store_trait, module_name: def.module_name, @@ -230,7 +239,11 @@ pub struct StorageLineDefExt { } impl StorageLineDefExt { - fn from_def(storage_def: StorageLineDef, def: &DeclStorageDef) -> Self { + fn from_def( + storage_def: StorageLineDef, + def: &DeclStorageDef, + hidden_crate: &proc_macro2::TokenStream, + ) -> Self { let is_generic = match &storage_def.storage_type { StorageLineTypeDef::Simple(value) => { ext::type_contains_ident(&value, &def.module_runtime_generic) @@ -244,12 +257,17 @@ impl StorageLineDefExt { || ext::type_contains_ident(&map.key2, &def.module_runtime_generic) || ext::type_contains_ident(&map.value, &def.module_runtime_generic) } + StorageLineTypeDef::NMap(map) => { + map.keys.iter().any(|key| ext::type_contains_ident(key, &def.module_runtime_generic)) + || ext::type_contains_ident(&map.value, &def.module_runtime_generic) + } }; let query_type = match &storage_def.storage_type { StorageLineTypeDef::Simple(value) => value.clone(), StorageLineTypeDef::Map(map) => map.value.clone(), StorageLineTypeDef::DoubleMap(map) => map.value.clone(), + StorageLineTypeDef::NMap(map) => map.value.clone(), }; let is_option = ext::extract_type_option(&query_type).is_some(); let value_type = ext::extract_type_option(&query_type).unwrap_or_else(|| query_type.clone()); @@ -295,6 +313,10 @@ impl StorageLineDefExt { let key2 = &map.key2; quote!( StorageDoubleMap<#key1, #key2, #value_type> ) }, + StorageLineTypeDef::NMap(map) => { + let keygen = map.to_keygen_struct(hidden_crate); + quote!( StorageNMap<#keygen, #value_type> ) + } }; let storage_trait = quote!( storage::#storage_trait_truncated ); @@ -332,6 +354,7 @@ impl StorageLineDefExt { pub enum StorageLineTypeDef { Map(MapDef), DoubleMap(Box), + NMap(NMapDef), Simple(syn::Type), } @@ -351,6 +374,42 @@ pub struct DoubleMapDef { pub value: syn::Type, } +pub struct NMapDef { + pub hashers: Vec, + pub keys: Vec, + pub value: syn::Type, +} + +impl NMapDef { + fn to_keygen_struct(&self, scrate: &proc_macro2::TokenStream) -> proc_macro2::TokenStream { + if self.keys.len() == 1 { + let hasher = &self.hashers[0].to_storage_hasher_struct(); + let key = &self.keys[0]; + return quote!( #scrate::storage::types::Key<#scrate::#hasher, #key> ); + } + + let key_hasher = self.keys.iter().zip(&self.hashers).map(|(key, hasher)| { + let hasher = hasher.to_storage_hasher_struct(); + quote!( #scrate::storage::types::Key<#scrate::#hasher, #key> ) + }) + .collect::>(); + quote!(( #(#key_hasher,)* )) + } + + fn to_key_tuple(&self) -> proc_macro2::TokenStream { + if self.keys.len() == 1 { + let key = &self.keys[0]; + return quote!(#key); + } + + let tuple = self.keys.iter().map(|key| { + quote!(#key) + }) + .collect::>(); + quote!(( #(#tuple,)* )) + } +} + pub struct ExtraGenesisLineDef { attrs: Vec, name: syn::Ident, @@ -402,26 +461,24 @@ pub fn decl_storage_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStr print_pallet_upgrade::maybe_print_pallet_upgrade(&def_ext); - let hidden_crate_name = def_ext.hidden_crate.as_ref().map(|i| i.to_string()) - .unwrap_or_else(|| "decl_storage".to_string()); - - let scrate = generate_crate_access(&hidden_crate_name, "frame-support"); - let scrate_decl = generate_hidden_includes(&hidden_crate_name, "frame-support"); - + let scrate = &def_ext.hidden_crate; + let scrate_decl = &def_ext.hidden_imports; let store_trait = store_trait::decl_and_impl(&def_ext); - let getters = getters::impl_getters(&scrate, &def_ext); - let metadata = metadata::impl_metadata(&scrate, &def_ext); - let instance_trait = instance_trait::decl_and_impl(&scrate, &def_ext); - let genesis_config = genesis_config::genesis_config_and_build_storage(&scrate, &def_ext); - let storage_struct = storage_struct::decl_and_impl(&scrate, &def_ext); + let getters = getters::impl_getters(&def_ext); + let metadata = metadata::impl_metadata(&def_ext); + let instance_trait = instance_trait::decl_and_impl(&def_ext); + let genesis_config = genesis_config::genesis_config_and_build_storage(&def_ext); + let storage_struct = storage_struct::decl_and_impl(&def_ext); quote!( use #scrate::{ StorageValue as _, StorageMap as _, StorageDoubleMap as _, + StorageNMap as _, StoragePrefixedMap as _, IterableStorageMap as _, + IterableStorageNMap as _, IterableStorageDoubleMap as _, }; diff --git a/frame/support/procedural/src/storage/parse.rs b/frame/support/procedural/src/storage/parse.rs index 2ff7f1fbf38c9..93a1b844a84a2 100644 --- a/frame/support/procedural/src/storage/parse.rs +++ b/frame/support/procedural/src/storage/parse.rs @@ -29,6 +29,7 @@ mod keyword { syn::custom_keyword!(get); syn::custom_keyword!(map); syn::custom_keyword!(double_map); + syn::custom_keyword!(nmap); syn::custom_keyword!(opaque_blake2_256); syn::custom_keyword!(opaque_blake2_128); syn::custom_keyword!(blake2_128_concat); @@ -199,6 +200,7 @@ impl_parse_for_opt!(DeclStorageBuild => keyword::build); enum DeclStorageType { Map(DeclStorageMap), DoubleMap(Box), + NMap(DeclStorageNMap), Simple(syn::Type), } @@ -208,6 +210,8 @@ impl syn::parse::Parse for DeclStorageType { Ok(Self::Map(input.parse()?)) } else if input.peek(keyword::double_map) { Ok(Self::DoubleMap(input.parse()?)) + } else if input.peek(keyword::nmap) { + Ok(Self::NMap(input.parse()?)) } else { Ok(Self::Simple(input.parse()?)) } @@ -235,7 +239,21 @@ struct DeclStorageDoubleMap { pub value: syn::Type, } -#[derive(ToTokens, Debug)] +#[derive(Parse, ToTokens, Debug)] +struct DeclStorageKey { + pub hasher: Opt, + pub key: syn::Type, +} + +#[derive(Parse, ToTokens, Debug)] +struct DeclStorageNMap { + pub map_keyword: keyword::nmap, + pub storage_keys: ext::PunctuatedTrailing, + pub ass_keyword: Token![=>], + pub value: syn::Type, +} + +#[derive(Clone, ToTokens, Debug)] enum Hasher { Blake2_256(keyword::opaque_blake2_256), Blake2_128(keyword::opaque_blake2_128), @@ -291,7 +309,7 @@ impl syn::parse::Parse for Opt { } } -#[derive(Parse, ToTokens, Debug)] +#[derive(Clone, Parse, ToTokens, Debug)] struct SetHasher { pub hasher_keyword: keyword::hasher, pub inner: ext::Parens, @@ -495,6 +513,18 @@ fn parse_storage_line_defs( value: map.value, }) ), + DeclStorageType::NMap(map) => super::StorageLineTypeDef::NMap( + super::NMapDef { + hashers: map + .storage_keys + .inner + .iter() + .map(|pair| Ok(pair.hasher.inner.clone().ok_or_else(no_hasher_error)?.into())) + .collect::, syn::Error>>()?, + keys: map.storage_keys.inner.iter().map(|pair| pair.key.clone()).collect(), + value: map.value, + } + ), DeclStorageType::Simple(expr) => super::StorageLineTypeDef::Simple(expr), }; diff --git a/frame/support/procedural/src/storage/print_pallet_upgrade.rs b/frame/support/procedural/src/storage/print_pallet_upgrade.rs index 447d13898e8df..a6f64a588b633 100644 --- a/frame/support/procedural/src/storage/print_pallet_upgrade.rs +++ b/frame/support/procedural/src/storage/print_pallet_upgrade.rs @@ -239,6 +239,15 @@ pub fn maybe_print_pallet_upgrade(def: &super::DeclStorageDefExt) { comma_default_value_getter_name = comma_default_value_getter_name, ) }, + StorageLineTypeDef::NMap(map) => { + format!("StorageNMap<_, {keygen}, {value_type}{comma_query_kind}\ + {comma_default_value_getter_name}>", + keygen = map.to_keygen_struct(&def.hidden_crate), + value_type = to_cleaned_string(&value_type), + comma_query_kind = comma_query_kind, + comma_default_value_getter_name = comma_default_value_getter_name, + ) + } StorageLineTypeDef::Simple(_) => { format!("StorageValue<_, {value_type}{comma_query_kind}\ {comma_default_value_getter_name}>", diff --git a/frame/support/procedural/src/storage/storage_struct.rs b/frame/support/procedural/src/storage/storage_struct.rs index 9c049789f9bd9..51b55bdc4f139 100644 --- a/frame/support/procedural/src/storage/storage_struct.rs +++ b/frame/support/procedural/src/storage/storage_struct.rs @@ -47,7 +47,8 @@ fn from_query_to_optional_value(is_option: bool) -> TokenStream { } } -pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStream { +pub fn decl_and_impl(def: &DeclStorageDefExt) -> TokenStream { + let scrate = &def.hidden_crate; let mut impls = TokenStream::new(); for line in &def.storage_lines { @@ -199,6 +200,43 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre #from_optional_value_to_query } + fn from_query_to_optional_value(v: Self::Query) -> Option<#value_type> { + #from_query_to_optional_value + } + } + ) + }, + StorageLineTypeDef::NMap(_) => { + quote!( + impl<#impl_trait> #scrate::storage::StoragePrefixedMap<#value_type> + for #storage_struct #optional_storage_where_clause + { + fn module_prefix() -> &'static [u8] { + <#instance_or_inherent as #scrate::traits::Instance>::PREFIX.as_bytes() + } + + fn storage_prefix() -> &'static [u8] { + #storage_name_bstr + } + } + + impl<#impl_trait> #scrate::#storage_generator_trait for #storage_struct + #optional_storage_where_clause + { + type Query = #query_type; + + fn module_prefix() -> &'static [u8] { + <#instance_or_inherent as #scrate::traits::Instance>::PREFIX.as_bytes() + } + + fn storage_prefix() -> &'static [u8] { + #storage_name_bstr + } + + fn from_optional_value_to_query(v: Option<#value_type>) -> Self::Query { + #from_optional_value_to_query + } + fn from_query_to_optional_value(v: Self::Query) -> Option<#value_type> { #from_query_to_optional_value } diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 7539c3c938295..d87ab8e6ed460 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -74,8 +74,8 @@ pub use self::hash::{ StorageHasher, ReversibleStorageHasher }; pub use self::storage::{ - StorageValue, StorageMap, StorageDoubleMap, StoragePrefixedMap, IterableStorageMap, - IterableStorageDoubleMap, migration, + StorageValue, StorageMap, StorageDoubleMap, StorageNMap, StoragePrefixedMap, + IterableStorageMap, IterableStorageDoubleMap, IterableStorageNMap, migration, bounded_vec::{self, BoundedVec}, }; pub use self::dispatch::{Parameter, Callable}; @@ -1237,7 +1237,10 @@ pub mod pallet_prelude { traits::{Get, Hooks, IsType, GetPalletVersion, EnsureOrigin, PalletInfoAccess}, dispatch::{DispatchResultWithPostInfo, Parameter, DispatchError, DispatchResult}, weights::{DispatchClass, Pays, Weight}, - storage::types::{StorageValue, StorageMap, StorageDoubleMap, ValueQuery, OptionQuery}, + storage::types::{ + Key as NMapKey, StorageDoubleMap, StorageMap, StorageNMap, StorageValue, ValueQuery, + OptionQuery, + }, storage::bounded_vec::BoundedVec, }; pub use codec::{Encode, Decode}; diff --git a/frame/support/src/storage/generator/mod.rs b/frame/support/src/storage/generator/mod.rs index 86eafe86f43f4..578831314c1f6 100644 --- a/frame/support/src/storage/generator/mod.rs +++ b/frame/support/src/storage/generator/mod.rs @@ -25,10 +25,12 @@ //! This is internal api and is subject to change. mod map; +mod nmap; mod double_map; mod value; pub use map::StorageMap; +pub use nmap::StorageNMap; pub use double_map::StorageDoubleMap; pub use value::StorageValue; diff --git a/frame/support/src/storage/generator/nmap.rs b/frame/support/src/storage/generator/nmap.rs new file mode 100755 index 0000000000000..d1f00adda5e55 --- /dev/null +++ b/frame/support/src/storage/generator/nmap.rs @@ -0,0 +1,541 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Generator for `StorageNMap` used by `decl_storage` and storage types. +//! +//! By default each key value is stored at: +//! ```nocompile +//! Twox128(pallet_prefix) ++ Twox128(storage_prefix) +//! ++ Hasher1(encode(key1)) ++ Hasher2(encode(key2)) ++ ... ++ HasherN(encode(keyN)) +//! ``` +//! +//! # Warning +//! +//! If the keys are not trusted (e.g. can be set by a user), a cryptographic `hasher` such as +//! `blake2_256` must be used. Otherwise, other values in storage with the same prefix can +//! be compromised. + +use crate::{ + hash::{StorageHasher, Twox128}, + storage::{ + self, + types::{ + EncodeLikeTuple, HasKeyPrefix, HasReversibleKeyPrefix, KeyGenerator, + ReversibleKeyGenerator, TupleToEncodedIter, + }, + unhashed, PrefixIterator, StorageAppend, + }, + Never, +}; +use codec::{Decode, Encode, EncodeLike, FullCodec}; +#[cfg(not(feature = "std"))] +use sp_std::prelude::*; + +/// Generator for `StorageNMap` used by `decl_storage` and storage types. +/// +/// By default each key value is stored at: +/// ```nocompile +/// Twox128(pallet_prefix) ++ Twox128(storage_prefix) +/// ++ Hasher1(encode(key1)) ++ Hasher2(encode(key2)) ++ ... ++ HasherN(encode(keyN)) +/// ``` +/// +/// # Warning +/// +/// If the keys are not trusted (e.g. can be set by a user), a cryptographic `hasher` such as +/// `blake2_256` must be used. Otherwise, other values in storage with the same prefix can +/// be compromised. +pub trait StorageNMap { + /// The type that get/take returns. + type Query; + + /// Module prefix. Used for generating final key. + fn module_prefix() -> &'static [u8]; + + /// Storage prefix. Used for generating final key. + fn storage_prefix() -> &'static [u8]; + + /// The full prefix; just the hash of `module_prefix` concatenated to the hash of + /// `storage_prefix`. + fn prefix_hash() -> Vec { + let module_prefix_hashed = Twox128::hash(Self::module_prefix()); + let storage_prefix_hashed = Twox128::hash(Self::storage_prefix()); + + let mut result = + Vec::with_capacity(module_prefix_hashed.len() + storage_prefix_hashed.len()); + + result.extend_from_slice(&module_prefix_hashed[..]); + result.extend_from_slice(&storage_prefix_hashed[..]); + + result + } + + /// Convert an optional value retrieved from storage to the type queried. + fn from_optional_value_to_query(v: Option) -> Self::Query; + + /// Convert a query to an optional value into storage. + fn from_query_to_optional_value(v: Self::Query) -> Option; + + /// Generate a partial key used in top storage. + fn storage_n_map_partial_key(key: KP) -> Vec + where + K: HasKeyPrefix, + { + let module_prefix_hashed = Twox128::hash(Self::module_prefix()); + let storage_prefix_hashed = Twox128::hash(Self::storage_prefix()); + let key_hashed = >::partial_key(key); + + let mut final_key = Vec::with_capacity( + module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.len(), + ); + + final_key.extend_from_slice(&module_prefix_hashed[..]); + final_key.extend_from_slice(&storage_prefix_hashed[..]); + final_key.extend_from_slice(key_hashed.as_ref()); + + final_key + } + + /// Generate the full key used in top storage. + fn storage_n_map_final_key(key: KArg) -> Vec + where + KG: KeyGenerator, + KArg: EncodeLikeTuple + TupleToEncodedIter, + { + let module_prefix_hashed = Twox128::hash(Self::module_prefix()); + let storage_prefix_hashed = Twox128::hash(Self::storage_prefix()); + let key_hashed = KG::final_key(key); + + let mut final_key = Vec::with_capacity( + module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.len(), + ); + + final_key.extend_from_slice(&module_prefix_hashed[..]); + final_key.extend_from_slice(&storage_prefix_hashed[..]); + final_key.extend_from_slice(key_hashed.as_ref()); + + final_key + } +} + +impl storage::StorageNMap for G +where + K: KeyGenerator, + V: FullCodec, + G: StorageNMap, +{ + type Query = G::Query; + + fn hashed_key_for + TupleToEncodedIter>(key: KArg) -> Vec { + Self::storage_n_map_final_key::(key) + } + + fn contains_key + TupleToEncodedIter>(key: KArg) -> bool { + unhashed::exists(&Self::storage_n_map_final_key::(key)) + } + + fn get + TupleToEncodedIter>(key: KArg) -> Self::Query { + G::from_optional_value_to_query(unhashed::get(&Self::storage_n_map_final_key::(key))) + } + + fn try_get + TupleToEncodedIter>(key: KArg) -> Result { + unhashed::get(&Self::storage_n_map_final_key::(key)).ok_or(()) + } + + fn take + TupleToEncodedIter>(key: KArg) -> Self::Query { + let final_key = Self::storage_n_map_final_key::(key); + + let value = unhashed::take(&final_key); + G::from_optional_value_to_query(value) + } + + fn swap(key1: KArg1, key2: KArg2) + where + KOther: KeyGenerator, + KArg1: EncodeLikeTuple + TupleToEncodedIter, + KArg2: EncodeLikeTuple + TupleToEncodedIter, + { + let final_x_key = Self::storage_n_map_final_key::(key1); + let final_y_key = Self::storage_n_map_final_key::(key2); + + let v1 = unhashed::get_raw(&final_x_key); + if let Some(val) = unhashed::get_raw(&final_y_key) { + unhashed::put_raw(&final_x_key, &val); + } else { + unhashed::kill(&final_x_key); + } + if let Some(val) = v1 { + unhashed::put_raw(&final_y_key, &val); + } else { + unhashed::kill(&final_y_key); + } + } + + fn insert(key: KArg, val: VArg) + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + VArg: EncodeLike, + { + unhashed::put(&Self::storage_n_map_final_key::(key), &val); + } + + fn remove + TupleToEncodedIter>(key: KArg) { + unhashed::kill(&Self::storage_n_map_final_key::(key)); + } + + fn remove_prefix(partial_key: KP) + where + K: HasKeyPrefix, + { + unhashed::kill_prefix(&Self::storage_n_map_partial_key(partial_key)); + } + + fn iter_prefix_values(partial_key: KP) -> PrefixIterator + where + K: HasKeyPrefix, + { + let prefix = Self::storage_n_map_partial_key(partial_key); + PrefixIterator { + prefix: prefix.clone(), + previous_key: prefix, + drain: false, + closure: |_raw_key, mut raw_value| V::decode(&mut raw_value), + } + } + + fn mutate(key: KArg, f: F) -> R + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Self::Query) -> R, + { + Self::try_mutate(key, |v| Ok::(f(v))) + .expect("`Never` can not be constructed; qed") + } + + fn try_mutate(key: KArg, f: F) -> Result + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Self::Query) -> Result + { + let final_key = Self::storage_n_map_final_key::(key); + let mut val = G::from_optional_value_to_query(unhashed::get(final_key.as_ref())); + + let ret = f(&mut val); + if ret.is_ok() { + match G::from_query_to_optional_value(val) { + Some(ref val) => unhashed::put(final_key.as_ref(), val), + None => unhashed::kill(final_key.as_ref()), + } + } + ret + } + + fn mutate_exists(key: KArg, f: F) -> R + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Option) -> R, + { + Self::try_mutate_exists(key, |v| Ok::(f(v))) + .expect("`Never` can not be constructed; qed") + } + + fn try_mutate_exists(key: KArg, f: F) -> Result + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Option) -> Result, + { + let final_key = Self::storage_n_map_final_key::(key); + let mut val = unhashed::get(final_key.as_ref()); + + let ret = f(&mut val); + if ret.is_ok() { + match val { + Some(ref val) => unhashed::put(final_key.as_ref(), val), + None => unhashed::kill(final_key.as_ref()), + } + } + ret + } + + fn append(key: KArg, item: EncodeLikeItem) + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + Item: Encode, + EncodeLikeItem: EncodeLike, + V: StorageAppend, + { + let final_key = Self::storage_n_map_final_key::(key); + sp_io::storage::append(&final_key, item.encode()); + } + + fn migrate_keys(key: KArg, hash_fns: K::HArg) -> Option + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + { + let old_key = { + let module_prefix_hashed = Twox128::hash(Self::module_prefix()); + let storage_prefix_hashed = Twox128::hash(Self::storage_prefix()); + let key_hashed = K::migrate_key(&key, hash_fns); + + let mut final_key = Vec::with_capacity( + module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.len(), + ); + + final_key.extend_from_slice(&module_prefix_hashed[..]); + final_key.extend_from_slice(&storage_prefix_hashed[..]); + final_key.extend_from_slice(key_hashed.as_ref()); + + final_key + }; + unhashed::take(old_key.as_ref()).map(|value| { + unhashed::put(Self::storage_n_map_final_key::(key).as_ref(), &value); + value + }) + } +} + +impl> + storage::IterableStorageNMap for G +{ + type Iterator = PrefixIterator<(K::Key, V)>; + + fn iter_prefix(kp: KP) -> PrefixIterator<(>::Suffix, V)> + where + K: HasReversibleKeyPrefix, + { + let prefix = G::storage_n_map_partial_key(kp); + PrefixIterator { + prefix: prefix.clone(), + previous_key: prefix, + drain: false, + closure: |raw_key_without_prefix, mut raw_value| { + let partial_key = K::decode_partial_key(raw_key_without_prefix)?; + Ok((partial_key, V::decode(&mut raw_value)?)) + }, + } + } + + fn drain_prefix(kp: KP) -> PrefixIterator<(>::Suffix, V)> + where + K: HasReversibleKeyPrefix, + { + let mut iter = Self::iter_prefix(kp); + iter.drain = true; + iter + } + + fn iter() -> Self::Iterator { + let prefix = G::prefix_hash(); + Self::Iterator { + prefix: prefix.clone(), + previous_key: prefix, + drain: false, + closure: |raw_key_without_prefix, mut raw_value| { + let (final_key, _) = K::decode_final_key(raw_key_without_prefix)?; + Ok((final_key, V::decode(&mut raw_value)?)) + }, + } + } + + fn drain() -> Self::Iterator { + let mut iterator = Self::iter(); + iterator.drain = true; + iterator + } + + fn translate Option>(mut f: F) { + let prefix = G::prefix_hash(); + let mut previous_key = prefix.clone(); + while let Some(next) = + sp_io::storage::next_key(&previous_key).filter(|n| n.starts_with(&prefix)) + { + previous_key = next; + let value = match unhashed::get::(&previous_key) { + Some(value) => value, + None => { + log::error!("Invalid translate: fail to decode old value"); + continue; + } + }; + + let final_key = match K::decode_final_key(&previous_key[prefix.len()..]) { + Ok((final_key, _)) => final_key, + Err(_) => { + log::error!("Invalid translate: fail to decode key"); + continue; + } + }; + + match f(final_key, value) { + Some(new) => unhashed::put::(&previous_key, &new), + None => unhashed::kill(&previous_key), + } + } + } +} + +/// Test iterators for StorageNMap +#[cfg(test)] +mod test_iterators { + use crate::{ + hash::StorageHasher, + storage::{generator::StorageNMap, unhashed, IterableStorageNMap}, + }; + use codec::{Decode, Encode}; + + pub trait Config: 'static { + type Origin; + type BlockNumber; + type PalletInfo: crate::traits::PalletInfo; + type DbWeight: crate::traits::Get; + } + + crate::decl_module! { + pub struct Module for enum Call where origin: T::Origin, system=self {} + } + + #[derive(PartialEq, Eq, Clone, Encode, Decode)] + struct NoDef(u32); + + crate::decl_storage! { + trait Store for Module as Test { + NMap: nmap hasher(blake2_128_concat) u16, hasher(twox_64_concat) u32 => u64; + } + } + + fn key_before_prefix(mut prefix: Vec) -> Vec { + let last = prefix.iter_mut().last().unwrap(); + assert!(*last != 0, "mock function not implemented for this prefix"); + *last -= 1; + prefix + } + + fn key_after_prefix(mut prefix: Vec) -> Vec { + let last = prefix.iter_mut().last().unwrap(); + assert!( + *last != 255, + "mock function not implemented for this prefix" + ); + *last += 1; + prefix + } + + #[test] + fn n_map_reversible_reversible_iteration() { + sp_io::TestExternalities::default().execute_with(|| { + // All map iterator + let prefix = NMap::prefix_hash(); + + unhashed::put(&key_before_prefix(prefix.clone()), &1u64); + unhashed::put(&key_after_prefix(prefix.clone()), &1u64); + + for i in 0..4 { + NMap::insert((i as u16, i as u32), i as u64); + } + + assert_eq!( + NMap::iter().collect::>(), + vec![((3, 3), 3), ((0, 0), 0), ((2, 2), 2), ((1, 1), 1)], + ); + + assert_eq!(NMap::iter_values().collect::>(), vec![3, 0, 2, 1],); + + assert_eq!( + NMap::drain().collect::>(), + vec![((3, 3), 3), ((0, 0), 0), ((2, 2), 2), ((1, 1), 1)], + ); + + assert_eq!(NMap::iter().collect::>(), vec![]); + assert_eq!( + unhashed::get(&key_before_prefix(prefix.clone())), + Some(1u64) + ); + assert_eq!(unhashed::get(&key_after_prefix(prefix.clone())), Some(1u64)); + + // Prefix iterator + let k1 = 3 << 8; + let prefix = NMap::storage_n_map_partial_key((k1,)); + + unhashed::put(&key_before_prefix(prefix.clone()), &1u64); + unhashed::put(&key_after_prefix(prefix.clone()), &1u64); + + for i in 0..4 { + NMap::insert((k1, i as u32), i as u64); + } + + assert_eq!( + NMap::iter_prefix((k1,)).collect::>(), + vec![(1, 1), (2, 2), (0, 0), (3, 3)], + ); + + assert_eq!( + NMap::iter_prefix_values((k1,)).collect::>(), + vec![1, 2, 0, 3], + ); + + assert_eq!( + NMap::drain_prefix((k1,)).collect::>(), + vec![(1, 1), (2, 2), (0, 0), (3, 3)], + ); + + assert_eq!(NMap::iter_prefix((k1,)).collect::>(), vec![]); + assert_eq!( + unhashed::get(&key_before_prefix(prefix.clone())), + Some(1u64) + ); + assert_eq!(unhashed::get(&key_after_prefix(prefix.clone())), Some(1u64)); + + // Translate + let prefix = NMap::prefix_hash(); + + unhashed::put(&key_before_prefix(prefix.clone()), &1u64); + unhashed::put(&key_after_prefix(prefix.clone()), &1u64); + for i in 0..4 { + NMap::insert((i as u16, i as u32), i as u64); + } + + // Wrong key1 + unhashed::put(&[prefix.clone(), vec![1, 2, 3]].concat(), &3u64.encode()); + + // Wrong key2 + unhashed::put( + &[ + prefix.clone(), + crate::Blake2_128Concat::hash(&1u16.encode()), + ] + .concat(), + &3u64.encode(), + ); + + // Wrong value + unhashed::put( + &[ + prefix.clone(), + crate::Blake2_128Concat::hash(&1u16.encode()), + crate::Twox64Concat::hash(&2u32.encode()), + ] + .concat(), + &vec![1], + ); + + NMap::translate(|(_k1, _k2), v: u64| Some(v * 2)); + assert_eq!( + NMap::iter().collect::>(), + vec![((3, 3), 6), ((0, 0), 0), ((2, 2), 4), ((1, 1), 2)], + ); + }) + } +} diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 437dd28060f74..b779e064ac206 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -20,9 +20,16 @@ use sp_core::storage::ChildInfo; use sp_std::prelude::*; use codec::{FullCodec, FullEncode, Encode, EncodeLike, Decode}; -use crate::hash::{Twox128, StorageHasher, ReversibleStorageHasher}; +use crate::{ + hash::{Twox128, StorageHasher, ReversibleStorageHasher}, + storage::types::{ + EncodeLikeTuple, HasKeyPrefix, HasReversibleKeyPrefix, KeyGenerator, + ReversibleKeyGenerator, TupleToEncodedIter, + }, +}; use sp_runtime::generic::{Digest, DigestItem}; pub use sp_runtime::TransactionOutcome; +pub use types::Key; pub mod unhashed; pub mod hashed; @@ -359,6 +366,39 @@ pub trait IterableStorageDoubleMap< fn translate Option>(f: F); } +/// A strongly-typed map with arbitrary number of keys in storage whose keys and values can be +/// iterated over. +pub trait IterableStorageNMap: StorageNMap { + /// The type that iterates over all `(key1, (key2, (key3, ... (keyN, ()))), value)` tuples + type Iterator: Iterator; + + /// Enumerate all elements in the map with prefix key `kp` in no particular order. If you add or + /// remove values whose prefix is `kp` to the map while doing this, you'll get undefined + /// results. + fn iter_prefix(kp: KP) -> PrefixIterator<(>::Suffix, V)> + where K: HasReversibleKeyPrefix; + + /// Remove all elements from the map with prefix key `kp` and iterate through them in no + /// particular order. If you add elements with prefix key `kp` to the map while doing this, + /// you'll get undefined results. + fn drain_prefix(kp: KP) -> PrefixIterator<(>::Suffix, V)> + where K: HasReversibleKeyPrefix; + + /// Enumerate all elements in the map in no particular order. If you add or remove values to + /// the map while doing this, you'll get undefined results. + fn iter() -> Self::Iterator; + + /// Remove all elements from the map and iterate through them in no particular order. If you + /// add elements to the map while doing this, you'll get undefined results. + fn drain() -> Self::Iterator; + + /// Translate the values of all elements by a function `f`, in the map in no particular order. + /// By returning `None` from `f` for an element, you'll remove it from the map. + /// + /// NOTE: If a value fail to decode because storage is corrupted then it is skipped. + fn translate Option>(f: F); +} + /// An implementation of a map with a two keys. /// /// It provides an important ability to efficiently remove all entries @@ -510,6 +550,121 @@ pub trait StorageDoubleMap { >(key1: KeyArg1, key2: KeyArg2) -> Option; } +/// An implementation of a map with an arbitrary number of keys. +/// +/// Details of implementation can be found at [`generator::StorageNMap`]. +pub trait StorageNMap { + /// The type that get/take returns. + type Query; + + /// Get the storage key used to fetch a value corresponding to a specific key. + fn hashed_key_for + TupleToEncodedIter>(key: KArg) -> Vec; + + /// Does the value (explicitly) exist in storage? + fn contains_key + TupleToEncodedIter>(key: KArg) -> bool; + + /// Load the value associated with the given key from the map. + fn get + TupleToEncodedIter>(key: KArg) -> Self::Query; + + /// Try to get the value for the given key from the map. + /// + /// Returns `Ok` if it exists, `Err` if not. + fn try_get + TupleToEncodedIter>(key: KArg) -> Result; + + /// Swap the values of two keys. + fn swap(key1: KArg1, key2: KArg2) + where + KOther: KeyGenerator, + KArg1: EncodeLikeTuple + TupleToEncodedIter, + KArg2: EncodeLikeTuple + TupleToEncodedIter; + + /// Store a value to be associated with the given key from the map. + fn insert(key: KArg, val: VArg) + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + VArg: EncodeLike; + + /// Remove the value under a key. + fn remove + TupleToEncodedIter>(key: KArg); + + /// Remove all values under the partial prefix key. + fn remove_prefix(partial_key: KP) where K: HasKeyPrefix; + + /// Iterate over values that share the partial prefix key. + fn iter_prefix_values(partial_key: KP) -> PrefixIterator where K: HasKeyPrefix; + + /// Mutate the value under a key. + fn mutate(key: KArg, f: F) -> R + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Self::Query) -> R; + + /// Mutate the item, only if an `Ok` value is returned. + fn try_mutate(key: KArg, f: F) -> Result + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Self::Query) -> Result; + + /// Mutate the value under a key. + /// + /// Deletes the item if mutated to a `None`. + fn mutate_exists(key: KArg, f: F) -> R + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Option) -> R; + + /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + fn try_mutate_exists(key: KArg, f: F) -> Result + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Option) -> Result; + + /// Take the value under a key. + fn take + TupleToEncodedIter>(key: KArg) -> Self::Query; + + /// Append the given items to the value in the storage. + /// + /// `V` is required to implement `codec::EncodeAppend`. + /// + /// # Warning + /// + /// If the storage item is not encoded properly, the storage will be overwritten + /// and set to `[item]`. Any default value set for the storage item will be ignored + /// on overwrite. + fn append(key: KArg, item: EncodeLikeItem) + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + Item: Encode, + EncodeLikeItem: EncodeLike, + V: StorageAppend; + + /// Read the length of the storage value without decoding the entire value under the + /// given `key`. + /// + /// `V` is required to implement [`StorageDecodeLength`]. + /// + /// If the value does not exists or it fails to decode the length, `None` is returned. + /// Otherwise `Some(len)` is returned. + /// + /// # Warning + /// + /// `None` does not mean that `get()` does not return a value. The default value is completly + /// ignored by this function. + fn decode_len + TupleToEncodedIter>(key: KArg) -> Option + where + V: StorageDecodeLength, + { + V::decode_len(&Self::hashed_key_for(key)) + } + + /// Migrate an item with the given `key` from defunct `hash_fns` to the current hashers. + /// + /// If the key doesn't exist, then it's a no-op. If it does, then it returns its value. + fn migrate_keys(key: KArg, hash_fns: K::HArg) -> Option + where + KArg: EncodeLikeTuple + TupleToEncodedIter; +} + /// Iterate over a prefix and decode raw_key and raw_value into `T`. /// /// If any decoding fails it skips it and continues to the next key. @@ -806,7 +961,7 @@ pub trait StorageDecodeLength: private::Sealed + codec::DecodeLength { } /// Provides `Sealed` trait to prevent implementing trait `StorageAppend` & `StorageDecodeLength` -/// outside of this crate. +/// & `EncodeLikeTuple` outside of this crate. mod private { use super::*; use bounded_vec::BoundedVec; @@ -818,6 +973,33 @@ mod private { impl Sealed for BoundedVec {} impl Sealed for bounded_btree_map::BoundedBTreeMap {} impl Sealed for bounded_btree_set::BoundedBTreeSet {} + + macro_rules! impl_sealed_for_tuple { + ($($elem:ident),+) => { + paste::paste! { + impl<$($elem: Encode,)+> Sealed for ($($elem,)+) {} + impl<$($elem: Encode,)+> Sealed for &($($elem,)+) {} + } + }; + } + + impl_sealed_for_tuple!(A); + impl_sealed_for_tuple!(A, B); + impl_sealed_for_tuple!(A, B, C); + impl_sealed_for_tuple!(A, B, C, D); + impl_sealed_for_tuple!(A, B, C, D, E); + impl_sealed_for_tuple!(A, B, C, D, E, F); + impl_sealed_for_tuple!(A, B, C, D, E, F, G); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M, O); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P, Q); + impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P, Q, R); } impl StorageAppend for Vec {} diff --git a/frame/support/src/storage/types/key.rs b/frame/support/src/storage/types/key.rs new file mode 100755 index 0000000000000..fb3c69ff20cde --- /dev/null +++ b/frame/support/src/storage/types/key.rs @@ -0,0 +1,957 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storage key type. + +use crate::hash::{ReversibleStorageHasher, StorageHasher}; +use codec::{Encode, EncodeLike, FullCodec}; +use paste::paste; +use sp_std::prelude::*; + +/// A type used exclusively by storage maps as their key type. +/// +/// The final key generated has the following form: +/// ```nocompile +/// Hasher1(encode(key1)) +/// ++ Hasher2(encode(key2)) +/// ++ ... +/// ++ HasherN(encode(keyN)) +/// ``` +pub struct Key(core::marker::PhantomData<(Hasher, KeyType)>); + +/// A trait that contains the current key as an associated type. +pub trait KeyGenerator { + type Key: EncodeLike; + type KArg: Encode; + type HashFn: FnOnce(&[u8]) -> Vec; + type HArg; + + const HASHER_METADATA: &'static [frame_metadata::StorageHasher]; + + /// Given a `key` tuple, calculate the final key by encoding each element individuallly and + /// hashing them using the corresponding hasher in the `KeyGenerator`. + fn final_key + TupleToEncodedIter>(key: KArg) -> Vec; + /// Given a `key` tuple, migrate the keys from using the old hashers as given by `hash_fns` + /// to using the newer hashers as specified by this `KeyGenerator`. + fn migrate_key + TupleToEncodedIter>( + key: &KArg, + hash_fns: Self::HArg, + ) -> Vec; +} + +/// A trait containing methods that are only implemented on the Key struct instead of the entire tuple. +pub trait KeyGeneratorInner: KeyGenerator { + type Hasher: StorageHasher; + + /// Hash a given `encoded` byte slice using the `KeyGenerator`'s associated `StorageHasher`. + fn final_hash(encoded: &[u8]) -> Vec; +} + +impl KeyGenerator for Key { + type Key = K; + type KArg = (K,); + type HashFn = Box Vec>; + type HArg = (Self::HashFn,); + + const HASHER_METADATA: &'static [frame_metadata::StorageHasher] = &[H::METADATA]; + + fn final_key + TupleToEncodedIter>(key: KArg) -> Vec { + H::hash( + &key.to_encoded_iter() + .next() + .expect("should have at least one element!"), + ) + .as_ref() + .to_vec() + } + + fn migrate_key + TupleToEncodedIter>( + key: &KArg, + hash_fns: Self::HArg, + ) -> Vec { + (hash_fns.0)( + &key.to_encoded_iter() + .next() + .expect("should have at least one element!"), + ) + } +} + +impl KeyGeneratorInner for Key { + type Hasher = H; + + fn final_hash(encoded: &[u8]) -> Vec { + H::hash(encoded).as_ref().to_vec() + } +} + +#[impl_trait_for_tuples::impl_for_tuples(2, 18)] +#[tuple_types_custom_trait_bound(KeyGeneratorInner)] +impl KeyGenerator for Tuple { + for_tuples!( type Key = ( #(Tuple::Key),* ); ); + for_tuples!( type KArg = ( #(Tuple::Key),* ); ); + for_tuples!( type HArg = ( #(Tuple::HashFn),* ); ); + type HashFn = Box Vec>; + + const HASHER_METADATA: &'static [frame_metadata::StorageHasher] = &[ + for_tuples!( #(Tuple::Hasher::METADATA),* ) + ]; + + fn final_key + TupleToEncodedIter>(key: KArg) -> Vec { + let mut final_key = Vec::new(); + let mut iter = key.to_encoded_iter(); + for_tuples!( + #( + let next_encoded = iter.next().expect("KArg number should be equal to Key number"); + final_key.extend_from_slice(&Tuple::final_hash(&next_encoded)); + )* + ); + final_key + } + + fn migrate_key + TupleToEncodedIter>( + key: &KArg, + hash_fns: Self::HArg, + ) -> Vec { + let mut migrated_key = Vec::new(); + let mut iter = key.to_encoded_iter(); + for_tuples!( + #( + let next_encoded = iter.next().expect("KArg number should be equal to Key number"); + migrated_key.extend_from_slice(&(hash_fns.Tuple)(&next_encoded)); + )* + ); + migrated_key + } +} + +/// Marker trait to indicate that each element in the tuple encodes like the corresponding element +/// in another tuple. +/// +/// This trait is sealed. +pub trait EncodeLikeTuple: crate::storage::private::Sealed {} + +macro_rules! impl_encode_like_tuples { + ($($elem:ident),+) => { + paste! { + impl<$($elem: Encode,)+ $([<$elem $elem>]: Encode + EncodeLike<$elem>,)+> + EncodeLikeTuple<($($elem,)+)> for + ($([<$elem $elem>],)+) {} + impl<$($elem: Encode,)+ $([<$elem $elem>]: Encode + EncodeLike<$elem>,)+> + EncodeLikeTuple<($($elem,)+)> for + &($([<$elem $elem>],)+) {} + } + }; +} + +impl_encode_like_tuples!(A); +impl_encode_like_tuples!(A, B); +impl_encode_like_tuples!(A, B, C); +impl_encode_like_tuples!(A, B, C, D); +impl_encode_like_tuples!(A, B, C, D, E); +impl_encode_like_tuples!(A, B, C, D, E, F); +impl_encode_like_tuples!(A, B, C, D, E, F, G); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I, J); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I, J, K); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I, J, K, L); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I, J, K, L, M); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I, J, K, L, M, O); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P, Q); +impl_encode_like_tuples!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P, Q, R); + +/// Trait to indicate that a tuple can be converted into an iterator of a vector of encoded bytes. +pub trait TupleToEncodedIter { + fn to_encoded_iter(&self) -> sp_std::vec::IntoIter>; +} + +#[impl_trait_for_tuples::impl_for_tuples(1, 18)] +#[tuple_types_custom_trait_bound(Encode)] +impl TupleToEncodedIter for Tuple { + fn to_encoded_iter(&self) -> sp_std::vec::IntoIter> { + [for_tuples!( #(self.Tuple.encode()),* )] + .to_vec() + .into_iter() + } +} + +impl TupleToEncodedIter for &T { + fn to_encoded_iter(&self) -> sp_std::vec::IntoIter> { + (*self).to_encoded_iter() + } +} + +/// A trait that indicates the hashers for the keys generated are all reversible. +pub trait ReversibleKeyGenerator: KeyGenerator { + type ReversibleHasher; + fn decode_final_key(key_material: &[u8]) -> Result<(Self::Key, &[u8]), codec::Error>; +} + +impl ReversibleKeyGenerator for Key { + type ReversibleHasher = H; + + fn decode_final_key(key_material: &[u8]) -> Result<(Self::Key, &[u8]), codec::Error> { + let mut current_key_material = Self::ReversibleHasher::reverse(key_material); + let key = K::decode(&mut current_key_material)?; + Ok((key, current_key_material)) + } +} + +#[impl_trait_for_tuples::impl_for_tuples(2, 18)] +#[tuple_types_custom_trait_bound(ReversibleKeyGenerator + KeyGeneratorInner)] +impl ReversibleKeyGenerator for Tuple { + for_tuples!( type ReversibleHasher = ( #(Tuple::ReversibleHasher),* ); ); + + fn decode_final_key(key_material: &[u8]) -> Result<(Self::Key, &[u8]), codec::Error> { + let mut current_key_material = key_material; + Ok(( + (for_tuples! { + #({ + let (key, material) = Tuple::decode_final_key(current_key_material)?; + current_key_material = material; + key + }),* + }), + current_key_material, + )) + } +} + +/// Trait indicating whether a KeyGenerator has the prefix P. +pub trait HasKeyPrefix

: KeyGenerator { + type Suffix; + + fn partial_key(prefix: P) -> Vec; +} + +/// Trait indicating whether a ReversibleKeyGenerator has the prefix P. +pub trait HasReversibleKeyPrefix

: ReversibleKeyGenerator + HasKeyPrefix

{ + fn decode_partial_key(key_material: &[u8]) -> Result; +} + +macro_rules! impl_key_prefix_for { + (($($keygen:ident),+), ($($prefix:ident),+), ($($suffix:ident),+)) => { + paste! { + impl<$($keygen: FullCodec,)+ $( [<$keygen $keygen>]: StorageHasher),+> + HasKeyPrefix<($($prefix),+)> for + ($(Key<[<$keygen $keygen>], $keygen>),+) + { + type Suffix = ($($suffix),+); + + fn partial_key(prefix: ($($prefix),+)) -> Vec { + <($(Key<[<$prefix $prefix>], $prefix>),+)>::final_key(prefix) + } + } + + impl<$($keygen: FullCodec,)+ $( [<$keygen $keygen>]: ReversibleStorageHasher),+> + HasReversibleKeyPrefix<($($prefix),+)> for + ($(Key<[<$keygen $keygen>], $keygen>),+) + { + fn decode_partial_key(key_material: &[u8]) -> Result { + <($(Key<[<$suffix $suffix>], $suffix>),+)>::decode_final_key(key_material).map(|k| k.0) + } + } + } + }; + (($($keygen:ident),+), $prefix:ident, ($($suffix:ident),+)) => { + paste! { + impl<$($keygen: FullCodec,)+ $( [<$keygen $keygen>]: StorageHasher),+> + HasKeyPrefix<($prefix,)> for + ($(Key<[<$keygen $keygen>], $keygen>),+) + { + type Suffix = ($($suffix),+); + + fn partial_key(prefix: ($prefix,)) -> Vec { + ], $prefix>>::final_key(prefix) + } + } + + impl<$($keygen: FullCodec,)+ $( [<$keygen $keygen>]: ReversibleStorageHasher),+> + HasReversibleKeyPrefix<($prefix,)> for + ($(Key<[<$keygen $keygen>], $keygen>),+) + { + fn decode_partial_key(key_material: &[u8]) -> Result { + <($(Key<[<$suffix $suffix>], $suffix>),+)>::decode_final_key(key_material).map(|k| k.0) + } + } + } + }; + (($($keygen:ident),+), ($($prefix:ident),+), $suffix:ident) => { + paste! { + impl<$($keygen: FullCodec,)+ $( [<$keygen $keygen>]: StorageHasher),+> + HasKeyPrefix<($($prefix),+)> for + ($(Key<[<$keygen $keygen>], $keygen>),+) + { + type Suffix = $suffix; + + fn partial_key(prefix: ($($prefix),+)) -> Vec { + <($(Key<[<$prefix $prefix>], $prefix>),+)>::final_key(prefix) + } + } + + impl<$($keygen: FullCodec,)+ $( [<$keygen $keygen>]: ReversibleStorageHasher),+> + HasReversibleKeyPrefix<($($prefix),+)> for + ($(Key<[<$keygen $keygen>], $keygen>),+) + { + fn decode_partial_key(key_material: &[u8]) -> Result { + ], $suffix>>::decode_final_key(key_material).map(|k| k.0) + } + } + } + }; +} + +impl HasKeyPrefix<(A,)> + for (Key, Key) +{ + type Suffix = B; + + fn partial_key(prefix: (A,)) -> Vec { + >::final_key(prefix) + } +} + +impl + HasReversibleKeyPrefix<(A,)> for (Key, Key) +{ + fn decode_partial_key(key_material: &[u8]) -> Result { + >::decode_final_key(key_material).map(|k| k.0) + } +} + +impl_key_prefix_for!((A, B, C), (A, B), C); +impl_key_prefix_for!((A, B, C), A, (B, C)); +impl_key_prefix_for!((A, B, C, D), (A, B, C), D); +impl_key_prefix_for!((A, B, C, D), (A, B), (C, D)); +impl_key_prefix_for!((A, B, C, D), A, (B, C, D)); +impl_key_prefix_for!((A, B, C, D, E), (A, B, C, D), E); +impl_key_prefix_for!((A, B, C, D, E), (A, B, C), (D, E)); +impl_key_prefix_for!((A, B, C, D, E), (A, B), (C, D, E)); +impl_key_prefix_for!((A, B, C, D, E), A, (B, C, D, E)); +impl_key_prefix_for!((A, B, C, D, E, F), (A, B, C, D, E), F); +impl_key_prefix_for!((A, B, C, D, E, F), (A, B, C, D), (E, F)); +impl_key_prefix_for!((A, B, C, D, E, F), (A, B, C), (D, E, F)); +impl_key_prefix_for!((A, B, C, D, E, F), (A, B), (C, D, E, F)); +impl_key_prefix_for!((A, B, C, D, E, F), A, (B, C, D, E, F)); +impl_key_prefix_for!((A, B, C, D, E, F, G), (A, B, C, D, E, F), G); +impl_key_prefix_for!((A, B, C, D, E, F, G), (A, B, C, D, E), (F, G)); +impl_key_prefix_for!((A, B, C, D, E, F, G), (A, B, C, D), (E, F, G)); +impl_key_prefix_for!((A, B, C, D, E, F, G), (A, B, C), (D, E, F, G)); +impl_key_prefix_for!((A, B, C, D, E, F, G), (A, B), (C, D, E, F, G)); +impl_key_prefix_for!((A, B, C, D, E, F, G), A, (B, C, D, E, F, G)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H), (A, B, C, D, E, F, G), H); +impl_key_prefix_for!((A, B, C, D, E, F, G, H), (A, B, C, D, E, F), (G, H)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H), (A, B, C, D, E), (F, G, H)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H), (A, B, C, D), (E, F, G, H)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H), (A, B, C), (D, E, F, G, H)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H), (A, B), (C, D, E, F, G, H)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H), A, (B, C, D, E, F, G, H)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H, I), (A, B, C, D, E, F, G, H), I); +impl_key_prefix_for!((A, B, C, D, E, F, G, H, I), (A, B, C, D, E, F, G), (H, I)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H, I), (A, B, C, D, E, F), (G, H, I)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H, I), (A, B, C, D, E), (F, G, H, I)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H, I), (A, B, C, D), (E, F, G, H, I)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H, I), (A, B, C), (D, E, F, G, H, I)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H, I), (A, B), (C, D, E, F, G, H, I)); +impl_key_prefix_for!((A, B, C, D, E, F, G, H, I), A, (B, C, D, E, F, G, H, I)); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + (A, B, C, D, E, F, G, H, I), + J +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + (A, B, C, D, E, F, G, H), + (I, J) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + (A, B, C, D, E, F, G), + (H, I, J) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + (A, B, C, D, E, F), + (G, H, I, J) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + (A, B, C, D, E), + (F, G, H, I, J) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + (A, B, C, D), + (E, F, G, H, I, J) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + (A, B, C), + (D, E, F, G, H, I, J) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + (A, B), + (C, D, E, F, G, H, I, J) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J), + A, + (B, C, D, E, F, G, H, I, J) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B, C, D, E, F, G, H, I, J), + K +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B, C, D, E, F, G, H, I), + (J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B, C, D, E, F, G, H), + (I, J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B, C, D, E, F, G), + (H, I, J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B, C, D, E, F), + (G, H, I, J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B, C, D, E), + (F, G, H, I, J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B, C, D), + (E, F, G, H, I, J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B, C), + (D, E, F, G, H, I, J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + (A, B), + (C, D, E, F, G, H, I, J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K), + A, + (B, C, D, E, F, G, H, I, J, K) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C, D, E, F, G, H, I, J, K), + L +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C, D, E, F, G, H, I, J), + (K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C, D, E, F, G, H, I), + (J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C, D, E, F, G, H), + (I, J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C, D, E, F, G), + (H, I, J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C, D, E, F), + (G, H, I, J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C, D, E), + (F, G, H, I, J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C, D), + (E, F, G, H, I, J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B, C), + (D, E, F, G, H, I, J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + (A, B), + (C, D, E, F, G, H, I, J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L), + A, + (B, C, D, E, F, G, H, I, J, K, L) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D, E, F, G, H, I, J, K, L), + M +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D, E, F, G, H, I, J, K), + (L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D, E, F, G, H, I, J), + (K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D, E, F, G, H, I), + (J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D, E, F, G, H), + (I, J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D, E, F, G), + (H, I, J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D, E, F), + (G, H, I, J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D, E), + (F, G, H, I, J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C, D), + (E, F, G, H, I, J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B, C), + (D, E, F, G, H, I, J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (A, B), + (C, D, E, F, G, H, I, J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M), + A, + (B, C, D, E, F, G, H, I, J, K, L, M) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E, F, G, H, I, J, K, L, M), + N +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E, F, G, H, I, J, K, L), + (M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E, F, G, H, I, J, K), + (L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E, F, G, H, I, J), + (K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E, F, G, H, I), + (J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E, F, G, H), + (I, J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E, F, G), + (H, I, J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E, F), + (G, H, I, J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D, E), + (F, G, H, I, J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C, D), + (E, F, G, H, I, J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B, C), + (D, E, F, G, H, I, J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (A, B), + (C, D, E, F, G, H, I, J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + A, + (B, C, D, E, F, G, H, I, J, K, L, M, N) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + O +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F, G, H, I, J, K, L), + (M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F, G, H, I, J, K), + (L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F, G, H, I, J), + (K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F, G, H, I), + (J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F, G, H), + (I, J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F, G), + (H, I, J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E, F), + (G, H, I, J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D, E), + (F, G, H, I, J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C, D), + (E, F, G, H, I, J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B, C), + (D, E, F, G, H, I, J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (A, B), + (C, D, E, F, G, H, I, J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + A, + (B, C, D, E, F, G, H, I, J, K, L, M, N, O) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + P +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G, H, I, J, K, L), + (M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G, H, I, J, K), + (L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G, H, I, J), + (K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G, H, I), + (J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G, H), + (I, J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F, G), + (H, I, J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E, F), + (G, H, I, J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D, E), + (F, G, H, I, J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C, D), + (E, F, G, H, I, J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B, C), + (D, E, F, G, H, I, J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (A, B), + (C, D, E, F, G, H, I, J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + A, + (B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + Q +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H, I, J, K, L), + (M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H, I, J, K), + (L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H, I, J), + (K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H, I), + (J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G, H), + (I, J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F, G), + (H, I, J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E, F), + (G, H, I, J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D, E), + (F, G, H, I, J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C, D), + (E, F, G, H, I, J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B, C), + (D, E, F, G, H, I, J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + (A, B), + (C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + A, + (B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q), + R +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P), + (Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O), + (P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I, J, K, L, M, N), + (O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I, J, K, L, M), + (N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I, J, K, L), + (M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I, J, K), + (L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I, J), + (K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H, I), + (J, K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G, H), + (I, J, K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F, G), + (H, I, J, K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E, F), + (G, H, I, J, K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D, E), + (F, G, H, I, J, K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C, D), + (E, F, G, H, I, J, K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B, C), + (D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + (A, B), + (C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) +); +impl_key_prefix_for!( + (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R), + A, + (B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) +); diff --git a/frame/support/src/storage/types/mod.rs b/frame/support/src/storage/types/mod.rs index 5bb6684b79259..5b7aa61d37693 100644 --- a/frame/support/src/storage/types/mod.rs +++ b/frame/support/src/storage/types/mod.rs @@ -21,13 +21,20 @@ use codec::FullCodec; use frame_metadata::{DefaultByte, StorageEntryModifier}; -mod value; -mod map; mod double_map; +mod key; +mod map; +mod nmap; +mod value; -pub use value::{StorageValue, StorageValueMetadata}; -pub use map::{StorageMap, StorageMapMetadata}; pub use double_map::{StorageDoubleMap, StorageDoubleMapMetadata}; +pub use key::{ + EncodeLikeTuple, HasKeyPrefix, HasReversibleKeyPrefix, Key, KeyGenerator, + ReversibleKeyGenerator, TupleToEncodedIter, +}; +pub use map::{StorageMap, StorageMapMetadata}; +pub use nmap::{StorageNMap, StorageNMapMetadata}; +pub use value::{StorageValue, StorageValueMetadata}; /// Trait implementing how the storage optional value is converted into the queried type. /// @@ -104,5 +111,5 @@ impl> DefaultByte OnEmpty::get().encode() } } -unsafe impl > Send for OnEmptyGetter {} -unsafe impl > Sync for OnEmptyGetter {} +unsafe impl> Send for OnEmptyGetter {} +unsafe impl> Sync for OnEmptyGetter {} diff --git a/frame/support/src/storage/types/nmap.rs b/frame/support/src/storage/types/nmap.rs new file mode 100755 index 0000000000000..1a2b6d4d55dcc --- /dev/null +++ b/frame/support/src/storage/types/nmap.rs @@ -0,0 +1,995 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storage map type. Implements StorageDoubleMap, StorageIterableDoubleMap, +//! StoragePrefixedDoubleMap traits and their methods directly. + +use crate::{ + storage::{ + types::{ + EncodeLikeTuple, HasKeyPrefix, HasReversibleKeyPrefix, OnEmptyGetter, + OptionQuery, QueryKindTrait, TupleToEncodedIter, + }, + KeyGenerator, PrefixIterator, StorageAppend, StorageDecodeLength, + }, + traits::{GetDefault, StorageInstance}, +}; +use codec::{Decode, Encode, EncodeLike, FullCodec}; +use frame_metadata::{DefaultByteGetter, StorageEntryModifier}; +use sp_std::prelude::*; + +/// A type that allow to store values for an arbitrary number of keys in the form of +/// `(Key, Key, ..., Key)`. +/// +/// Each value is stored at: +/// ```nocompile +/// Twox128(Prefix::pallet_prefix()) +/// ++ Twox128(Prefix::STORAGE_PREFIX) +/// ++ Hasher1(encode(key1)) +/// ++ Hasher2(encode(key2)) +/// ++ ... +/// ++ HasherN(encode(keyN)) +/// ``` +/// +/// # Warning +/// +/// If the keys are not trusted (e.g. can be set by a user), a cryptographic `hasher` +/// such as `blake2_128_concat` must be used for the key hashers. Otherwise, other values +/// in storage can be compromised. +pub struct StorageNMap( + core::marker::PhantomData<(Prefix, Key, Value, QueryKind, OnEmpty)>, +); + +impl crate::storage::generator::StorageNMap + for StorageNMap +where + Prefix: StorageInstance, + Key: super::key::KeyGenerator, + Value: FullCodec, + QueryKind: QueryKindTrait, + OnEmpty: crate::traits::Get + 'static, +{ + type Query = QueryKind::Query; + fn module_prefix() -> &'static [u8] { + Prefix::pallet_prefix().as_bytes() + } + fn storage_prefix() -> &'static [u8] { + Prefix::STORAGE_PREFIX.as_bytes() + } + fn from_optional_value_to_query(v: Option) -> Self::Query { + QueryKind::from_optional_value_to_query(v) + } + fn from_query_to_optional_value(v: Self::Query) -> Option { + QueryKind::from_query_to_optional_value(v) + } +} + +impl crate::storage::StoragePrefixedMap + for StorageNMap +where + Prefix: StorageInstance, + Key: super::key::KeyGenerator, + Value: FullCodec, + QueryKind: QueryKindTrait, + OnEmpty: crate::traits::Get + 'static, +{ + fn module_prefix() -> &'static [u8] { + >::module_prefix() + } + fn storage_prefix() -> &'static [u8] { + >::storage_prefix() + } +} + +impl StorageNMap +where + Prefix: StorageInstance, + Key: super::key::KeyGenerator, + Value: FullCodec, + QueryKind: QueryKindTrait, + OnEmpty: crate::traits::Get + 'static, +{ + /// Get the storage key used to fetch a value corresponding to a specific key. + pub fn hashed_key_for + TupleToEncodedIter>(key: KArg) -> Vec { + >::hashed_key_for(key) + } + + /// Does the value (explicitly) exist in storage? + pub fn contains_key + TupleToEncodedIter>(key: KArg) -> bool { + >::contains_key(key) + } + + /// Load the value associated with the given key from the map. + pub fn get + TupleToEncodedIter>(key: KArg) -> QueryKind::Query { + >::get(key) + } + + /// Try to get the value for the given key from the map. + /// + /// Returns `Ok` if it exists, `Err` if not. + pub fn try_get + TupleToEncodedIter>( + key: KArg, + ) -> Result { + >::try_get(key) + } + + /// Take a value from storage, removing it afterwards. + pub fn take + TupleToEncodedIter>(key: KArg) -> QueryKind::Query { + >::take(key) + } + + /// Swap the values of two key-pairs. + pub fn swap(key1: KArg1, key2: KArg2) + where + KOther: KeyGenerator, + KArg1: EncodeLikeTuple + TupleToEncodedIter, + KArg2: EncodeLikeTuple + TupleToEncodedIter, + { + >::swap::(key1, key2) + } + + /// Store a value to be associated with the given keys from the map. + pub fn insert(key: KArg, val: VArg) + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + VArg: EncodeLike, + { + >::insert(key, val) + } + + /// Remove the value under the given keys. + pub fn remove + TupleToEncodedIter>(key: KArg) { + >::remove(key) + } + + /// Remove all values under the first key. + pub fn remove_prefix(partial_key: KP) + where + Key: HasKeyPrefix, + { + >::remove_prefix(partial_key) + } + + /// Iterate over values that share the first key. + pub fn iter_prefix_values(partial_key: KP) -> PrefixIterator + where + Key: HasKeyPrefix, + { + >::iter_prefix_values(partial_key) + } + + /// Mutate the value under the given keys. + pub fn mutate(key: KArg, f: F) -> R + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut QueryKind::Query) -> R, + { + >::mutate(key, f) + } + + /// Mutate the value under the given keys when the closure returns `Ok`. + pub fn try_mutate(key: KArg, f: F) -> Result + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut QueryKind::Query) -> Result, + { + >::try_mutate(key, f) + } + + /// Mutate the value under the given keys. Deletes the item if mutated to a `None`. + pub fn mutate_exists(key: KArg, f: F) -> R + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Option) -> R, + { + >::mutate_exists(key, f) + } + + /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + pub fn try_mutate_exists(key: KArg, f: F) -> Result + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + F: FnOnce(&mut Option) -> Result, + { + >::try_mutate_exists(key, f) + } + + /// Append the given item to the value in the storage. + /// + /// `Value` is required to implement [`StorageAppend`]. + /// + /// # Warning + /// + /// If the storage item is not encoded properly, the storage will be overwritten + /// and set to `[item]`. Any default value set for the storage item will be ignored + /// on overwrite. + pub fn append(key: KArg, item: EncodeLikeItem) + where + KArg: EncodeLikeTuple + TupleToEncodedIter, + Item: Encode, + EncodeLikeItem: EncodeLike, + Value: StorageAppend, + { + >::append(key, item) + } + + /// Read the length of the storage value without decoding the entire value under the + /// given `key1` and `key2`. + /// + /// `Value` is required to implement [`StorageDecodeLength`]. + /// + /// If the value does not exists or it fails to decode the length, `None` is returned. + /// Otherwise `Some(len)` is returned. + /// + /// # Warning + /// + /// `None` does not mean that `get()` does not return a value. The default value is completly + /// ignored by this function. + pub fn decode_len + TupleToEncodedIter>(key: KArg) -> Option + where + Value: StorageDecodeLength, + { + >::decode_len(key) + } + + /// Migrate an item with the given `key` from defunct `hash_fns` to the current hashers. + /// + /// If the key doesn't exist, then it's a no-op. If it does, then it returns its value. + pub fn migrate_keys(key: KArg, hash_fns: Key::HArg) -> Option + where + KArg: EncodeLikeTuple + TupleToEncodedIter + { + >::migrate_keys::<_>(key, hash_fns) + } + + /// Remove all value of the storage. + pub fn remove_all() { + >::remove_all() + } + + /// Iter over all value of the storage. + /// + /// NOTE: If a value failed to decode becaues storage is corrupted then it is skipped. + pub fn iter_values() -> crate::storage::PrefixIterator { + >::iter_values() + } + + /// Translate the values of all elements by a function `f`, in the map in no particular order. + /// By returning `None` from `f` for an element, you'll remove it from the map. + /// + /// NOTE: If a value fail to decode because storage is corrupted then it is skipped. + /// + /// # Warning + /// + /// This function must be used with care, before being updated the storage still contains the + /// old type, thus other calls (such as `get`) will fail at decoding it. + /// + /// # Usage + /// + /// This would typically be called inside the module implementation of on_runtime_upgrade. + pub fn translate_values Option>(f: F) { + >::translate_values(f) + } +} + +impl StorageNMap +where + Prefix: StorageInstance, + Key: super::key::ReversibleKeyGenerator, + Value: FullCodec, + QueryKind: QueryKindTrait, + OnEmpty: crate::traits::Get + 'static, +{ + /// Enumerate all elements in the map with prefix key `kp` in no particular order. + /// + /// If you add or remove values whose prefix key is `kp` to the map while doing this, you'll get + /// undefined results. + pub fn iter_prefix( + kp: KP, + ) -> crate::storage::PrefixIterator<(>::Suffix, Value)> + where + Key: HasReversibleKeyPrefix, + { + >::iter_prefix(kp) + } + + /// Remove all elements from the map with prefix key `kp` and iterate through them in no + /// particular order. + /// + /// If you add elements with prefix key `k1` to the map while doing this, you'll get undefined + /// results. + pub fn drain_prefix( + kp: KP, + ) -> crate::storage::PrefixIterator<(>::Suffix, Value)> + where + Key: HasReversibleKeyPrefix, + { + >::drain_prefix(kp) + } + + /// Enumerate all elements in the map in no particular order. + /// + /// If you add or remove values to the map while doing this, you'll get undefined results. + pub fn iter() -> crate::storage::PrefixIterator<(Key::Key, Value)> { + >::iter() + } + + /// Remove all elements from the map and iterate through them in no particular order. + /// + /// If you add elements to the map while doing this, you'll get undefined results. + pub fn drain() -> crate::storage::PrefixIterator<(Key::Key, Value)> { + >::drain() + } + + /// Translate the values of all elements by a function `f`, in the map in no particular order. + /// + /// By returning `None` from `f` for an element, you'll remove it from the map. + /// + /// NOTE: If a value fail to decode because storage is corrupted then it is skipped. + pub fn translate Option>(f: F) { + >::translate(f) + } +} + +/// Part of storage metadata for a storage n map. +/// +/// NOTE: Generic hashers is supported. +pub trait StorageNMapMetadata { + const MODIFIER: StorageEntryModifier; + const NAME: &'static str; + const DEFAULT: DefaultByteGetter; + const HASHERS: &'static [frame_metadata::StorageHasher]; +} + +impl StorageNMapMetadata + for StorageNMap +where + Prefix: StorageInstance, + Key: super::key::KeyGenerator, + Value: FullCodec, + QueryKind: QueryKindTrait, + OnEmpty: crate::traits::Get + 'static, +{ + const MODIFIER: StorageEntryModifier = QueryKind::METADATA; + const NAME: &'static str = Prefix::STORAGE_PREFIX; + const DEFAULT: DefaultByteGetter = DefaultByteGetter( + &OnEmptyGetter::(core::marker::PhantomData), + ); + const HASHERS: &'static [frame_metadata::StorageHasher] = Key::HASHER_METADATA; +} + +#[cfg(test)] +mod test { + use super::*; + use crate::hash::*; + use crate::storage::types::{Key, ValueQuery}; + use frame_metadata::StorageEntryModifier; + use sp_io::{hashing::twox_128, TestExternalities}; + + struct Prefix; + impl StorageInstance for Prefix { + fn pallet_prefix() -> &'static str { + "test" + } + const STORAGE_PREFIX: &'static str = "foo"; + } + + struct ADefault; + impl crate::traits::Get for ADefault { + fn get() -> u32 { + 98 + } + } + + #[test] + fn test_1_key() { + type A = StorageNMap, u32, OptionQuery>; + type AValueQueryWithAnOnEmpty = + StorageNMap, u32, ValueQuery, ADefault>; + type B = StorageNMap, u32, ValueQuery>; + type C = StorageNMap, u8, ValueQuery>; + type WithLen = StorageNMap, Vec>; + + TestExternalities::default().execute_with(|| { + let mut k: Vec = vec![]; + k.extend(&twox_128(b"test")); + k.extend(&twox_128(b"foo")); + k.extend(&3u16.blake2_128_concat()); + assert_eq!(A::hashed_key_for((&3,)).to_vec(), k); + + assert_eq!(A::contains_key((3,)), false); + assert_eq!(A::get((3,)), None); + assert_eq!(AValueQueryWithAnOnEmpty::get((3,)), 98); + + A::insert((3,), 10); + assert_eq!(A::contains_key((3,)), true); + assert_eq!(A::get((3,)), Some(10)); + assert_eq!(AValueQueryWithAnOnEmpty::get((3,)), 10); + + A::swap::, _, _>((3,), (2,)); + assert_eq!(A::contains_key((3,)), false); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((3,)), None); + assert_eq!(AValueQueryWithAnOnEmpty::get((3,)), 98); + assert_eq!(A::get((2,)), Some(10)); + assert_eq!(AValueQueryWithAnOnEmpty::get((2,)), 10); + + A::remove((2,)); + assert_eq!(A::contains_key((2,)), false); + assert_eq!(A::get((2,)), None); + + AValueQueryWithAnOnEmpty::mutate((2,), |v| *v = *v * 2); + AValueQueryWithAnOnEmpty::mutate((2,), |v| *v = *v * 2); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((2,)), Some(98 * 4)); + + A::remove((2,)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate((2,), |v| { + *v = *v * 2; + Ok(()) + }); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate((2,), |v| { + *v = *v * 2; + Ok(()) + }); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((2,)), Some(98 * 4)); + + A::remove((2,)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate((2,), |v| { + *v = *v * 2; + Err(()) + }); + assert_eq!(A::contains_key((2,)), false); + + A::remove((2,)); + AValueQueryWithAnOnEmpty::mutate_exists((2,), |v| { + assert!(v.is_none()); + *v = Some(10); + }); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((2,)), Some(10)); + AValueQueryWithAnOnEmpty::mutate_exists((2,), |v| { + *v = Some(v.unwrap() * 10); + }); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((2,)), Some(100)); + + A::remove((2,)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate_exists((2,), |v| { + assert!(v.is_none()); + *v = Some(10); + Ok(()) + }); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((2,)), Some(10)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate_exists((2,), |v| { + *v = Some(v.unwrap() * 10); + Ok(()) + }); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((2,)), Some(100)); + assert_eq!(A::try_get((2,)), Ok(100)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate_exists((2,), |v| { + *v = Some(v.unwrap() * 10); + Err(()) + }); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((2,)), Some(100)); + + A::insert((2,), 10); + assert_eq!(A::take((2,)), Some(10)); + assert_eq!(A::contains_key((2,)), false); + assert_eq!(AValueQueryWithAnOnEmpty::take((2,)), 98); + assert_eq!(A::contains_key((2,)), false); + assert_eq!(A::try_get((2,)), Err(())); + + B::insert((2,), 10); + assert_eq!( + A::migrate_keys((2,), (Box::new(|key| Blake2_256::hash(key).to_vec()),),), + Some(10) + ); + assert_eq!(A::contains_key((2,)), true); + assert_eq!(A::get((2,)), Some(10)); + + A::insert((3,), 10); + A::insert((4,), 10); + A::remove_all(); + assert_eq!(A::contains_key((3,)), false); + assert_eq!(A::contains_key((4,)), false); + + A::insert((3,), 10); + A::insert((4,), 10); + assert_eq!(A::iter_values().collect::>(), vec![10, 10]); + + C::insert((3,), 10); + C::insert((4,), 10); + A::translate_values::(|v| Some((v * 2).into())); + assert_eq!(A::iter().collect::>(), vec![(4, 20), (3, 20)]); + + A::insert((3,), 10); + A::insert((4,), 10); + assert_eq!(A::iter().collect::>(), vec![(4, 10), (3, 10)]); + assert_eq!(A::drain().collect::>(), vec![(4, 10), (3, 10)]); + assert_eq!(A::iter().collect::>(), vec![]); + + C::insert((3,), 10); + C::insert((4,), 10); + A::translate::(|k1, v| Some((k1 as u16 * v as u16).into())); + assert_eq!(A::iter().collect::>(), vec![(4, 40), (3, 30)]); + + assert_eq!(A::MODIFIER, StorageEntryModifier::Optional); + assert_eq!( + AValueQueryWithAnOnEmpty::MODIFIER, + StorageEntryModifier::Default + ); + assert_eq!(A::NAME, "foo"); + assert_eq!( + AValueQueryWithAnOnEmpty::DEFAULT.0.default_byte(), + 98u32.encode() + ); + assert_eq!(A::DEFAULT.0.default_byte(), Option::::None.encode()); + + WithLen::remove_all(); + assert_eq!(WithLen::decode_len((3,)), None); + WithLen::append((0,), 10); + assert_eq!(WithLen::decode_len((0,)), Some(1)); + }); + } + + #[test] + fn test_2_keys() { + type A = StorageNMap< + Prefix, + (Key, Key), + u32, + OptionQuery, + >; + type AValueQueryWithAnOnEmpty = StorageNMap< + Prefix, + (Key, Key), + u32, + ValueQuery, + ADefault, + >; + type B = StorageNMap, Key), u32, ValueQuery>; + type C = StorageNMap< + Prefix, + (Key, Key), + u8, + ValueQuery, + >; + type WithLen = + StorageNMap, Key), Vec>; + + TestExternalities::default().execute_with(|| { + let mut k: Vec = vec![]; + k.extend(&twox_128(b"test")); + k.extend(&twox_128(b"foo")); + k.extend(&3u16.blake2_128_concat()); + k.extend(&30u8.twox_64_concat()); + assert_eq!(A::hashed_key_for((3, 30)).to_vec(), k); + + assert_eq!(A::contains_key((3, 30)), false); + assert_eq!(A::get((3, 30)), None); + assert_eq!(AValueQueryWithAnOnEmpty::get((3, 30)), 98); + + A::insert((3, 30), 10); + assert_eq!(A::contains_key((3, 30)), true); + assert_eq!(A::get((3, 30)), Some(10)); + assert_eq!(AValueQueryWithAnOnEmpty::get((3, 30)), 10); + + A::swap::<(Key, Key), _, _>((3, 30), (2, 20)); + assert_eq!(A::contains_key((3, 30)), false); + assert_eq!(A::contains_key((2, 20)), true); + assert_eq!(A::get((3, 30)), None); + assert_eq!(AValueQueryWithAnOnEmpty::get((3, 30)), 98); + assert_eq!(A::get((2, 20)), Some(10)); + assert_eq!(AValueQueryWithAnOnEmpty::get((2, 20)), 10); + + A::remove((2, 20)); + assert_eq!(A::contains_key((2, 20)), false); + assert_eq!(A::get((2, 20)), None); + + AValueQueryWithAnOnEmpty::mutate((2, 20), |v| *v = *v * 2); + AValueQueryWithAnOnEmpty::mutate((2, 20), |v| *v = *v * 2); + assert_eq!(A::contains_key((2, 20)), true); + assert_eq!(A::get((2, 20)), Some(98 * 4)); + + A::remove((2, 20)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate((2, 20), |v| { + *v = *v * 2; + Err(()) + }); + assert_eq!(A::contains_key((2, 20)), false); + + A::remove((2, 20)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate((2, 20), |v| { + *v = *v * 2; + Err(()) + }); + assert_eq!(A::contains_key((2, 20)), false); + + A::remove((2, 20)); + AValueQueryWithAnOnEmpty::mutate_exists((2, 20), |v| { + assert!(v.is_none()); + *v = Some(10); + }); + assert_eq!(A::contains_key((2, 20)), true); + assert_eq!(A::get((2, 20)), Some(10)); + AValueQueryWithAnOnEmpty::mutate_exists((2, 20), |v| { + *v = Some(v.unwrap() * 10); + }); + assert_eq!(A::contains_key((2, 20)), true); + assert_eq!(A::get((2, 20)), Some(100)); + + A::remove((2, 20)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate_exists((2, 20), |v| { + assert!(v.is_none()); + *v = Some(10); + Ok(()) + }); + assert_eq!(A::contains_key((2, 20)), true); + assert_eq!(A::get((2, 20)), Some(10)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate_exists((2, 20), |v| { + *v = Some(v.unwrap() * 10); + Ok(()) + }); + assert_eq!(A::contains_key((2, 20)), true); + assert_eq!(A::get((2, 20)), Some(100)); + assert_eq!(A::try_get((2, 20)), Ok(100)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate_exists((2, 20), |v| { + *v = Some(v.unwrap() * 10); + Err(()) + }); + assert_eq!(A::contains_key((2, 20)), true); + assert_eq!(A::get((2, 20)), Some(100)); + + A::insert((2, 20), 10); + assert_eq!(A::take((2, 20)), Some(10)); + assert_eq!(A::contains_key((2, 20)), false); + assert_eq!(AValueQueryWithAnOnEmpty::take((2, 20)), 98); + assert_eq!(A::contains_key((2, 20)), false); + assert_eq!(A::try_get((2, 20)), Err(())); + + B::insert((2, 20), 10); + assert_eq!( + A::migrate_keys( + (2, 20), + ( + Box::new(|key| Blake2_256::hash(key).to_vec()), + Box::new(|key| Twox128::hash(key).to_vec()), + ), + ), + Some(10) + ); + assert_eq!(A::contains_key((2, 20)), true); + assert_eq!(A::get((2, 20)), Some(10)); + + A::insert((3, 30), 10); + A::insert((4, 40), 10); + A::remove_all(); + assert_eq!(A::contains_key((3, 30)), false); + assert_eq!(A::contains_key((4, 40)), false); + + A::insert((3, 30), 10); + A::insert((4, 40), 10); + assert_eq!(A::iter_values().collect::>(), vec![10, 10]); + + C::insert((3, 30), 10); + C::insert((4, 40), 10); + A::translate_values::(|v| Some((v * 2).into())); + assert_eq!( + A::iter().collect::>(), + vec![((4, 40), 20), ((3, 30), 20)] + ); + + A::insert((3, 30), 10); + A::insert((4, 40), 10); + assert_eq!( + A::iter().collect::>(), + vec![((4, 40), 10), ((3, 30), 10)] + ); + assert_eq!( + A::drain().collect::>(), + vec![((4, 40), 10), ((3, 30), 10)] + ); + assert_eq!(A::iter().collect::>(), vec![]); + + C::insert((3, 30), 10); + C::insert((4, 40), 10); + A::translate::(|(k1, k2), v| Some((k1 * k2 as u16 * v as u16).into())); + assert_eq!( + A::iter().collect::>(), + vec![((4, 40), 1600), ((3, 30), 900)] + ); + + assert_eq!(A::MODIFIER, StorageEntryModifier::Optional); + assert_eq!( + AValueQueryWithAnOnEmpty::MODIFIER, + StorageEntryModifier::Default + ); + assert_eq!(A::NAME, "foo"); + assert_eq!( + AValueQueryWithAnOnEmpty::DEFAULT.0.default_byte(), + 98u32.encode() + ); + assert_eq!(A::DEFAULT.0.default_byte(), Option::::None.encode()); + + WithLen::remove_all(); + assert_eq!(WithLen::decode_len((3, 30)), None); + WithLen::append((0, 100), 10); + assert_eq!(WithLen::decode_len((0, 100)), Some(1)); + + A::insert((3, 30), 11); + A::insert((3, 31), 12); + A::insert((4, 40), 13); + A::insert((4, 41), 14); + assert_eq!( + A::iter_prefix_values((3,)).collect::>(), + vec![12, 11] + ); + assert_eq!( + A::iter_prefix_values((4,)).collect::>(), + vec![13, 14] + ); + }); + } + + #[test] + fn test_3_keys() { + type A = StorageNMap< + Prefix, + ( + Key, + Key, + Key, + ), + u32, + OptionQuery, + >; + type AValueQueryWithAnOnEmpty = StorageNMap< + Prefix, + ( + Key, + Key, + Key, + ), + u32, + ValueQuery, + ADefault, + >; + type B = StorageNMap< + Prefix, + ( + Key, + Key, + Key, + ), + u32, + ValueQuery, + >; + type C = StorageNMap< + Prefix, + ( + Key, + Key, + Key, + ), + u8, + ValueQuery, + >; + type WithLen = StorageNMap< + Prefix, + ( + Key, + Key, + Key, + ), + Vec, + >; + + TestExternalities::default().execute_with(|| { + let mut k: Vec = vec![]; + k.extend(&twox_128(b"test")); + k.extend(&twox_128(b"foo")); + k.extend(&1u16.blake2_128_concat()); + k.extend(&10u16.blake2_128_concat()); + k.extend(&100u16.twox_64_concat()); + assert_eq!(A::hashed_key_for((1, 10, 100)).to_vec(), k); + + assert_eq!(A::contains_key((1, 10, 100)), false); + assert_eq!(A::get((1, 10, 100)), None); + assert_eq!(AValueQueryWithAnOnEmpty::get((1, 10, 100)), 98); + + A::insert((1, 10, 100), 30); + assert_eq!(A::contains_key((1, 10, 100)), true); + assert_eq!(A::get((1, 10, 100)), Some(30)); + assert_eq!(AValueQueryWithAnOnEmpty::get((1, 10, 100)), 30); + + A::swap::< + ( + Key, + Key, + Key, + ), + _, + _, + >((1, 10, 100), (2, 20, 200)); + assert_eq!(A::contains_key((1, 10, 100)), false); + assert_eq!(A::contains_key((2, 20, 200)), true); + assert_eq!(A::get((1, 10, 100)), None); + assert_eq!(AValueQueryWithAnOnEmpty::get((1, 10, 100)), 98); + assert_eq!(A::get((2, 20, 200)), Some(30)); + assert_eq!(AValueQueryWithAnOnEmpty::get((2, 20, 200)), 30); + + A::remove((2, 20, 200)); + assert_eq!(A::contains_key((2, 20, 200)), false); + assert_eq!(A::get((2, 20, 200)), None); + + AValueQueryWithAnOnEmpty::mutate((2, 20, 200), |v| *v = *v * 2); + AValueQueryWithAnOnEmpty::mutate((2, 20, 200), |v| *v = *v * 2); + assert_eq!(A::contains_key((2, 20, 200)), true); + assert_eq!(A::get((2, 20, 200)), Some(98 * 4)); + + A::remove((2, 20, 200)); + let _: Result<(), ()> = AValueQueryWithAnOnEmpty::try_mutate((2, 20, 200), |v| { + *v = *v * 2; + Err(()) + }); + assert_eq!(A::contains_key((2, 20, 200)), false); + + A::remove((2, 20, 200)); + AValueQueryWithAnOnEmpty::mutate_exists((2, 20, 200), |v| { + assert!(v.is_none()); + *v = Some(10); + }); + assert_eq!(A::contains_key((2, 20, 200)), true); + assert_eq!(A::get((2, 20, 200)), Some(10)); + AValueQueryWithAnOnEmpty::mutate_exists((2, 20, 200), |v| { + *v = Some(v.unwrap() * 10); + }); + assert_eq!(A::contains_key((2, 20, 200)), true); + assert_eq!(A::get((2, 20, 200)), Some(100)); + + A::remove((2, 20, 200)); + let _: Result<(), ()> = + AValueQueryWithAnOnEmpty::try_mutate_exists((2, 20, 200), |v| { + assert!(v.is_none()); + *v = Some(10); + Ok(()) + }); + assert_eq!(A::contains_key((2, 20, 200)), true); + assert_eq!(A::get((2, 20, 200)), Some(10)); + let _: Result<(), ()> = + AValueQueryWithAnOnEmpty::try_mutate_exists((2, 20, 200), |v| { + *v = Some(v.unwrap() * 10); + Ok(()) + }); + assert_eq!(A::contains_key((2, 20, 200)), true); + assert_eq!(A::get((2, 20, 200)), Some(100)); + assert_eq!(A::try_get((2, 20, 200)), Ok(100)); + let _: Result<(), ()> = + AValueQueryWithAnOnEmpty::try_mutate_exists((2, 20, 200), |v| { + *v = Some(v.unwrap() * 10); + Err(()) + }); + assert_eq!(A::contains_key((2, 20, 200)), true); + assert_eq!(A::get((2, 20, 200)), Some(100)); + + A::insert((2, 20, 200), 10); + assert_eq!(A::take((2, 20, 200)), Some(10)); + assert_eq!(A::contains_key((2, 20, 200)), false); + assert_eq!(AValueQueryWithAnOnEmpty::take((2, 20, 200)), 98); + assert_eq!(A::contains_key((2, 20, 200)), false); + assert_eq!(A::try_get((2, 20, 200)), Err(())); + + B::insert((2, 20, 200), 10); + assert_eq!( + A::migrate_keys( + (2, 20, 200), + ( + Box::new(|key| Blake2_256::hash(key).to_vec()), + Box::new(|key| Blake2_256::hash(key).to_vec()), + Box::new(|key| Twox128::hash(key).to_vec()), + ), + ), + Some(10) + ); + assert_eq!(A::contains_key((2, 20, 200)), true); + assert_eq!(A::get((2, 20, 200)), Some(10)); + + A::insert((3, 30, 300), 10); + A::insert((4, 40, 400), 10); + A::remove_all(); + assert_eq!(A::contains_key((3, 30, 300)), false); + assert_eq!(A::contains_key((4, 40, 400)), false); + + A::insert((3, 30, 300), 10); + A::insert((4, 40, 400), 10); + assert_eq!(A::iter_values().collect::>(), vec![10, 10]); + + C::insert((3, 30, 300), 10); + C::insert((4, 40, 400), 10); + A::translate_values::(|v| Some((v * 2).into())); + assert_eq!( + A::iter().collect::>(), + vec![((4, 40, 400), 20), ((3, 30, 300), 20)] + ); + + A::insert((3, 30, 300), 10); + A::insert((4, 40, 400), 10); + assert_eq!( + A::iter().collect::>(), + vec![((4, 40, 400), 10), ((3, 30, 300), 10)] + ); + assert_eq!( + A::drain().collect::>(), + vec![((4, 40, 400), 10), ((3, 30, 300), 10)] + ); + assert_eq!(A::iter().collect::>(), vec![]); + + C::insert((3, 30, 300), 10); + C::insert((4, 40, 400), 10); + A::translate::(|(k1, k2, k3), v| { + Some((k1 * k2 as u16 * v as u16 / k3 as u16).into()) + }); + assert_eq!( + A::iter().collect::>(), + vec![((4, 40, 400), 4), ((3, 30, 300), 3)] + ); + + assert_eq!(A::MODIFIER, StorageEntryModifier::Optional); + assert_eq!( + AValueQueryWithAnOnEmpty::MODIFIER, + StorageEntryModifier::Default + ); + assert_eq!(A::NAME, "foo"); + assert_eq!( + AValueQueryWithAnOnEmpty::DEFAULT.0.default_byte(), + 98u32.encode() + ); + assert_eq!(A::DEFAULT.0.default_byte(), Option::::None.encode()); + + WithLen::remove_all(); + assert_eq!(WithLen::decode_len((3, 30, 300)), None); + WithLen::append((0, 100, 1000), 10); + assert_eq!(WithLen::decode_len((0, 100, 1000)), Some(1)); + + A::insert((3, 30, 300), 11); + A::insert((3, 30, 301), 12); + A::insert((4, 40, 400), 13); + A::insert((4, 40, 401), 14); + assert_eq!( + A::iter_prefix_values((3,)).collect::>(), + vec![11, 12] + ); + assert_eq!( + A::iter_prefix_values((4,)).collect::>(), + vec![14, 13] + ); + assert_eq!( + A::iter_prefix_values((3, 30)).collect::>(), + vec![11, 12] + ); + assert_eq!( + A::iter_prefix_values((4, 40)).collect::>(), + vec![14, 13] + ); + }); + } +} diff --git a/frame/support/test/tests/construct_runtime.rs b/frame/support/test/tests/construct_runtime.rs index a1ec744e42733..76e28a3b152ff 100644 --- a/frame/support/test/tests/construct_runtime.rs +++ b/frame/support/test/tests/construct_runtime.rs @@ -505,7 +505,7 @@ fn test_metadata() { signed_extensions: vec![DecodeDifferent::Encode("UnitSignedExtension")], }, }; - pretty_assertions::assert_eq!(Runtime::metadata().1, RuntimeMetadata::V12(expected_metadata)); + pretty_assertions::assert_eq!(Runtime::metadata().1, RuntimeMetadata::V13(expected_metadata)); } #[test] diff --git a/frame/support/test/tests/pallet.rs b/frame/support/test/tests/pallet.rs index 3bde38c78e2c1..5db5856fd9d97 100644 --- a/frame/support/test/tests/pallet.rs +++ b/frame/support/test/tests/pallet.rs @@ -217,6 +217,21 @@ pub mod pallet { #[pallet::storage] pub type DoubleMap2 = StorageDoubleMap<_, Twox64Concat, u16, Blake2_128Concat, u32, u64>; + #[pallet::storage] + #[pallet::getter(fn nmap)] + pub type NMap = StorageNMap<_, storage::Key, u32>; + + #[pallet::storage] + #[pallet::getter(fn nmap2)] + pub type NMap2 = StorageNMap< + _, + ( + NMapKey, + NMapKey, + ), + u64, + >; + #[pallet::storage] #[pallet::getter(fn conditional_value)] #[cfg(feature = "conditional-storage")] @@ -239,6 +254,18 @@ pub mod pallet { u32, >; + #[cfg(feature = "conditional-storage")] + #[pallet::storage] + #[pallet::getter(fn conditional_nmap)] + pub type ConditionalNMap = StorageNMap< + _, + ( + storage::Key, + storage::Key, + ), + u32, + >; + #[pallet::genesis_config] #[derive(Default)] pub struct GenesisConfig { @@ -578,11 +605,25 @@ fn storage_expand() { assert_eq!(unhashed::get::(&k), Some(3u64)); assert_eq!(&k[..32], &>::final_prefix()); + pallet::NMap::::insert((&1,), &3); + let mut k = [twox_128(b"Example"), twox_128(b"NMap")].concat(); + k.extend(1u8.using_encoded(blake2_128_concat)); + assert_eq!(unhashed::get::(&k), Some(3u32)); + assert_eq!(&k[..32], &>::final_prefix()); + + pallet::NMap2::::insert((&1, &2), &3); + let mut k = [twox_128(b"Example"), twox_128(b"NMap2")].concat(); + k.extend(1u16.using_encoded(twox_64_concat)); + k.extend(2u32.using_encoded(blake2_128_concat)); + assert_eq!(unhashed::get::(&k), Some(3u64)); + assert_eq!(&k[..32], &>::final_prefix()); + #[cfg(feature = "conditional-storage")] { pallet::ConditionalValue::::put(1); pallet::ConditionalMap::::insert(1, 2); pallet::ConditionalDoubleMap::::insert(1, 2, 3); + pallet::ConditionalNMap::::insert((1, 2), 3); } }) } @@ -708,6 +749,36 @@ fn metadata() { default: DecodeDifferent::Decoded(vec![0]), documentation: DecodeDifferent::Decoded(vec![]), }, + StorageEntryMetadata { + name: DecodeDifferent::Decoded("NMap".to_string()), + modifier: StorageEntryModifier::Optional, + ty: StorageEntryType::NMap { + keys: DecodeDifferent::Decoded(vec!["u8".to_string()]), + hashers: DecodeDifferent::Decoded(vec![ + StorageHasher::Blake2_128Concat, + ]), + value: DecodeDifferent::Decoded("u32".to_string()), + }, + default: DecodeDifferent::Decoded(vec![0]), + documentation: DecodeDifferent::Decoded(vec![]), + }, + StorageEntryMetadata { + name: DecodeDifferent::Decoded("NMap2".to_string()), + modifier: StorageEntryModifier::Optional, + ty: StorageEntryType::NMap { + keys: DecodeDifferent::Decoded(vec![ + "u16".to_string(), + "u32".to_string(), + ]), + hashers: DecodeDifferent::Decoded(vec![ + StorageHasher::Twox64Concat, + StorageHasher::Blake2_128Concat, + ]), + value: DecodeDifferent::Decoded("u64".to_string()), + }, + default: DecodeDifferent::Decoded(vec![0]), + documentation: DecodeDifferent::Decoded(vec![]), + }, #[cfg(feature = "conditional-storage")] StorageEntryMetadata { name: DecodeDifferent::Decoded("ConditionalValue".to_string()), modifier: StorageEntryModifier::Optional, @@ -740,6 +811,20 @@ fn metadata() { default: DecodeDifferent::Decoded(vec![0]), documentation: DecodeDifferent::Decoded(vec![]), }, + #[cfg(feature = "conditional-storage")] StorageEntryMetadata { + name: DecodeDifferent::Decoded("ConditionalNMap".to_string()), + modifier: StorageEntryModifier::Optional, + ty: StorageEntryType::NMap { + keys: DecodeDifferent::Decoded(vec!["u8".to_string(), "u16".to_string()]), + hashers: DecodeDifferent::Decoded(vec![ + StorageHasher::Blake2_128Concat, + StorageHasher::Twox64Concat, + ]), + value: DecodeDifferent::Decoded("u32".to_string()), + }, + default: DecodeDifferent::Decoded(vec![0]), + documentation: DecodeDifferent::Decoded(vec![]), + }, ]), })), calls: Some(DecodeDifferent::Decoded(vec![ @@ -857,7 +942,7 @@ fn metadata() { }; let metadata = match Runtime::metadata().1 { - RuntimeMetadata::V12(metadata) => metadata, + RuntimeMetadata::V13(metadata) => metadata, _ => panic!("metadata has been bump, test needs to be updated"), }; diff --git a/frame/support/test/tests/pallet_compatibility.rs b/frame/support/test/tests/pallet_compatibility.rs index a953b19607d90..130014f1e9eb1 100644 --- a/frame/support/test/tests/pallet_compatibility.rs +++ b/frame/support/test/tests/pallet_compatibility.rs @@ -266,7 +266,7 @@ mod test { fn metadata() { let metadata = Runtime::metadata(); let modules = match metadata.1 { - frame_metadata::RuntimeMetadata::V12(frame_metadata::RuntimeMetadataV12 { + frame_metadata::RuntimeMetadata::V13(frame_metadata::RuntimeMetadataV13 { modules: frame_metadata::DecodeDifferent::Encode(m), .. }) => m, diff --git a/frame/support/test/tests/pallet_compatibility_instance.rs b/frame/support/test/tests/pallet_compatibility_instance.rs index 5ce20012c736d..d80d9ba3dff7d 100644 --- a/frame/support/test/tests/pallet_compatibility_instance.rs +++ b/frame/support/test/tests/pallet_compatibility_instance.rs @@ -281,7 +281,7 @@ mod test { fn metadata() { let metadata = Runtime::metadata(); let modules = match metadata.1 { - frame_metadata::RuntimeMetadata::V12(frame_metadata::RuntimeMetadataV12 { + frame_metadata::RuntimeMetadata::V13(frame_metadata::RuntimeMetadataV13 { modules: frame_metadata::DecodeDifferent::Encode(m), .. }) => m, diff --git a/frame/support/test/tests/pallet_instance.rs b/frame/support/test/tests/pallet_instance.rs index f0143b9c40d6c..46ff301f6712d 100644 --- a/frame/support/test/tests/pallet_instance.rs +++ b/frame/support/test/tests/pallet_instance.rs @@ -134,6 +134,21 @@ pub mod pallet { pub type DoubleMap2 = StorageDoubleMap<_, Twox64Concat, u16, Blake2_128Concat, u32, u64>; + #[pallet::storage] + #[pallet::getter(fn nmap)] + pub type NMap = StorageNMap<_, storage::Key, u32>; + + #[pallet::storage] + #[pallet::getter(fn nmap2)] + pub type NMap2 = StorageNMap< + _, + ( + storage::Key, + storage::Key, + ), + u64, + >; + #[pallet::genesis_config] #[derive(Default)] pub struct GenesisConfig { @@ -447,6 +462,19 @@ fn storage_expand() { k.extend(2u32.using_encoded(blake2_128_concat)); assert_eq!(unhashed::get::(&k), Some(3u64)); assert_eq!(&k[..32], &>::final_prefix()); + + >::insert((&1,), &3); + let mut k = [twox_128(b"Example"), twox_128(b"NMap")].concat(); + k.extend(1u8.using_encoded(blake2_128_concat)); + assert_eq!(unhashed::get::(&k), Some(3u32)); + assert_eq!(&k[..32], &>::final_prefix()); + + >::insert((&1, &2), &3); + let mut k = [twox_128(b"Example"), twox_128(b"NMap2")].concat(); + k.extend(1u16.using_encoded(twox_64_concat)); + k.extend(2u32.using_encoded(blake2_128_concat)); + assert_eq!(unhashed::get::(&k), Some(3u64)); + assert_eq!(&k[..32], &>::final_prefix()); }); TestExternalities::default().execute_with(|| { @@ -479,6 +507,19 @@ fn storage_expand() { k.extend(2u32.using_encoded(blake2_128_concat)); assert_eq!(unhashed::get::(&k), Some(3u64)); assert_eq!(&k[..32], &>::final_prefix()); + + >::insert((&1,), &3); + let mut k = [twox_128(b"Instance1Example"), twox_128(b"NMap")].concat(); + k.extend(1u8.using_encoded(blake2_128_concat)); + assert_eq!(unhashed::get::(&k), Some(3u32)); + assert_eq!(&k[..32], &>::final_prefix()); + + >::insert((&1, &2), &3); + let mut k = [twox_128(b"Instance1Example"), twox_128(b"NMap2")].concat(); + k.extend(1u16.using_encoded(twox_64_concat)); + k.extend(2u32.using_encoded(blake2_128_concat)); + assert_eq!(unhashed::get::(&k), Some(3u64)); + assert_eq!(&k[..32], &>::final_prefix()); }); } @@ -617,6 +658,36 @@ fn metadata() { default: DecodeDifferent::Decoded(vec![0]), documentation: DecodeDifferent::Decoded(vec![]), }, + StorageEntryMetadata { + name: DecodeDifferent::Decoded("NMap".to_string()), + modifier: StorageEntryModifier::Optional, + ty: StorageEntryType::NMap { + keys: DecodeDifferent::Decoded(vec!["u8".to_string()]), + hashers: DecodeDifferent::Decoded(vec![ + StorageHasher::Blake2_128Concat, + ]), + value: DecodeDifferent::Decoded("u32".to_string()), + }, + default: DecodeDifferent::Decoded(vec![0]), + documentation: DecodeDifferent::Decoded(vec![]), + }, + StorageEntryMetadata { + name: DecodeDifferent::Decoded("NMap2".to_string()), + modifier: StorageEntryModifier::Optional, + ty: StorageEntryType::NMap { + keys: DecodeDifferent::Decoded(vec![ + "u16".to_string(), + "u32".to_string(), + ]), + hashers: DecodeDifferent::Decoded(vec![ + StorageHasher::Twox64Concat, + StorageHasher::Blake2_128Concat, + ]), + value: DecodeDifferent::Decoded("u64".to_string()), + }, + default: DecodeDifferent::Decoded(vec![0]), + documentation: DecodeDifferent::Decoded(vec![]), + }, ]), })), calls: Some(DecodeDifferent::Decoded(vec![ @@ -696,7 +767,7 @@ fn metadata() { let metadata = match Runtime::metadata().1 { - RuntimeMetadata::V12(metadata) => metadata, + RuntimeMetadata::V13(metadata) => metadata, _ => panic!("metadata has been bump, test needs to be updated"), }; diff --git a/frame/support/test/tests/pallet_ui/storage_not_storage_type.stderr b/frame/support/test/tests/pallet_ui/storage_not_storage_type.stderr index ec4bde22ac7a8..73fda60942475 100644 --- a/frame/support/test/tests/pallet_ui/storage_not_storage_type.stderr +++ b/frame/support/test/tests/pallet_ui/storage_not_storage_type.stderr @@ -1,4 +1,4 @@ -error: Invalid pallet::storage, expected ident: `StorageValue` or `StorageMap` or `StorageDoubleMap` in order to expand metadata, found `u8` +error: Invalid pallet::storage, expected ident: `StorageValue` or `StorageMap` or `StorageDoubleMap` or `StorageNMap` in order to expand metadata, found `u8` --> $DIR/storage_not_storage_type.rs:19:16 | 19 | type Foo = u8;