Metadata V15: Expose pallet documentation (paritytech#13452)

* frame/proc: Helpers to parse pallet documentation attributes

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* frame/proc: Expand pallet with runtime metadata documentation

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* frame/dispatch: Implement doc function getter for dispatch

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* frame/tests: Check exposed runtime metadata documentation

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* frame/tests: Add UI tests for `pallet_doc` attribute

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* frame/proc: Document pallet_doc attribute

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* frame/support: Use `derive_syn_parse`

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Update frame/support/procedural/src/lib.rs

Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>

* frame/support: Improve documentation

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

---------

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>
Co-authored-by: parity-processbot <>
Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>

frame/support/procedural/src/lib.rs

@@ -481,6 +481,57 @@ pub fn construct_runtime(input: TokenStream) -> TokenStream {
 /// </pre></div>
 ///
 /// See `frame_support::pallet` docs for more info.
+///
+/// ## Runtime Metadata Documentation
+///
+/// The documentation added to this pallet is included in the runtime metadata.
+///
+/// The documentation can be defined in the following ways:
+///
+/// ```ignore
+/// #[pallet::pallet]
+/// /// Documentation for pallet 1
+/// #[doc = "Documentation for pallet 2"]
+/// #[doc = include_str!("../README.md")]
+/// #[pallet_doc("../doc1.md")]
+/// #[pallet_doc("../doc2.md")]
+/// pub struct Pallet<T>(_);
+/// ```
+///
+/// The runtime metadata for this pallet contains the following
+/// - " Documentation for pallet 1" (captured from `///`)
+/// - "Documentation for pallet 2" (captured from `#[doc]`)
+/// - content of ../README.md (captured from `#[doc]` with `include_str!`)
+/// - content of "../doc1.md" (captured from `pallet_doc`)
+/// - content of "../doc2.md" (captured from `pallet_doc`)
+///
+/// ### `doc` attribute
+///
+/// The value of the `doc` attribute is included in the runtime metadata, as well as
+/// expanded on the pallet module. The previous example is expanded to:
+///
+/// ```ignore
+/// /// Documentation for pallet 1
+/// /// Documentation for pallet 2
+/// /// Content of README.md
+/// pub struct Pallet<T>(_);
+/// ```
+///
+/// If you want to specify the file from which the documentation is loaded, you can use the
+/// `include_str` macro. However, if you only want the documentation to be included in the
+/// runtime metadata, use the `pallet_doc` attribute.
+///
+/// ### `pallet_doc` attribute
+///
+/// Unlike the `doc` attribute, the documentation provided to the `pallet_doc` attribute is
+/// not inserted on the module.
+///
+/// The `pallet_doc` attribute can only be provided with one argument,
+/// which is the file path that holds the documentation to be added to the metadata.
+///
+/// This approach is beneficial when you use the `include_str` macro at the beginning of the file
+/// and want that documentation to extend to the runtime metadata, without reiterating the
+/// documentation on the module itself.
 #[proc_macro_attribute]
 pub fn pallet(attr: TokenStream, item: TokenStream) -> TokenStream {
  pallet::pallet(attr, item)

frame/support/procedural/src/pallet/expand/documentation.rs

@@ -0,0 +1,221 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::pallet::Def;
+use derive_syn_parse::Parse;
+use proc_macro2::TokenStream;
+use quote::ToTokens;
+use syn::{
+ parse::{self, Parse, ParseStream},
+ spanned::Spanned,
+ Attribute, Lit,
+};
+
+const DOC: &'static str = "doc";
+const PALLET_DOC: &'static str = "pallet_doc";
+
+mod keywords {
+ syn::custom_keyword!(include_str);
+}
+
+/// Get the documentation file path from the `pallet_doc` attribute.
+///
+/// Supported format:
+/// `#[pallet_doc(PATH)]`: The path of the file from which the documentation is loaded
+fn parse_pallet_doc_value(attr: &Attribute) -> syn::Result<DocMetaValue> {
+ let span = attr.span();
+
+ let meta = attr.parse_meta()?;
+ let syn::Meta::List(metalist) = meta else {
+ let msg = "The `pallet_doc` attribute must receive arguments as a list. Supported format: `pallet_doc(PATH)`";
+ return Err(syn::Error::new(span, msg))
+ };
+
+ let paths: Vec<_> = metalist
+ .nested
+ .into_iter()
+ .map(|nested| {
+ let syn::NestedMeta::Lit(lit) = nested else {
+ let msg = "The `pallet_doc` received an unsupported argument. Supported format: `pallet_doc(PATH)`";
+ return Err(syn::Error::new(span, msg))
+ };
+
+ Ok(lit)
+ })
+ .collect::<syn::Result<_>>()?;
+
+ if paths.len() != 1 {
+ let msg = "The `pallet_doc` attribute must receive only one argument. Supported format: `pallet_doc(PATH)`";
+ return Err(syn::Error::new(span, msg))
+ }
+
+ Ok(DocMetaValue::Path(paths[0].clone()))
+}
+
+/// Get the value from the `doc` comment attribute:
+///
+/// Supported formats:
+/// - `#[doc = "A doc string"]`: Documentation as a string literal
+/// - `#[doc = include_str!(PATH)]`: Documentation obtained from a path
+fn parse_doc_value(attr: &Attribute) -> Option<DocMetaValue> {
+ let Some(ident) = attr.path.get_ident() else {
+ return None
+ };
+ if ident != DOC {
+ return None
+ }
+
+ let parser = |input: ParseStream| DocParser::parse(input);
+ let result = parse::Parser::parse2(parser, attr.tokens.clone()).ok()?;
+
+ if let Some(lit) = result.lit {
+ Some(DocMetaValue::Lit(lit))
+ } else if let Some(include_doc) = result.include_doc {
+ Some(DocMetaValue::Path(include_doc.lit))
+ } else {
+ None
+ }
+}
+
+/// Parse the include_str attribute.
+#[derive(Debug, Parse)]
+struct IncludeDocParser {
+ _include_str: keywords::include_str,
+ _eq_token: syn::token::Bang,
+ #[paren]
+ _paren: syn::token::Paren,
+ #[inside(_paren)]
+ lit: Lit,
+}
+
+/// Parse the doc literal.
+#[derive(Debug, Parse)]
+struct DocParser {
+ _eq_token: syn::token::Eq,
+ #[peek(Lit)]
+ lit: Option<Lit>,
+ #[parse_if(lit.is_none())]
+ include_doc: Option<IncludeDocParser>,
+}
+
+/// Supported documentation tokens.
+#[derive(Debug)]
+enum DocMetaValue {
+ /// Documentation with string literals.
+ ///
+ /// `#[doc = "Lit"]`
+ Lit(Lit),
+ /// Documentation with `include_str!` macro.
+ ///
+ /// The string literal represents the file `PATH`.
+ ///
+ /// `#[doc = include_str!(PATH)]`
+ Path(Lit),
+}
+
+impl ToTokens for DocMetaValue {
+ fn to_tokens(&self, tokens: &mut TokenStream) {
+ match self {
+ DocMetaValue::Lit(lit) => lit.to_tokens(tokens),
+ DocMetaValue::Path(path_lit) => {
+ let decl = quote::quote!(include_str!(#path_lit));
+ tokens.extend(decl)
+ },
+ }
+ }
+}
+
+/// Extract the documentation from the given pallet definition
+/// to include in the runtime metadata.
+///
+/// Implement a `pallet_documentation_metadata` function to fetch the
+/// documentation that is included in the metadata.
+///
+/// The documentation is placed at the top of the module similar to:
+///
+/// ```ignore
+/// #[pallet]
+/// /// Documentation for pallet
+/// #[doc = "Documentation for pallet"]
+/// #[doc = include_str!("../README.md")]
+/// #[pallet_doc("../documentation1.md")]
+/// #[pallet_doc("../documentation2.md")]
+/// pub mod pallet {}
+/// ```
+///
+/// # pallet_doc
+///
+/// The `pallet_doc` attribute can only be provided with one argument,
+/// which is the file path that holds the documentation to be added to the metadata.
+///
+/// Unlike the `doc` attribute, the documentation provided to the `proc_macro` attribute is
+/// not inserted at the beginning of the module.
+pub fn expand_documentation(def: &mut Def) -> proc_macro2::TokenStream {
+ let frame_support = &def.frame_support;
+ let type_impl_gen = &def.type_impl_generics(proc_macro2::Span::call_site());
+ let type_use_gen = &def.type_use_generics(proc_macro2::Span::call_site());
+ let pallet_ident = &def.pallet_struct.pallet;
+ let where_clauses = &def.config.where_clause;
+
+ // TODO: Use [drain_filter](https://doc.rust-lang.org/std/vec/struct.Vec.html#method.drain_filter) when it is stable.
+
+ // The `pallet_doc` attributes are excluded from the generation of the pallet,
+ // but they are included in the runtime metadata.
+ let mut pallet_docs = Vec::with_capacity(def.item.attrs.len());
+ let mut index = 0;
+ while index < def.item.attrs.len() {
+ let attr = &def.item.attrs[index];
+ if let Some(ident) = attr.path.get_ident() {
+ if ident == PALLET_DOC {
+ let elem = def.item.attrs.remove(index);
+ pallet_docs.push(elem);
+ // Do not increment the index, we have just removed the
+ // element from the attributes.
+ continue
+ }
+ }
+
+ index += 1;
+ }
+
+ // Capture the `#[doc = include_str!("../README.md")]` and `#[doc = "Documentation"]`.
+ let mut docs: Vec<_> = def.item.attrs.iter().filter_map(parse_doc_value).collect();
+
+ // Capture the `#[pallet_doc("../README.md")]`.
+ let pallet_docs: Vec<_> = match pallet_docs
+ .into_iter()
+ .map(|attr| parse_pallet_doc_value(&attr))
+ .collect::<syn::Result<_>>()
+ {
+ Ok(docs) => docs,
+ Err(err) => return err.into_compile_error(),
+ };
+
+ docs.extend(pallet_docs);
+
+ quote::quote!(
+ impl<#type_impl_gen> #pallet_ident<#type_use_gen> #where_clauses{
+
+ #[doc(hidden)]
+ pub fn pallet_documentation_metadata()
+ -> #frame_support::sp_std::vec::Vec<&'static str>
+ {
+ #frame_support::sp_std::vec![ #( #docs ),* ]
+ }
+ }
+ )
+}

frame/support/procedural/src/pallet/expand/mod.rs

@@ -18,6 +18,7 @@
 mod call;
 mod config;
 mod constants;
+mod documentation;
 mod error;
 mod event;
 mod genesis_build;
@@ -52,6 +53,8 @@ pub fn merge_where_clauses(clauses: &[&Option<syn::WhereClause>]) -> Option<syn:
 /// * create some new types,
 /// * impl stuff on them.
 pub fn expand(mut def: Def) -> proc_macro2::TokenStream {
+ // Remove the `pallet_doc` attribute first.
+ let metadata_docs = documentation::expand_documentation(&mut def);
  let constants = constants::expand_constants(&mut def);
  let pallet_struct = pallet_struct::expand_pallet_struct(&mut def);
  let config = config::expand_config(&mut def);
@@ -82,6 +85,7 @@ pub fn expand(mut def: Def) -> proc_macro2::TokenStream {
  }
 
  let new_items = quote::quote!(
+ #metadata_docs
  #constants
  #pallet_struct
  #config

frame/support/src/dispatch.rs

@@ -2948,6 +2948,10 @@ macro_rules! decl_module {
  { $( $other_where_bounds )* }
  $( $error_type )*
  }
+ $crate::__impl_docs_metadata! {
+ $mod_type<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?>
+ { $( $other_where_bounds )* }
+ }
  $crate::__impl_module_constants_metadata ! {
  $mod_type<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?>
  { $( $other_where_bounds )* }
@@ -3018,6 +3022,26 @@ macro_rules! __impl_error_metadata {
  };
 }
 
+/// Implement metadata for pallet documentation.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! __impl_docs_metadata {
+ (
+ $mod_type:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>
+ { $( $other_where_bounds:tt )* }
+ ) => {
+ impl<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?> $mod_type<$trait_instance $(, $instance)?>
+ where $( $other_where_bounds )*
+ {
+ #[doc(hidden)]
+ #[allow(dead_code)]
+ pub fn pallet_documentation_metadata() -> $crate::sp_std::vec::Vec<&'static str> {
+ $crate::sp_std::vec![]
+ }
+ }
+ };
+}
+
 /// Implement metadata for module constants.
 #[macro_export]
 #[doc(hidden)]

frame/support/test/tests/pallet.rs

@@ -101,6 +101,10 @@ impl SomeAssociation2 for u64 {
 }
 
 #[frame_support::pallet]
+/// Pallet documentation
+// Comments should not be included in the pallet documentation
+#[pallet_doc("../../README.md")]
+#[doc = include_str!("../../README.md")]
 pub mod pallet {
  use super::*;
  use frame_support::pallet_prelude::*;
@@ -1589,6 +1593,14 @@ fn metadata() {
  pretty_assertions::assert_eq!(actual_metadata.pallets, expected_metadata.pallets);
 }
 
+#[test]
+fn test_pallet_runtime_docs() {
+ let docs = crate::pallet::Pallet::<Runtime>::pallet_documentation_metadata();
+ let readme = "Support code for the runtime.\n\nLicense: Apache-2.0";
+ let expected = vec![" Pallet documentation", readme, readme];
+ assert_eq!(docs, expected);
+}
+
 #[test]
 fn test_pallet_info_access() {
  assert_eq!(<System as frame_support::traits::PalletInfoAccess>::name(), "System");

frame/support/test/tests/pallet_ui/pallet_doc_arg_non_path.rs

@@ -0,0 +1,16 @@
+#[frame_support::pallet]
+// Must receive a string literal pointing to a path
+#[pallet_doc(X)]
+mod pallet {
+ #[pallet::config]
+ pub trait Config: frame_system::Config
+ where
+ <Self as frame_system::Config>::Index: From<u128>,
+ {
+ }
+
+ #[pallet::pallet]
+ pub struct Pallet<T>(core::marker::PhantomData<T>);
+}
+
+fn main() {}

frame/support/test/tests/pallet_ui/pallet_doc_arg_non_path.stderr

@@ -0,0 +1,5 @@
+error: The `pallet_doc` received an unsupported argument. Supported format: `pallet_doc(PATH)`
+ --> tests/pallet_ui/pallet_doc_arg_non_path.rs:3:1
+ |
+3 | #[pallet_doc(X)]
+ | ^