From e37b75ea6dee9f73a3a1406392c389b2118bc43b Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sun, 9 Aug 2020 00:52:09 +0300 Subject: [PATCH 1/2] rustc_span: Remove `Symbol::is_doc_keyword` The check in rustdoc using it is artificial and not helpful --- src/librustc_span/symbol.rs | 5 ----- src/librustdoc/clean/mod.rs | 7 ++----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/librustc_span/symbol.rs b/src/librustc_span/symbol.rs index 7843c04f25596..a8c23884c896a 100644 --- a/src/librustc_span/symbol.rs +++ b/src/librustc_span/symbol.rs @@ -1554,11 +1554,6 @@ impl Symbol { self == kw::Try } - /// Used for sanity checking rustdoc keyword sections. - pub fn is_doc_keyword(self) -> bool { - self <= kw::Union - } - /// A keyword or reserved identifier that can be used as a path segment. pub fn is_path_segment_keyword(self) -> bool { self == kw::Super diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index 3ad357e583cf1..309f01be7edc9 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -169,11 +169,8 @@ impl Clean for CrateNum { for attr in attrs.lists(sym::doc) { if let Some(v) = attr.value_str() { if attr.has_name(sym::keyword) { - if v.is_doc_keyword() { - keyword = Some(v.to_string()); - break; - } - // FIXME: should warn on unknown keywords? + keyword = Some(v.to_string()); + break; } } } From f02b03787d9ed416355b0fb7f05deb753b0e3690 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sun, 9 Aug 2020 00:53:07 +0300 Subject: [PATCH 2/2] rustc_span: Generate keyword classification functions automatically --- Cargo.lock | 1 + src/librustc_macros/Cargo.toml | 1 + src/librustc_macros/src/symbols.rs | 81 +++++++++++++++++++++++------- src/librustc_span/symbol.rs | 25 +++++---- 4 files changed, 77 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1fa3bef07c7e..57d179bf248f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3625,6 +3625,7 @@ dependencies = [ name = "rustc_macros" version = "0.1.0" dependencies = [ + "indexmap", "proc-macro2", "quote", "syn", diff --git a/src/librustc_macros/Cargo.toml b/src/librustc_macros/Cargo.toml index 73eb0dd56d772..a264a887c593d 100644 --- a/src/librustc_macros/Cargo.toml +++ b/src/librustc_macros/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" proc-macro = true [dependencies] +indexmap = "1" synstructure = "0.12.1" syn = { version = "1", features = ["full"] } proc-macro2 = "1" diff --git a/src/librustc_macros/src/symbols.rs b/src/librustc_macros/src/symbols.rs index 352665f0ab199..791a19659117e 100644 --- a/src/librustc_macros/src/symbols.rs +++ b/src/librustc_macros/src/symbols.rs @@ -1,5 +1,8 @@ +use indexmap::IndexMap; use proc_macro::TokenStream; +use proc_macro2::TokenTree; use quote::quote; +use std::collections::hash_map::RandomState; use std::collections::HashSet; use syn::parse::{Parse, ParseStream, Result}; use syn::{braced, parse_macro_input, Ident, LitStr, Token}; @@ -44,22 +47,43 @@ impl Parse for Symbol { } } -/// A type used to greedily parse another type until the input is empty. -struct List(Vec); +// Map from an optional keyword class to the list of keywords in it. +// FIXME: the indexmap crate thinks `has_std` is false when building `rustc_macros`, +// so we have to provide the hasher manually. +struct Keywords(IndexMap, Vec, RandomState>); -impl Parse for List { +impl Parse for Keywords { + fn parse(input: ParseStream<'_>) -> Result { + let mut classes = IndexMap::<_, Vec<_>, _>::with_hasher(Default::default()); + let mut current_class = None; + while !input.is_empty() { + if input.peek(Token![fn]) { + input.parse::()?; + current_class = Some(input.parse::()?); + input.parse::()?; + } else { + classes.entry(current_class.clone()).or_default().push(input.parse()?); + } + } + Ok(Keywords(classes)) + } +} + +struct Symbols(Vec); + +impl Parse for Symbols { fn parse(input: ParseStream<'_>) -> Result { let mut list = Vec::new(); while !input.is_empty() { list.push(input.parse()?); } - Ok(List(list)) + Ok(Symbols(list)) } } struct Input { - keywords: List, - symbols: List, + keywords: Keywords, + symbols: Symbols, } impl Parse for Input { @@ -85,6 +109,7 @@ pub fn symbols(input: TokenStream) -> TokenStream { let mut symbols_stream = quote! {}; let mut digits_stream = quote! {}; let mut prefill_stream = quote! {}; + let mut keyword_class_stream = quote! {}; let mut counter = 0u32; let mut keys = HashSet::::new(); let mut prev_key: Option = None; @@ -106,18 +131,34 @@ pub fn symbols(input: TokenStream) -> TokenStream { }; // Generate the listed keywords. - for keyword in &input.keywords.0 { - let name = &keyword.name; - let value = &keyword.value; - check_dup(&value.value(), &mut errors); - prefill_stream.extend(quote! { - #value, - }); - keyword_stream.extend(quote! { - #[allow(non_upper_case_globals)] - pub const #name: Symbol = Symbol::new(#counter); - }); - counter += 1; + for (class, keywords) in &input.keywords.0 { + let mut class_stream = quote! {}; + for keyword in keywords { + let name = &keyword.name; + let value = &keyword.value; + check_dup(&value.value(), &mut errors); + prefill_stream.extend(quote! { + #value, + }); + keyword_stream.extend(quote! { + #[allow(non_upper_case_globals)] + pub const #name: Symbol = Symbol::new(#counter); + }); + class_stream.extend(quote! { + | kw::#name + }); + counter += 1; + } + if let Some(class) = class { + keyword_class_stream.extend(quote! { + fn #class(self) -> bool { + match self { + #class_stream => true, + _ => false + } + } + }); + } } // Generate the listed symbols. @@ -185,6 +226,10 @@ pub fn symbols(input: TokenStream) -> TokenStream { ]) } } + + impl Symbol { + #keyword_class_stream + } }); // To see the generated code generated, uncomment this line, recompile, and diff --git a/src/librustc_span/symbol.rs b/src/librustc_span/symbol.rs index a8c23884c896a..c3e328871da45 100644 --- a/src/librustc_span/symbol.rs +++ b/src/librustc_span/symbol.rs @@ -25,12 +25,14 @@ symbols! { Keywords { // Special reserved identifiers used internally for elided lifetimes, // unnamed method parameters, crate root module, error recovery etc. + fn is_special: Invalid: "", PathRoot: "{{root}}", DollarCrate: "$crate", Underscore: "_", - // Keywords that are used in stable Rust. + // Keywords that are used in stable Rust on all editions. + fn is_used_keyword_20xx: As: "as", Break: "break", Const: "const", @@ -67,7 +69,8 @@ symbols! { Where: "where", While: "while", - // Keywords that are used in unstable Rust or reserved for future use. + // Keywords that are used in unstable Rust or reserved for future use on all editions. + fn is_unused_keyword_20xx: Abstract: "abstract", Become: "become", Box: "box", @@ -82,18 +85,22 @@ symbols! { Yield: "yield", // Edition-specific keywords that are used in stable Rust. + fn is_used_keyword_2018: Async: "async", // >= 2018 Edition only Await: "await", // >= 2018 Edition only Dyn: "dyn", // >= 2018 Edition only // Edition-specific keywords that are used in unstable Rust or reserved for future use. + fn is_unused_keyword_2018: Try: "try", // >= 2018 Edition only // Special lifetime names + fn is_special_lifetime: UnderscoreLifetime: "'_", StaticLifetime: "'static", // Weak keywords, have special meaning only in specific contexts. + fn is_weak_keyword: Auto: "auto", Catch: "catch", Default: "default", @@ -1546,14 +1553,6 @@ pub mod sym { } impl Symbol { - fn is_used_keyword_2018(self) -> bool { - self >= kw::Async && self <= kw::Dyn - } - - fn is_unused_keyword_2018(self) -> bool { - self == kw::Try - } - /// A keyword or reserved identifier that can be used as a path segment. pub fn is_path_segment_keyword(self) -> bool { self == kw::Super @@ -1579,20 +1578,20 @@ impl Ident { // Returns `true` for reserved identifiers used internally for elided lifetimes, // unnamed method parameters, crate root module, error recovery etc. pub fn is_special(self) -> bool { - self.name <= kw::Underscore + self.name.is_special() } /// Returns `true` if the token is a keyword used in the language. pub fn is_used_keyword(self) -> bool { // Note: `span.edition()` is relatively expensive, don't call it unless necessary. - self.name >= kw::As && self.name <= kw::While + self.name.is_used_keyword_20xx() || self.name.is_used_keyword_2018() && self.span.rust_2018() } /// Returns `true` if the token is a keyword reserved for possible future use. pub fn is_unused_keyword(self) -> bool { // Note: `span.edition()` is relatively expensive, don't call it unless necessary. - self.name >= kw::Abstract && self.name <= kw::Yield + self.name.is_unused_keyword_20xx() || self.name.is_unused_keyword_2018() && self.span.rust_2018() }