Inlined symbols

Cargo.lock

@@ -3688,6 +3688,7 @@ dependencies = [
 name = "rustc_macros"
 version = "0.1.0"
 dependencies = [
+ "indexmap",
  "proc-macro2",
  "quote",
  "syn",

compiler/rustc_macros/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2018"
 proc-macro = true
 
 [dependencies]
+indexmap = "1"
 synstructure = "0.12.1"
 syn = { version = "1", features = ["full"] }
 proc-macro2 = "1"

compiler/rustc_macros/src/symbols.rs

@@ -1,5 +1,8 @@
+use indexmap::IndexMap;
 use proc_macro::TokenStream;
+use proc_macro2::TokenTree;
 use quote::quote;
+use std::collections::hash_map::RandomState;
 use std::collections::HashSet;
 use syn::parse::{Parse, ParseStream, Result};
 use syn::{braced, parse_macro_input, Ident, LitStr, Token};
@@ -44,22 +47,43 @@ impl Parse for Symbol {
  }
 }
 
-/// A type used to greedily parse another type until the input is empty.
-struct List<T>(Vec<T>);
+// Map from an optional keyword class to the list of keywords in it.
+// FIXME: the indexmap crate thinks `has_std` is false when building `rustc_macros`,
+// so we have to provide the hasher manually.
+struct Keywords(IndexMap<Option<Ident>, Vec<Keyword>, RandomState>);
 
-impl<T: Parse> Parse for List<T> {
+impl Parse for Keywords {
+ fn parse(input: ParseStream<'_>) -> Result<Self> {
+ let mut classes = IndexMap::<_, Vec<_>, _>::with_hasher(Default::default());
+ let mut current_class = None;
+ while !input.is_empty() {
+ if input.peek(Token![fn]) {
+ input.parse::<TokenTree>()?;
+ current_class = Some(input.parse::<Ident>()?);
+ input.parse::<Token![:]>()?;
+ } else {
+ classes.entry(current_class.clone()).or_default().push(input.parse()?);
+ }
+ }
+ Ok(Keywords(classes))
+ }
+}
+
+struct Symbols(Vec<Symbol>);
+
+impl Parse for Symbols {
  fn parse(input: ParseStream<'_>) -> Result<Self> {
  let mut list = Vec::new();
  while !input.is_empty() {
  list.push(input.parse()?);
  }
- Ok(List(list))
+ Ok(Symbols(list))
  }
 }
 
 struct Input {
- keywords: List<Keyword>,
- symbols: List<Symbol>,
+ keywords: Keywords,
+ symbols: Symbols,
 }
 
 impl Parse for Input {
@@ -78,14 +102,25 @@ impl Parse for Input {
  }
 }
 
+/// WARNING: this function must behave equivalently to
+/// `Symbol::try_new_inlined()`. It does, modulo the fact that it accepts fewer
+/// inputs, panicking on any string containing non-ASCII or NUL bytes. This is
+/// fine because static symbols never contain such bytes. Once those bytes are
+/// excluded, it reduces to a mere length check.
+fn is_inlinable(s: &str) -> bool {
+ assert!(s.as_bytes().iter().all(|&b| 0 < b && b < 0x80));
+ s.len() <= 4
+}
+
 pub fn symbols(input: TokenStream) -> TokenStream {
  let input = parse_macro_input!(input as Input);
 
  let mut keyword_stream = quote! {};
  let mut symbols_stream = quote! {};
  let mut digits_stream = quote! {};
- let mut prefill_stream = quote! {};
- let mut counter = 0u32;
+ let mut prefill_tabled_stream = quote! {};
+ let mut tabled_counter = 0u32;
+ let mut keyword_class_stream = quote! {};
  let mut keys = HashSet::<String>::new();
  let mut prev_key: Option<String> = None;
  let mut errors = Vec::<String>::new();
@@ -106,18 +141,42 @@ pub fn symbols(input: TokenStream) -> TokenStream {
  };
 
  // Generate the listed keywords.
- for keyword in &input.keywords.0 {
- let name = &keyword.name;
- let value = &keyword.value;
- check_dup(&value.value(), &mut errors);
- prefill_stream.extend(quote! {
- #value,
- });
- keyword_stream.extend(quote! {
- #[allow(non_upper_case_globals)]
- pub const #name: Symbol = Symbol::new(#counter);
- });
- counter += 1;
+ for (class, keywords) in &input.keywords.0 {
+ let mut class_stream = quote! {};
+ for keyword in keywords {
+ let name = &keyword.name;
+ let value = &keyword.value;
+ let v = value.value();
+ check_dup(&v, &mut errors);
+ if is_inlinable(&v) {
+ keyword_stream.extend(quote! {
+ #[allow(non_upper_case_globals)]
+ pub const #name: Symbol = Symbol::new_inlined(#value);
+ });
+ } else {
+ prefill_tabled_stream.extend(quote! {
+ #value,
+ });
+ keyword_stream.extend(quote! {
+ #[allow(non_upper_case_globals)]
+ pub const #name: Symbol = Symbol::new_tabled(#tabled_counter);
+ });
+ tabled_counter += 1;
+ }
+ class_stream.extend(quote! {
+ | kw::#name
+ });
+ }
+ if let Some(class) = class {
+ keyword_class_stream.extend(quote! {
+ fn #class(self) -> bool {
+ match self {
+ #class_stream => true,
+ _ => false
+ }
+ }
+ });
+ }
  }
 
  // Generate the listed symbols.
@@ -129,28 +188,32 @@ pub fn symbols(input: TokenStream) -> TokenStream {
  };
  check_dup(&value, &mut errors);
  check_order(&name.to_string(), &mut errors);
- prefill_stream.extend(quote! {
- #value,
- });
- symbols_stream.extend(quote! {
- #[allow(rustc::default_hash_types)]
- #[allow(non_upper_case_globals)]
- pub const #name: Symbol = Symbol::new(#counter);
- });
- counter += 1;
+ if is_inlinable(&value) {
+ symbols_stream.extend(quote! {
+ #[allow(rustc::default_hash_types)]
+ #[allow(non_upper_case_globals)]
+ pub const #name: Symbol = Symbol::new_inlined(#value);
+ });
+ } else {
+ prefill_tabled_stream.extend(quote! {
+ #value,
+ });
+ symbols_stream.extend(quote! {
+ #[allow(rustc::default_hash_types)]
+ #[allow(non_upper_case_globals)]
+ pub const #name: Symbol = Symbol::new_tabled(#tabled_counter);
+ });
+ tabled_counter += 1;
+ }
  }
 
  // Generate symbols for the strings "0", "1", ..., "9".
  for n in 0..10 {
  let n = n.to_string();
  check_dup(&n, &mut errors);
- prefill_stream.extend(quote! {
- #n,
- });
  digits_stream.extend(quote! {
- Symbol::new(#counter),
+ Symbol::new_inlined(#n),
  });
- counter += 1;
  }
 
  if !errors.is_empty() {
@@ -180,11 +243,15 @@ pub fn symbols(input: TokenStream) -> TokenStream {
 
  impl Interner {
  pub fn fresh() -> Self {
- Interner::prefill(&[
- #prefill_stream
+ Interner::prefill_tabled(&[
+ #prefill_tabled_stream
  ])
  }
  }
+
+ impl Symbol {
+ #keyword_class_stream
+ }
  });
 
  // To see the generated code generated, uncomment this line, recompile, and