Auto merge of rust-lang#17603 - Veykril:tt-symbols, r=Veykril

Switch token trees to use Symbols

src/tools/rust-analyzer/Cargo.lock

@@ -146,6 +146,7 @@ dependencies = [
  "arbitrary",
  "derive_arbitrary",
  "expect-test",
+ "intern",
  "mbe",
  "oorandom",
  "rustc-hash",
@@ -1045,6 +1046,7 @@ version = "0.0.0"
 dependencies = [
  "arrayvec",
  "cov-mark",
+ "intern",
  "parser",
  "rustc-hash",
  "smallvec",
@@ -1324,8 +1326,8 @@ version = "0.0.0"
 dependencies = [
  "base-db",
  "indexmap",
+ "intern",
  "la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "mbe",
  "paths",
  "rustc-hash",
  "serde",
@@ -1343,6 +1345,7 @@ version = "0.0.0"
 dependencies = [
  "base-db",
  "expect-test",
+ "intern",
  "libloading",
  "mbe",
  "memmap2",
@@ -1413,6 +1416,7 @@ dependencies = [
  "cargo_metadata",
  "cfg",
  "expect-test",
+ "intern",
  "itertools",
  "la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "paths",
@@ -1653,6 +1657,7 @@ dependencies = [
  "ide",
  "ide-db",
  "ide-ssr",
+ "intern",
  "itertools",
  "load-cargo",
  "lsp-server 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1966,6 +1971,7 @@ dependencies = [
  "base-db",
  "cfg",
  "hir-expand",
+ "intern",
  "rustc-hash",
  "span",
  "stdx",
@@ -2218,8 +2224,8 @@ name = "tt"
 version = "0.0.0"
 dependencies = [
  "arrayvec",
+ "intern",
  "ra-ap-rustc_lexer",
- "smol_str",
  "stdx",
  "text-size",
 ]

src/tools/rust-analyzer/Cargo.toml

@@ -50,7 +50,7 @@ debug = 2
 [workspace.dependencies]
 # local crates
 base-db = { path = "./crates/base-db", version = "0.0.0" }
-cfg = { path = "./crates/cfg", version = "0.0.0" }
+cfg = { path = "./crates/cfg", version = "0.0.0", features = ["tt"] }
 flycheck = { path = "./crates/flycheck", version = "0.0.0" }
 hir = { path = "./crates/hir", version = "0.0.0" }
 hir-def = { path = "./crates/hir-def", version = "0.0.0" }

src/tools/rust-analyzer/crates/cfg/Cargo.toml

@@ -15,7 +15,8 @@ doctest = false
 rustc-hash.workspace = true
 
 # locals deps
-tt.workspace = true
+tt = { workspace = true, optional = true }
+intern.workspace = true
 
 [dev-dependencies]
 expect-test = "1.4.1"

src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs

@@ -2,20 +2,20 @@
 //!
 //! See: <https://doc.rust-lang.org/reference/conditional-compilation.html#conditional-compilation>
 
-use std::{fmt, slice::Iter as SliceIter};
+use std::fmt;
 
-use tt::SmolStr;
+use intern::Symbol;
 
 /// A simple configuration value passed in from the outside.
-#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum CfgAtom {
  /// eg. `#[cfg(test)]`
- Flag(SmolStr),
+ Flag(Symbol),
  /// eg. `#[cfg(target_os = "linux")]`
  ///
  /// Note that a key can have multiple values that are all considered "active" at the same time.
  /// For example, `#[cfg(target_feature = "sse")]` and `#[cfg(target_feature = "sse2")]`.
- KeyValue { key: SmolStr, value: SmolStr },
+ KeyValue { key: Symbol, value: Symbol },
 }
 
 impl fmt::Display for CfgAtom {
@@ -44,6 +44,7 @@ impl From<CfgAtom> for CfgExpr {
 }
 
 impl CfgExpr {
+ #[cfg(feature = "tt")]
  pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
  next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
  }
@@ -63,10 +64,14 @@ impl CfgExpr {
  }
  }
 }
-fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
+
+#[cfg(feature = "tt")]
+fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
+ use intern::sym;
+
  let name = match it.next() {
  None => return None,
- Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),
+ Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
  Some(_) => return Some(CfgExpr::Invalid),
  };
 
@@ -77,10 +82,7 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
  Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
  it.next();
  it.next();
- // FIXME: escape? raw string?
- let value =
- SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"'));
- CfgAtom::KeyValue { key: name, value }.into()
+ CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into()
  }
  _ => return Some(CfgExpr::Invalid),
  }
@@ -89,10 +91,12 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
  it.next();
  let mut sub_it = subtree.token_trees.iter();
  let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it)).collect();
- match name.as_str() {
- "all" => CfgExpr::All(subs),
- "any" => CfgExpr::Any(subs),
- "not" => CfgExpr::Not(Box::new(subs.pop().unwrap_or(CfgExpr::Invalid))),
+ match &name {
+ s if *s == sym::all => CfgExpr::All(subs),
+ s if *s == sym::any => CfgExpr::Any(subs),
+ s if *s == sym::not => {
+ CfgExpr::Not(Box::new(subs.pop().unwrap_or(CfgExpr::Invalid)))
+ }
  _ => CfgExpr::Invalid,
  }
  }
@@ -112,11 +116,11 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
 impl arbitrary::Arbitrary<'_> for CfgAtom {
  fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<Self> {
  if u.arbitrary()? {
- Ok(CfgAtom::Flag(String::arbitrary(u)?.into()))
+ Ok(CfgAtom::Flag(Symbol::intern(<_>::arbitrary(u)?)))
  } else {
  Ok(CfgAtom::KeyValue {
- key: String::arbitrary(u)?.into(),
- value: String::arbitrary(u)?.into(),
+ key: Symbol::intern(<_>::arbitrary(u)?),
+ value: Symbol::intern(<_>::arbitrary(u)?),
  })
  }
  }

src/tools/rust-analyzer/crates/cfg/src/dnf.rs

@@ -66,9 +66,9 @@ impl DnfExpr {
  }
  }
 
- res.enabled.sort_unstable();
+ res.enabled.sort_unstable_by(compare);
  res.enabled.dedup();
- res.disabled.sort_unstable();
+ res.disabled.sort_unstable_by(compare);
  res.disabled.dedup();
  Some(res)
  }
@@ -114,14 +114,25 @@ impl DnfExpr {
  };
 
  // Undo the FxHashMap randomization for consistent output.
- diff.enable.sort_unstable();
- diff.disable.sort_unstable();
+ diff.enable.sort_unstable_by(compare);
+ diff.disable.sort_unstable_by(compare);
 
  Some(diff)
  })
  }
 }
 
+fn compare(a: &CfgAtom, b: &CfgAtom) -> std::cmp::Ordering {
+ match (a, b) {
+ (CfgAtom::Flag(a), CfgAtom::Flag(b)) => a.as_str().cmp(b.as_str()),
+ (CfgAtom::Flag(_), CfgAtom::KeyValue { .. }) => std::cmp::Ordering::Less,
+ (CfgAtom::KeyValue { .. }, CfgAtom::Flag(_)) => std::cmp::Ordering::Greater,
+ (CfgAtom::KeyValue { key, value }, CfgAtom::KeyValue { key: key2, value: value2 }) => {
+ key.as_str().cmp(key2.as_str()).then(value.as_str().cmp(value2.as_str()))
+ }
+ }
+}
+
 impl fmt::Display for DnfExpr {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  if self.conjunctions.len() != 1 {

src/tools/rust-analyzer/crates/cfg/src/lib.rs

@@ -8,7 +8,8 @@ mod tests;
 use std::fmt;
 
 use rustc_hash::FxHashSet;
-use tt::SmolStr;
+
+use intern::Symbol;
 
 pub use cfg_expr::{CfgAtom, CfgExpr};
 pub use dnf::DnfExpr;
@@ -48,11 +49,11 @@ impl CfgOptions {
  cfg.fold(&|atom| self.enabled.contains(atom))
  }
 
- pub fn insert_atom(&mut self, key: SmolStr) {
+ pub fn insert_atom(&mut self, key: Symbol) {
  self.enabled.insert(CfgAtom::Flag(key));
  }
 
- pub fn insert_key_value(&mut self, key: SmolStr, value: SmolStr) {
+ pub fn insert_key_value(&mut self, key: Symbol, value: Symbol) {
  self.enabled.insert(CfgAtom::KeyValue { key, value });
  }
 
@@ -66,19 +67,16 @@ impl CfgOptions {
  }
  }
 
- pub fn get_cfg_keys(&self) -> impl Iterator<Item = &SmolStr> {
+ pub fn get_cfg_keys(&self) -> impl Iterator<Item = &Symbol> {
  self.enabled.iter().map(|it| match it {
  CfgAtom::Flag(key) => key,
  CfgAtom::KeyValue { key, .. } => key,
  })
  }
 
- pub fn get_cfg_values<'a>(
- &'a self,
- cfg_key: &'a str,
- ) -> impl Iterator<Item = &'a SmolStr> + 'a {
+ pub fn get_cfg_values<'a>(&'a self, cfg_key: &'a str) -> impl Iterator<Item = &'a Symbol> + 'a {
  self.enabled.iter().filter_map(move |it| match it {
- CfgAtom::KeyValue { key, value } if cfg_key == key => Some(value),
+ CfgAtom::KeyValue { key, value } if cfg_key == key.as_str() => Some(value),
  _ => None,
  })
  }

src/tools/rust-analyzer/crates/cfg/src/tests.rs

@@ -1,5 +1,6 @@
 use arbitrary::{Arbitrary, Unstructured};
 use expect_test::{expect, Expect};
+use intern::Symbol;
 use mbe::{syntax_node_to_token_tree, DocCommentDesugarMode, DummyTestSpanMap, DUMMY};
 use syntax::{ast, AstNode, Edition};
 
@@ -65,22 +66,25 @@ fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) {
 
 #[test]
 fn test_cfg_expr_parser() {
- assert_parse_result("#![cfg(foo)]", CfgAtom::Flag("foo".into()).into());
- assert_parse_result("#![cfg(foo,)]", CfgAtom::Flag("foo".into()).into());
+ assert_parse_result("#![cfg(foo)]", CfgAtom::Flag(Symbol::intern("foo")).into());
+ assert_parse_result("#![cfg(foo,)]", CfgAtom::Flag(Symbol::intern("foo")).into());
  assert_parse_result(
  "#![cfg(not(foo))]",
- CfgExpr::Not(Box::new(CfgAtom::Flag("foo".into()).into())),
+ CfgExpr::Not(Box::new(CfgAtom::Flag(Symbol::intern("foo")).into())),
  );
  assert_parse_result("#![cfg(foo(bar))]", CfgExpr::Invalid);
 
  // Only take the first
- assert_parse_result(r#"#![cfg(foo, bar = "baz")]"#, CfgAtom::Flag("foo".into()).into());
+ assert_parse_result(
+ r#"#![cfg(foo, bar = "baz")]"#,
+ CfgAtom::Flag(Symbol::intern("foo")).into(),
+ );
 
  assert_parse_result(
  r#"#![cfg(all(foo, bar = "baz"))]"#,
  CfgExpr::All(vec![
- CfgAtom::Flag("foo".into()).into(),
- CfgAtom::KeyValue { key: "bar".into(), value: "baz".into() }.into(),
+ CfgAtom::Flag(Symbol::intern("foo")).into(),
+ CfgAtom::KeyValue { key: Symbol::intern("bar"), value: Symbol::intern("baz") }.into(),
  ]),
  );
 
@@ -90,7 +94,7 @@ fn test_cfg_expr_parser() {
  CfgExpr::Not(Box::new(CfgExpr::Invalid)),
  CfgExpr::All(vec![]),
  CfgExpr::Invalid,
- CfgAtom::KeyValue { key: "bar".into(), value: "baz".into() }.into(),
+ CfgAtom::KeyValue { key: Symbol::intern("bar"), value: Symbol::intern("baz") }.into(),
  ]),
  );
 }
@@ -167,7 +171,7 @@ fn hints() {
 
  check_enable_hints("#![cfg(all(a, b))]", &opts, &["enable a and b"]);
 
- opts.insert_atom("test".into());
+ opts.insert_atom(Symbol::intern("test"));
 
  check_enable_hints("#![cfg(test)]", &opts, &[]);
  check_enable_hints("#![cfg(not(test))]", &opts, &["disable test"]);
@@ -180,16 +184,16 @@ fn hints_impossible() {
 
  check_enable_hints("#![cfg(all(test, not(test)))]", &opts, &[]);
 
- opts.insert_atom("test".into());
+ opts.insert_atom(Symbol::intern("test"));
 
  check_enable_hints("#![cfg(all(test, not(test)))]", &opts, &[]);
 }
 
 #[test]
 fn why_inactive() {
  let mut opts = CfgOptions::default();
- opts.insert_atom("test".into());
- opts.insert_atom("test2".into());
+ opts.insert_atom(Symbol::intern("test"));
+ opts.insert_atom(Symbol::intern("test2"));
 
  check_why_inactive("#![cfg(a)]", &opts, expect![["a is disabled"]]);
  check_why_inactive("#![cfg(not(test))]", &opts, expect![["test is enabled"]]);