Auto merge of #17603 - Veykril:tt-symbols, r=Veykril

Switch token trees to use Symbols

Author: bors

src/tools/rust-analyzer/Cargo.lock

@@ -146,6 +146,7 @@ dependencies = [
  "arbitrary",
  "derive_arbitrary",
  "expect-test",
+ "intern",
  "mbe",
  "oorandom",
  "rustc-hash",
@@ -1045,6 +1046,7 @@ version = "0.0.0"
 dependencies = [
  "arrayvec",
  "cov-mark",
+ "intern",
  "parser",
  "rustc-hash",
  "smallvec",
@@ -1324,8 +1326,8 @@ version = "0.0.0"
 dependencies = [
  "base-db",
  "indexmap",
+ "intern",
  "la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "mbe",
  "paths",
  "rustc-hash",
  "serde",
@@ -1343,6 +1345,7 @@ version = "0.0.0"
 dependencies = [
  "base-db",
  "expect-test",
+ "intern",
  "libloading",
  "mbe",
  "memmap2",
@@ -1413,6 +1416,7 @@ dependencies = [
  "cargo_metadata",
  "cfg",
  "expect-test",
+ "intern",
  "itertools",
  "la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "paths",
@@ -1653,6 +1657,7 @@ dependencies = [
  "ide",
  "ide-db",
  "ide-ssr",
+ "intern",
  "itertools",
  "load-cargo",
  "lsp-server 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1966,6 +1971,7 @@ dependencies = [
  "base-db",
  "cfg",
  "hir-expand",
+ "intern",
  "rustc-hash",
  "span",
  "stdx",
@@ -2218,8 +2224,8 @@ name = "tt"
 version = "0.0.0"
 dependencies = [
  "arrayvec",
+ "intern",
  "ra-ap-rustc_lexer",
- "smol_str",
  "stdx",
  "text-size",
 ]

src/tools/rust-analyzer/Cargo.toml

@@ -50,7 +50,7 @@ debug = 2
 [workspace.dependencies]
 # local crates
 base-db = { path = "./crates/base-db", version = "0.0.0" }
-cfg = { path = "./crates/cfg", version = "0.0.0" }
+cfg = { path = "./crates/cfg", version = "0.0.0", features = ["tt"] }
 flycheck = { path = "./crates/flycheck", version = "0.0.0" }
 hir = { path = "./crates/hir", version = "0.0.0" }
 hir-def = { path = "./crates/hir-def", version = "0.0.0" }

src/tools/rust-analyzer/crates/cfg/Cargo.toml

@@ -15,7 +15,8 @@ doctest = false
 rustc-hash.workspace = true
 
 # locals deps
-tt.workspace = true
+tt = { workspace = true, optional = true }
+intern.workspace = true
 
 [dev-dependencies]
 expect-test = "1.4.1"

src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs

@@ -2,20 +2,20 @@
 //!
 //! See: <https://doc.rust-lang.org/reference/conditional-compilation.html#conditional-compilation>
 
-use std::{fmt, slice::Iter as SliceIter};
+use std::fmt;
 
-use tt::SmolStr;
+use intern::Symbol;
 
 /// A simple configuration value passed in from the outside.
-#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum CfgAtom {
     /// eg. `#[cfg(test)]`
-    Flag(SmolStr),
+    Flag(Symbol),
     /// eg. `#[cfg(target_os = "linux")]`
     ///
     /// Note that a key can have multiple values that are all considered "active" at the same time.
     /// For example, `#[cfg(target_feature = "sse")]` and `#[cfg(target_feature = "sse2")]`.
-    KeyValue { key: SmolStr, value: SmolStr },
+    KeyValue { key: Symbol, value: Symbol },
 }
 
 impl fmt::Display for CfgAtom {
@@ -44,6 +44,7 @@ impl From<CfgAtom> for CfgExpr {
 }
 
 impl CfgExpr {
+    #[cfg(feature = "tt")]
     pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
         next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
     }
@@ -63,10 +64,14 @@ impl CfgExpr {
         }
     }
 }
-fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
+
+#[cfg(feature = "tt")]
+fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
+    use intern::sym;
+
     let name = match it.next() {
         None => return None,
-        Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),
+        Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
         Some(_) => return Some(CfgExpr::Invalid),
     };
 
@@ -77,10 +82,7 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
                 Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
                     it.next();
                     it.next();
-                    // FIXME: escape? raw string?
-                    let value =
-                        SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"'));
-                    CfgAtom::KeyValue { key: name, value }.into()
+                    CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into()
                 }
                 _ => return Some(CfgExpr::Invalid),
             }
@@ -89,10 +91,12 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
             it.next();
             let mut sub_it = subtree.token_trees.iter();
             let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it)).collect();
-            match name.as_str() {
-                "all" => CfgExpr::All(subs),
-                "any" => CfgExpr::Any(subs),
-                "not" => CfgExpr::Not(Box::new(subs.pop().unwrap_or(CfgExpr::Invalid))),
+            match &name {
+                s if *s == sym::all => CfgExpr::All(subs),
+                s if *s == sym::any => CfgExpr::Any(subs),
+                s if *s == sym::not => {
+                    CfgExpr::Not(Box::new(subs.pop().unwrap_or(CfgExpr::Invalid)))
+                }
                 _ => CfgExpr::Invalid,
             }
         }
@@ -112,11 +116,11 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
 impl arbitrary::Arbitrary<'_> for CfgAtom {
     fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<Self> {
         if u.arbitrary()? {
-            Ok(CfgAtom::Flag(String::arbitrary(u)?.into()))
+            Ok(CfgAtom::Flag(Symbol::intern(<_>::arbitrary(u)?)))
         } else {
             Ok(CfgAtom::KeyValue {
-                key: String::arbitrary(u)?.into(),
-                value: String::arbitrary(u)?.into(),
+                key: Symbol::intern(<_>::arbitrary(u)?),
+                value: Symbol::intern(<_>::arbitrary(u)?),
             })
         }
     }

src/tools/rust-analyzer/crates/cfg/src/dnf.rs

@@ -66,9 +66,9 @@ impl DnfExpr {
             }
         }
 
-        res.enabled.sort_unstable();
+        res.enabled.sort_unstable_by(compare);
         res.enabled.dedup();
-        res.disabled.sort_unstable();
+        res.disabled.sort_unstable_by(compare);
         res.disabled.dedup();
         Some(res)
     }
@@ -114,14 +114,25 @@ impl DnfExpr {
             };
 
             // Undo the FxHashMap randomization for consistent output.
-            diff.enable.sort_unstable();
-            diff.disable.sort_unstable();
+            diff.enable.sort_unstable_by(compare);
+            diff.disable.sort_unstable_by(compare);
 
             Some(diff)
         })
     }
 }
 
+fn compare(a: &CfgAtom, b: &CfgAtom) -> std::cmp::Ordering {
+    match (a, b) {
+        (CfgAtom::Flag(a), CfgAtom::Flag(b)) => a.as_str().cmp(b.as_str()),
+        (CfgAtom::Flag(_), CfgAtom::KeyValue { .. }) => std::cmp::Ordering::Less,
+        (CfgAtom::KeyValue { .. }, CfgAtom::Flag(_)) => std::cmp::Ordering::Greater,
+        (CfgAtom::KeyValue { key, value }, CfgAtom::KeyValue { key: key2, value: value2 }) => {
+            key.as_str().cmp(key2.as_str()).then(value.as_str().cmp(value2.as_str()))
+        }
+    }
+}
+
 impl fmt::Display for DnfExpr {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         if self.conjunctions.len() != 1 {

src/tools/rust-analyzer/crates/cfg/src/lib.rs

@@ -8,7 +8,8 @@ mod tests;
 use std::fmt;
 
 use rustc_hash::FxHashSet;
-use tt::SmolStr;
+
+use intern::Symbol;
 
 pub use cfg_expr::{CfgAtom, CfgExpr};
 pub use dnf::DnfExpr;
@@ -48,11 +49,11 @@ impl CfgOptions {
         cfg.fold(&|atom| self.enabled.contains(atom))
     }
 
-    pub fn insert_atom(&mut self, key: SmolStr) {
+    pub fn insert_atom(&mut self, key: Symbol) {
         self.enabled.insert(CfgAtom::Flag(key));
     }
 
-    pub fn insert_key_value(&mut self, key: SmolStr, value: SmolStr) {
+    pub fn insert_key_value(&mut self, key: Symbol, value: Symbol) {
         self.enabled.insert(CfgAtom::KeyValue { key, value });
     }
 
@@ -66,19 +67,16 @@ impl CfgOptions {
         }
     }
 
-    pub fn get_cfg_keys(&self) -> impl Iterator<Item = &SmolStr> {
+    pub fn get_cfg_keys(&self) -> impl Iterator<Item = &Symbol> {
         self.enabled.iter().map(|it| match it {
             CfgAtom::Flag(key) => key,
             CfgAtom::KeyValue { key, .. } => key,
         })
     }
 
-    pub fn get_cfg_values<'a>(
-        &'a self,
-        cfg_key: &'a str,
-    ) -> impl Iterator<Item = &'a SmolStr> + 'a {
+    pub fn get_cfg_values<'a>(&'a self, cfg_key: &'a str) -> impl Iterator<Item = &'a Symbol> + 'a {
         self.enabled.iter().filter_map(move |it| match it {
-            CfgAtom::KeyValue { key, value } if cfg_key == key => Some(value),
+            CfgAtom::KeyValue { key, value } if cfg_key == key.as_str() => Some(value),
             _ => None,
         })
     }

src/tools/rust-analyzer/crates/cfg/src/tests.rs

@@ -1,5 +1,6 @@
 use arbitrary::{Arbitrary, Unstructured};
 use expect_test::{expect, Expect};
+use intern::Symbol;
 use mbe::{syntax_node_to_token_tree, DocCommentDesugarMode, DummyTestSpanMap, DUMMY};
 use syntax::{ast, AstNode, Edition};
 
@@ -65,22 +66,25 @@ fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) {
 
 #[test]
 fn test_cfg_expr_parser() {
-    assert_parse_result("#![cfg(foo)]", CfgAtom::Flag("foo".into()).into());
-    assert_parse_result("#![cfg(foo,)]", CfgAtom::Flag("foo".into()).into());
+    assert_parse_result("#![cfg(foo)]", CfgAtom::Flag(Symbol::intern("foo")).into());
+    assert_parse_result("#![cfg(foo,)]", CfgAtom::Flag(Symbol::intern("foo")).into());
     assert_parse_result(
         "#![cfg(not(foo))]",
-        CfgExpr::Not(Box::new(CfgAtom::Flag("foo".into()).into())),
+        CfgExpr::Not(Box::new(CfgAtom::Flag(Symbol::intern("foo")).into())),
     );
     assert_parse_result("#![cfg(foo(bar))]", CfgExpr::Invalid);
 
     // Only take the first
-    assert_parse_result(r#"#![cfg(foo, bar = "baz")]"#, CfgAtom::Flag("foo".into()).into());
+    assert_parse_result(
+        r#"#![cfg(foo, bar = "baz")]"#,
+        CfgAtom::Flag(Symbol::intern("foo")).into(),
+    );
 
     assert_parse_result(
         r#"#![cfg(all(foo, bar = "baz"))]"#,
         CfgExpr::All(vec![
-            CfgAtom::Flag("foo".into()).into(),
-            CfgAtom::KeyValue { key: "bar".into(), value: "baz".into() }.into(),
+            CfgAtom::Flag(Symbol::intern("foo")).into(),
+            CfgAtom::KeyValue { key: Symbol::intern("bar"), value: Symbol::intern("baz") }.into(),
         ]),
     );
 
@@ -90,7 +94,7 @@ fn test_cfg_expr_parser() {
             CfgExpr::Not(Box::new(CfgExpr::Invalid)),
             CfgExpr::All(vec![]),
             CfgExpr::Invalid,
-            CfgAtom::KeyValue { key: "bar".into(), value: "baz".into() }.into(),
+            CfgAtom::KeyValue { key: Symbol::intern("bar"), value: Symbol::intern("baz") }.into(),
         ]),
     );
 }
@@ -167,7 +171,7 @@ fn hints() {
 
     check_enable_hints("#![cfg(all(a, b))]", &opts, &["enable a and b"]);
 
-    opts.insert_atom("test".into());
+    opts.insert_atom(Symbol::intern("test"));
 
     check_enable_hints("#![cfg(test)]", &opts, &[]);
     check_enable_hints("#![cfg(not(test))]", &opts, &["disable test"]);
@@ -180,16 +184,16 @@ fn hints_impossible() {
 
     check_enable_hints("#![cfg(all(test, not(test)))]", &opts, &[]);
 
-    opts.insert_atom("test".into());
+    opts.insert_atom(Symbol::intern("test"));
 
     check_enable_hints("#![cfg(all(test, not(test)))]", &opts, &[]);
 }
 
 #[test]
 fn why_inactive() {
     let mut opts = CfgOptions::default();
-    opts.insert_atom("test".into());
-    opts.insert_atom("test2".into());
+    opts.insert_atom(Symbol::intern("test"));
+    opts.insert_atom(Symbol::intern("test2"));
 
     check_why_inactive("#![cfg(a)]", &opts, expect![["a is disabled"]]);
     check_why_inactive("#![cfg(not(test))]", &opts, expect![["test is enabled"]]);