Auto merge of rust-lang#13717 - lowr:fix/proc-macro-ident-is-raw, r=Veykril

Handle raw identifiers in proc macro server

Fixes rust-lang#13706

When proc macros create `proc_macro::Ident`s, they pass an identifier text without "r#" prefix and a flag `is_raw` to proc macro server. Our `tt::Ident` currently stores the text *with* "r#" so we need to adjust them somewhere.

Rather than following rustc and adding `is_raw` field to our `tt::Ident`, I opted for adjusting the representation of identifiers in proc macro server, because we don't need the field outside it.

It's hard to write regression test for this, but at least I:
- ran `cargo +nightly t --features sysroot-abi` and all the tests passed
- built proc macro server with `cargo +nightly b -r --bin rust-analyzer-proc-macro-srv --features sysroot-abi` and made sure rust-lang#13706 resolved
  - For the record, the nightly versions used are `rustc 1.67.0-nightly (32e613b 2022-12-02)` and `cargo 1.67.0-nightly (e027c4b5d 2022-11-25)`.

Author: bors

crates/proc-macro-srv/src/abis/abi_1_58/ra_server.rs

@@ -471,8 +471,12 @@ impl server::Punct for RustAnalyzer {
 }
 
 impl server::Ident for RustAnalyzer {
-    fn new(&mut self, string: &str, span: Self::Span, _is_raw: bool) -> Self::Ident {
-        IdentId(self.ident_interner.intern(&IdentData(tt::Ident { text: string.into(), id: span })))
+    fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
+        IdentId(self.ident_interner.intern(&IdentData(tt::Ident::new_with_is_raw(
+            string.into(),
+            span,
+            is_raw,
+        ))))
     }
 
     fn span(&mut self, ident: Self::Ident) -> Self::Span {

crates/proc-macro-srv/src/abis/abi_1_63/ra_server.rs

@@ -486,8 +486,12 @@ impl server::Punct for RustAnalyzer {
 }
 
 impl server::Ident for RustAnalyzer {
-    fn new(&mut self, string: &str, span: Self::Span, _is_raw: bool) -> Self::Ident {
-        IdentId(self.ident_interner.intern(&IdentData(tt::Ident { text: string.into(), id: span })))
+    fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
+        IdentId(self.ident_interner.intern(&IdentData(tt::Ident::new_with_is_raw(
+            string.into(),
+            span,
+            is_raw,
+        ))))
     }
 
     fn span(&mut self, ident: Self::Ident) -> Self::Span {

crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs

@@ -107,8 +107,8 @@ impl server::TokenStream for RustAnalyzer {
             }
 
             bridge::TokenTree::Ident(ident) => {
-                // FIXME: handle raw idents
                 let text = ident.sym.text();
+                let text = if ident.is_raw { tt::SmolStr::from_iter(["r#", &text]) } else { text };
                 let ident: tt::Ident = tt::Ident { text, id: ident.span };
                 let leaf = tt::Leaf::from(ident);
                 let tree = TokenTree::from(leaf);
@@ -182,9 +182,8 @@ impl server::TokenStream for RustAnalyzer {
             .map(|tree| match tree {
                 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
                     bridge::TokenTree::Ident(bridge::Ident {
-                        sym: Symbol::intern(&ident.text),
-                        // FIXME: handle raw idents
-                        is_raw: false,
+                        sym: Symbol::intern(&ident.text.trim_start_matches("r#")),
+                        is_raw: ident.text.starts_with("r#"),
                         span: ident.id,
                     })
                 }

crates/proc-macro-srv/src/tests/mod.rs

@@ -63,7 +63,7 @@ fn test_fn_like_macro_clone_raw_ident() {
         "r#async",
         expect![[r#"
             SUBTREE $
-              IDENT   async 4294967295"#]],
+              IDENT   r#async 4294967295"#]],
     );
 }
 
@@ -86,15 +86,13 @@ fn test_fn_like_mk_literals() {
 
 #[test]
 fn test_fn_like_mk_idents() {
-    // FIXME: this test is wrong: raw should be 'r#raw' but ABIs 1.64 and below
-    // simply ignore `is_raw` when implementing the `Ident` interface.
     assert_expand(
         "fn_like_mk_idents",
         r#""#,
         expect![[r#"
             SUBTREE $
               IDENT   standard 4294967295
-              IDENT   raw 4294967295"#]],
+              IDENT   r#raw 4294967295"#]],
     );
 }
 

crates/tt/src/lib.rs

@@ -86,10 +86,20 @@ pub enum Spacing {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Ident {
+    /// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier.
     pub text: SmolStr,
     pub id: TokenId,
 }
 
+impl Ident {
+    /// Constructor intended to be used only by proc macro server. `text` should not contain raw
+    /// identifier prefix.
+    pub fn new_with_is_raw(text: SmolStr, id: TokenId, is_raw: bool) -> Self {
+        let text = if is_raw { SmolStr::from_iter(["r#", &text]) } else { text };
+        Ident { text, id }
+    }
+}
+
 impl Leaf {
     pub fn id(&self) -> TokenId {
         match self {