Auto merge of rust-lang#16446 - Tyrubias:literal_from_str, r=Veykril

Implement `literal_from_str` for proc macro server

Closes rust-lang#16233

Todos and unanswered questions:

- [x] Is this the correct approach? Can both the legacy and `rust_analyzer_span` servers depend on the `syntax` crate?
- [ ] How should we handle suffixes for string literals? It doesn't seem like `rust-analyzer` preservers suffix information after parsing.
- [x] Why are the `expect` tests failing? Specifically `test_fn_like_macro_clone_literals`

Author: bors

Cargo.lock

@@ -29,6 +29,7 @@ paths.workspace = true
 base-db.workspace = true
 span.workspace = true
 proc-macro-api.workspace = true
+syntax.workspace = true
 
 [dev-dependencies]
 expect-test = "1.4.0"

crates/proc-macro-srv/Cargo.toml

@@ -17,6 +17,7 @@ pub mod rust_analyzer_span;
 mod symbol;
 pub mod token_id;
 pub use symbol::*;
+use syntax::ast::{self, IsString};
 use tt::Spacing;
 
 fn delim_to_internal<S>(d: proc_macro::Delimiter, span: bridge::DelimSpan<S>) -> tt::Delimiter<S> {
@@ -54,6 +55,32 @@ fn spacing_to_external(spacing: Spacing) -> proc_macro::Spacing {
     }
 }
 
+fn literal_to_external(literal_kind: ast::LiteralKind) -> Option<proc_macro::bridge::LitKind> {
+    match literal_kind {
+        ast::LiteralKind::String(data) => Some(if data.is_raw() {
+            bridge::LitKind::StrRaw(data.raw_delimiter_count()?)
+        } else {
+            bridge::LitKind::Str
+        }),
+
+        ast::LiteralKind::ByteString(data) => Some(if data.is_raw() {
+            bridge::LitKind::ByteStrRaw(data.raw_delimiter_count()?)
+        } else {
+            bridge::LitKind::ByteStr
+        }),
+        ast::LiteralKind::CString(data) => Some(if data.is_raw() {
+            bridge::LitKind::CStrRaw(data.raw_delimiter_count()?)
+        } else {
+            bridge::LitKind::CStr
+        }),
+        ast::LiteralKind::IntNumber(_) => Some(bridge::LitKind::Integer),
+        ast::LiteralKind::FloatNumber(_) => Some(bridge::LitKind::Float),
+        ast::LiteralKind::Char(_) => Some(bridge::LitKind::Char),
+        ast::LiteralKind::Byte(_) => Some(bridge::LitKind::Byte),
+        ast::LiteralKind::Bool(_) => None,
+    }
+}
+
 struct LiteralFormatter<S>(bridge::Literal<S, Symbol>);
 
 impl<S> LiteralFormatter<S> {

crates/proc-macro-srv/src/server.rs

@@ -13,10 +13,11 @@ use std::{
 use ::tt::{TextRange, TextSize};
 use proc_macro::bridge::{self, server};
 use span::{Span, FIXUP_ERASED_FILE_AST_ID_MARKER};
+use syntax::ast::{self, IsString};
 
 use crate::server::{
-    delim_to_external, delim_to_internal, token_stream::TokenStreamBuilder, LiteralFormatter,
-    Symbol, SymbolInternerRef, SYMBOL_INTERNER,
+    delim_to_external, delim_to_internal, literal_to_external, token_stream::TokenStreamBuilder,
+    LiteralFormatter, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
 };
 mod tt {
     pub use ::tt::*;
@@ -70,11 +71,33 @@ impl server::FreeFunctions for RaSpanServer {
         &mut self,
         s: &str,
     ) -> Result<bridge::Literal<Self::Span, Self::Symbol>, ()> {
-        // FIXME: keep track of LitKind and Suffix
+        let literal = ast::Literal::parse(s).ok_or(())?;
+        let literal = literal.tree();
+
+        let kind = literal_to_external(literal.kind()).ok_or(())?;
+
+        // FIXME: handle more than just int and float suffixes
+        let suffix = match literal.kind() {
+            ast::LiteralKind::FloatNumber(num) => num.suffix().map(ToString::to_string),
+            ast::LiteralKind::IntNumber(num) => num.suffix().map(ToString::to_string),
+            _ => None,
+        };
+
+        let text = match literal.kind() {
+            ast::LiteralKind::String(data) => data.text_without_quotes().to_string(),
+            ast::LiteralKind::ByteString(data) => data.text_without_quotes().to_string(),
+            ast::LiteralKind::CString(data) => data.text_without_quotes().to_string(),
+            _ => s.to_string(),
+        };
+        let text = if let Some(ref suffix) = suffix { text.strip_suffix(suffix) } else { None }
+            .unwrap_or(&text);
+
+        let suffix = suffix.map(|suffix| Symbol::intern(self.interner, &suffix));
+
         Ok(bridge::Literal {
-            kind: bridge::LitKind::Err,
-            symbol: Symbol::intern(self.interner, s),
-            suffix: None,
+            kind,
+            symbol: Symbol::intern(self.interner, text),
+            suffix,
             span: self.call_site,
         })
     }

crates/proc-macro-srv/src/server/rust_analyzer_span.rs

@@ -6,10 +6,11 @@ use std::{
 };
 
 use proc_macro::bridge::{self, server};
+use syntax::ast::{self, IsString};
 
 use crate::server::{
-    delim_to_external, delim_to_internal, token_stream::TokenStreamBuilder, LiteralFormatter,
-    Symbol, SymbolInternerRef, SYMBOL_INTERNER,
+    delim_to_external, delim_to_internal, literal_to_external, token_stream::TokenStreamBuilder,
+    LiteralFormatter, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
 };
 mod tt {
     pub use proc_macro_api::msg::TokenId;
@@ -62,11 +63,33 @@ impl server::FreeFunctions for TokenIdServer {
         &mut self,
         s: &str,
     ) -> Result<bridge::Literal<Self::Span, Self::Symbol>, ()> {
-        // FIXME: keep track of LitKind and Suffix
+        let literal = ast::Literal::parse(s).ok_or(())?;
+        let literal = literal.tree();
+
+        let kind = literal_to_external(literal.kind()).ok_or(())?;
+
+        // FIXME: handle more than just int and float suffixes
+        let suffix = match literal.kind() {
+            ast::LiteralKind::FloatNumber(num) => num.suffix().map(ToString::to_string),
+            ast::LiteralKind::IntNumber(num) => num.suffix().map(ToString::to_string),
+            _ => None,
+        };
+
+        let text = match literal.kind() {
+            ast::LiteralKind::String(data) => data.text_without_quotes().to_string(),
+            ast::LiteralKind::ByteString(data) => data.text_without_quotes().to_string(),
+            ast::LiteralKind::CString(data) => data.text_without_quotes().to_string(),
+            _ => s.to_string(),
+        };
+        let text = if let Some(ref suffix) = suffix { text.strip_suffix(suffix) } else { None }
+            .unwrap_or(&text);
+
+        let suffix = suffix.map(|suffix| Symbol::intern(self.interner, &suffix));
+
         Ok(bridge::Literal {
-            kind: bridge::LitKind::Err,
-            symbol: Symbol::intern(self.interner, s),
-            suffix: None,
+            kind,
+            symbol: Symbol::intern(self.interner, text),
+            suffix,
             span: self.call_site,
         })
     }

crates/proc-macro-srv/src/server/token_id.rs

@@ -204,6 +204,16 @@ pub trait IsString: AstToken {
         assert!(TextRange::up_to(contents_range.len()).contains_range(range));
         Some(range + contents_range.start())
     }
+    fn raw_delimiter_count(&self) -> Option<u8> {
+        let text = self.text();
+        let quote_range = self.text_range_between_quotes()?;
+        let range_start = self.syntax().text_range().start();
+        text[TextRange::up_to((quote_range - range_start).start())]
+            .matches('#')
+            .count()
+            .try_into()
+            .ok()
+    }
 }
 
 impl IsString for ast::String {

crates/syntax/src/ast/token_ext.rs

@@ -182,6 +182,28 @@ impl SourceFile {
     }
 }
 
+impl ast::Literal {
+    pub fn parse(text: &str) -> Option<Parse<ast::Literal>> {
+        let lexed = parser::LexedStr::new(text);
+        let parser_input = lexed.to_input();
+        let parser_output = parser::TopEntryPoint::Expr.parse(&parser_input);
+        let (green, mut errors, _) = parsing::build_tree(lexed, parser_output);
+        let root = SyntaxNode::new_root(green.clone());
+
+        errors.extend(validation::validate(&root));
+
+        if root.kind() == SyntaxKind::LITERAL {
+            Some(Parse {
+                green,
+                errors: if errors.is_empty() { None } else { Some(errors.into()) },
+                _ty: PhantomData,
+            })
+        } else {
+            None
+        }
+    }
+}
+
 impl ast::TokenTree {
     pub fn reparse_as_comma_separated_expr(self) -> Parse<ast::MacroEagerInput> {
         let tokens = self.syntax().descendants_with_tokens().filter_map(NodeOrToken::into_token);