Merge #9550

9550: Proc macro multi abi proof of concept r=matklad a=alexjg

#8925 was irritating me so I thought I would have a bash at fixing it. What I've done here is copy the `crates/proc_macro_srv/src/proc_macro` code (which is copied from `<RUST>/library/proc_macro`) to `crates/proc_macro_srv/src/proc_macro_nightly` and the modified the nightly version to include the changes from #9047 and https://github.com/yotamofek/rust-analyzer/commit/aeb7b183a2302af968f1110fb3cbd00c76a34974

This gives us the code to support both stable and nightly ABIs. Then we use the `proc_macro_api::version::read_dylib_info` to determine which version of the ABI to load when creating a `ProcMacroLibraryLibLoading` (which is now an enum). 

This seems to work for me.  The code could be cleaned up but I wanted to see if the approach makes sense before I spend more time on it.

I've split the change into two commits, the first is just copying and modifying the `proc_macro` crate, the second contains most of the interesting work around figuring out which ABI to use.

Co-authored-by: Alex Good <alex@memoryandthought.me>
Co-authored-by: alexjg <alex@memoryandthought.me>

Author: bors[bot]

crates/proc_macro_srv/src/abis/abi_1_47/mod.rs

@@ -0,0 +1,106 @@
+//! Macro ABI for version 1.47 of rustc
+
+#[allow(dead_code)]
+#[doc(hidden)]
+mod proc_macro;
+
+#[allow(dead_code)]
+#[doc(hidden)]
+mod rustc_server;
+use libloading::Library;
+
+use proc_macro_api::ProcMacroKind;
+
+use super::PanicMessage;
+
+pub use rustc_server::TokenStream;
+
+pub(crate) struct Abi {
+    exported_macros: Vec<proc_macro::bridge::client::ProcMacro>,
+}
+
+impl From<proc_macro::bridge::PanicMessage> for PanicMessage {
+    fn from(p: proc_macro::bridge::PanicMessage) -> Self {
+        Self { message: p.as_str().map(|s| s.to_string()) }
+    }
+}
+
+impl Abi {
+    pub unsafe fn from_lib(lib: &Library, symbol_name: String) -> Result<Abi, libloading::Error> {
+        let macros: libloading::Symbol<&&[proc_macro::bridge::client::ProcMacro]> =
+            lib.get(symbol_name.as_bytes())?;
+        Ok(Self { exported_macros: macros.to_vec() })
+    }
+
+    pub fn expand(
+        &self,
+        macro_name: &str,
+        macro_body: &tt::Subtree,
+        attributes: Option<&tt::Subtree>,
+    ) -> Result<tt::Subtree, PanicMessage> {
+        let parsed_body = rustc_server::TokenStream::with_subtree(macro_body.clone());
+
+        let parsed_attributes = attributes.map_or(rustc_server::TokenStream::new(), |attr| {
+            rustc_server::TokenStream::with_subtree(attr.clone())
+        });
+
+        for proc_macro in &self.exported_macros {
+            match proc_macro {
+                proc_macro::bridge::client::ProcMacro::CustomDerive {
+                    trait_name, client, ..
+                } if *trait_name == macro_name => {
+                    let res = client.run(
+                        &proc_macro::bridge::server::SameThread,
+                        rustc_server::Rustc::default(),
+                        parsed_body,
+                        false,
+                    );
+                    return res.map(|it| it.into_subtree()).map_err(PanicMessage::from);
+                }
+                proc_macro::bridge::client::ProcMacro::Bang { name, client }
+                    if *name == macro_name =>
+                {
+                    let res = client.run(
+                        &proc_macro::bridge::server::SameThread,
+                        rustc_server::Rustc::default(),
+                        parsed_body,
+                        false,
+                    );
+                    return res.map(|it| it.into_subtree()).map_err(PanicMessage::from);
+                }
+                proc_macro::bridge::client::ProcMacro::Attr { name, client }
+                    if *name == macro_name =>
+                {
+                    let res = client.run(
+                        &proc_macro::bridge::server::SameThread,
+                        rustc_server::Rustc::default(),
+                        parsed_attributes,
+                        parsed_body,
+                        false,
+                    );
+                    return res.map(|it| it.into_subtree()).map_err(PanicMessage::from);
+                }
+                _ => continue,
+            }
+        }
+
+        Err(proc_macro::bridge::PanicMessage::String("Nothing to expand".to_string()).into())
+    }
+
+    pub fn list_macros(&self) -> Vec<(String, ProcMacroKind)> {
+        self.exported_macros
+            .iter()
+            .map(|proc_macro| match proc_macro {
+                proc_macro::bridge::client::ProcMacro::CustomDerive { trait_name, .. } => {
+                    (trait_name.to_string(), ProcMacroKind::CustomDerive)
+                }
+                proc_macro::bridge::client::ProcMacro::Bang { name, .. } => {
+                    (name.to_string(), ProcMacroKind::FuncLike)
+                }
+                proc_macro::bridge::client::ProcMacro::Attr { name, .. } => {
+                    (name.to_string(), ProcMacroKind::Attr)
+                }
+            })
+            .collect()
+    }
+}

crates/proc_macro_srv/src/proc_macro/bridge/buffer.rs renamed to crates/proc_macro_srv/src/abis/abi_1_47/proc_macro/bridge/buffer.rs

@@ -3,6 +3,7 @@
 //! Copy from <https://github.com/rust-lang/rust/blob/6050e523bae6de61de4e060facc43dc512adaccd/src/libproc_macro/bridge/client.rs>
 //! augmented with removing unstable features
 
+use super::super::TokenStream as CrateTokenStream;
 use super::*;
 
 macro_rules! define_handles {
@@ -401,26 +402,26 @@ fn run_client<A: for<'a, 's> DecodeMut<'a, 's, ()>, R: Encode<()>>(
     b
 }
 
-impl Client<fn(crate::TokenStream) -> crate::TokenStream> {
-    pub fn expand1(f: fn(crate::TokenStream) -> crate::TokenStream) -> Self {
+impl Client<fn(CrateTokenStream) -> CrateTokenStream> {
+    pub fn expand1(f: fn(CrateTokenStream) -> CrateTokenStream) -> Self {
         extern "C" fn run(
             bridge: Bridge<'_>,
-            f: impl FnOnce(crate::TokenStream) -> crate::TokenStream,
+            f: impl FnOnce(CrateTokenStream) -> CrateTokenStream,
         ) -> Buffer<u8> {
-            run_client(bridge, |input| f(crate::TokenStream(input)).0)
+            run_client(bridge, |input| f(CrateTokenStream(input)).0)
         }
         Client { get_handle_counters: HandleCounters::get, run, f }
     }
 }
 
-impl Client<fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream> {
-    pub fn expand2(f: fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream) -> Self {
+impl Client<fn(CrateTokenStream, CrateTokenStream) -> CrateTokenStream> {
+    pub fn expand2(f: fn(CrateTokenStream, CrateTokenStream) -> CrateTokenStream) -> Self {
         extern "C" fn run(
             bridge: Bridge<'_>,
-            f: impl FnOnce(crate::TokenStream, crate::TokenStream) -> crate::TokenStream,
+            f: impl FnOnce(CrateTokenStream, CrateTokenStream) -> CrateTokenStream,
         ) -> Buffer<u8> {
             run_client(bridge, |(input, input2)| {
-                f(crate::TokenStream(input), crate::TokenStream(input2)).0
+                f(CrateTokenStream(input), CrateTokenStream(input2)).0
             })
         }
         Client { get_handle_counters: HandleCounters::get, run, f }
@@ -433,17 +434,17 @@ pub enum ProcMacro {
     CustomDerive {
         trait_name: &'static str,
         attributes: &'static [&'static str],
-        client: Client<fn(crate::TokenStream) -> crate::TokenStream>,
+        client: Client<fn(CrateTokenStream) -> CrateTokenStream>,
     },
 
     Attr {
         name: &'static str,
-        client: Client<fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream>,
+        client: Client<fn(CrateTokenStream, CrateTokenStream) -> CrateTokenStream>,
     },
 
     Bang {
         name: &'static str,
-        client: Client<fn(crate::TokenStream) -> crate::TokenStream>,
+        client: Client<fn(CrateTokenStream) -> CrateTokenStream>,
     },
 }
 
@@ -465,19 +466,19 @@ impl ProcMacro {
     pub fn custom_derive(
         trait_name: &'static str,
         attributes: &'static [&'static str],
-        expand: fn(crate::TokenStream) -> crate::TokenStream,
+        expand: fn(CrateTokenStream) -> CrateTokenStream,
     ) -> Self {
         ProcMacro::CustomDerive { trait_name, attributes, client: Client::expand1(expand) }
     }
 
     pub fn attr(
         name: &'static str,
-        expand: fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream,
+        expand: fn(CrateTokenStream, CrateTokenStream) -> CrateTokenStream,
     ) -> Self {
         ProcMacro::Attr { name, client: Client::expand2(expand) }
     }
 
-    pub fn bang(name: &'static str, expand: fn(crate::TokenStream) -> crate::TokenStream) -> Self {
+    pub fn bang(name: &'static str, expand: fn(CrateTokenStream) -> CrateTokenStream) -> Self {
         ProcMacro::Bang { name, client: Client::expand1(expand) }
     }
 }

crates/proc_macro_srv/src/proc_macro/bridge/client.rs renamed to crates/proc_macro_srv/src/abis/abi_1_47/proc_macro/bridge/client.rs

@@ -13,7 +13,7 @@
 
 #![deny(unsafe_code)]
 
-pub use crate::proc_macro::{Delimiter, Level, LineColumn, Spacing};
+pub use super::{Delimiter, Level, LineColumn, Spacing};
 use std::fmt;
 use std::hash::Hash;
 use std::marker;

crates/proc_macro_srv/src/proc_macro/bridge/closure.rs renamed to crates/proc_macro_srv/src/abis/abi_1_47/proc_macro/bridge/closure.rs

@@ -3,6 +3,7 @@
 //! Copy from <https://github.com/rust-lang/rust/blob/6050e523bae6de61de4e060facc43dc512adaccd/src/libproc_macro/bridge/server.rs>
 //! augmented with removing unstable features
 
+use super::super::TokenStream as ProcMacroTokenStream;
 use super::*;
 
 // FIXME(eddyb) generate the definition of `HandleStore` in `server.rs`.
@@ -308,7 +309,7 @@ fn run_server<
     Result::decode(&mut &b[..], &mut dispatcher.handle_store)
 }
 
-impl client::Client<fn(crate::TokenStream) -> crate::TokenStream> {
+impl client::Client<fn(ProcMacroTokenStream) -> ProcMacroTokenStream> {
     pub fn run<S: Server>(
         &self,
         strategy: &impl ExecutionStrategy,
@@ -330,7 +331,7 @@ impl client::Client<fn(crate::TokenStream) -> crate::TokenStream> {
     }
 }
 
-impl client::Client<fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream> {
+impl client::Client<fn(ProcMacroTokenStream, ProcMacroTokenStream) -> ProcMacroTokenStream> {
     pub fn run<S: Server>(
         &self,
         strategy: &impl ExecutionStrategy,

crates/proc_macro_srv/src/proc_macro/bridge/handle.rs renamed to crates/proc_macro_srv/src/abis/abi_1_47/proc_macro/bridge/handle.rs

@@ -3,7 +3,7 @@
 //! Copy from <https://github.com/rust-lang/rust/blob/6050e523bae6de61de4e060facc43dc512adaccd/src/libproc_macro/diagnostic.rs>
 //! augmented with removing unstable features
 
-use crate::proc_macro::Span;
+use super::Span;
 
 /// An enum representing a diagnostic level.
 #[derive(Copy, Clone, Debug)]
@@ -146,15 +146,15 @@ impl Diagnostic {
 
     /// Emit the diagnostic.
     pub fn emit(self) {
-        fn to_internal(spans: Vec<Span>) -> crate::proc_macro::bridge::client::MultiSpan {
-            let mut multi_span = crate::proc_macro::bridge::client::MultiSpan::new();
+        fn to_internal(spans: Vec<Span>) -> super::bridge::client::MultiSpan {
+            let mut multi_span = super::bridge::client::MultiSpan::new();
             for span in spans {
                 multi_span.push(span.0);
             }
             multi_span
         }
 
-        let mut diag = crate::proc_macro::bridge::client::Diagnostic::new(
+        let mut diag = super::bridge::client::Diagnostic::new(
             self.level,
             &self.message[..],
             to_internal(self.spans),