Manually cleanup token stream when macro expansion aborts.

Author: cjgillot

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -442,23 +442,38 @@ fn make_token_stream(
         }
         token_and_spacing = iter.next();
     }
-    let mut final_buf = stack.pop().expect("Missing final buf!");
-    if break_last_token {
-        let last_token = final_buf.inner.pop().unwrap();
-        if let AttrTokenTree::Token(last_token, spacing) = last_token {
-            let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
-
-            // An 'unglued' token is always two ASCII characters
-            let mut first_span = last_token.span.shrink_to_lo();
-            first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
-
-            final_buf
+    while let Some(FrameData { open_delim_sp, mut inner }) = stack.pop() {
+        // A former macro expansion could give us malformed tokens.
+        // In that case, manually close all open delimitors so downstream users
+        // don't ICE on them.
+        if let Some((delim, open_sp)) = open_delim_sp {
+            let dspan = DelimSpan::from_pair(open_sp, rustc_span::DUMMY_SP);
+            let stream = AttrTokenStream::new(inner);
+            let delimited = AttrTokenTree::Delimited(dspan, delim, stream);
+            stack
+                .last_mut()
+                .unwrap_or_else(|| panic!("Bottom token frame is missing for recovered token"))
                 .inner
-                .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing));
+                .push(delimited);
         } else {
-            panic!("Unexpected last token {:?}", last_token)
+            if break_last_token {
+                let last_token = inner.pop().unwrap();
+                if let AttrTokenTree::Token(last_token, spacing) = last_token {
+                    let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
+
+                    // An 'unglued' token is always two ASCII characters
+                    let mut first_span = last_token.span.shrink_to_lo();
+                    first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
+
+                    inner
+                        .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing));
+                } else {
+                    panic!("Unexpected last token {:?}", last_token)
+                }
+            }
+            assert!(stack.is_empty(), "Stack should be empty: stack={:?}", stack);
+            return AttrTokenStream::new(inner);
         }
     }
-    assert!(stack.is_empty(), "Stack should be empty: final_buf={:?} stack={:?}", final_buf, stack);
-    AttrTokenStream::new(final_buf.inner)
+    panic!("Missing final buf!")
 }

src/test/ui/macros/syntax-error-recovery.rs

@@ -0,0 +1,17 @@
+macro_rules! values {
+    ($($token:ident($value:literal) $(as $inner:ty)? => $attr:meta,)*) => {
+        #[derive(Debug)]
+        pub enum TokenKind {
+            $(
+                #[$attr]
+                $token $($inner)? = $value,
+            )*
+        }
+    };
+}
+//~^^^^^ ERROR expected one of `(`, `,`, `=`, `{`, or `}`, found `(String)`
+//~| ERROR macro expansion ignores token `(String)` and any following
+
+values!(STRING(1) as (String) => cfg(test),);
+
+fn main() {}

src/test/ui/macros/syntax-error-recovery.stderr

@@ -0,0 +1,24 @@
+error: expected one of `(`, `,`, `=`, `{`, or `}`, found `(String)`
+  --> $DIR/syntax-error-recovery.rs:7:26
+   |
+LL |                 $token $($inner)? = $value,
+   |                          ^^^^^^ expected one of `(`, `,`, `=`, `{`, or `}`
+...
+LL | values!(STRING(1) as (String) => cfg(test),);
+   | -------------------------------------------- in this macro invocation
+   |
+   = note: this error originates in the macro `values` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: macro expansion ignores token `(String)` and any following
+  --> $DIR/syntax-error-recovery.rs:7:26
+   |
+LL |                 $token $($inner)? = $value,
+   |                          ^^^^^^
+...
+LL | values!(STRING(1) as (String) => cfg(test),);
+   | -------------------------------------------- caused by the macro expansion here
+   |
+   = note: the usage of `values!` is likely invalid in item context
+
+error: aborting due to 2 previous errors
+