syntax: Introduce a struct MacArgs for macro arguments

Author: petrochenkov

src/librustc_lint/builtin.rs

@@ -1453,7 +1453,7 @@ impl EarlyLintPass for KeywordIdents {
         self.check_tokens(cx, mac_def.stream());
     }
     fn check_mac(&mut self, cx: &EarlyContext<'_>, mac: &ast::Mac) {
-        self.check_tokens(cx, mac.tts.clone().into());
+        self.check_tokens(cx, mac.args.inner_tokens());
     }
     fn check_ident(&mut self, cx: &EarlyContext<'_>, ident: ast::Ident) {
         self.check_ident_token(cx, UnderMacro(false), ident);

src/librustc_parse/parser/attr.rs

@@ -244,7 +244,7 @@ impl<'a> Parser<'a> {
         Ok(attrs)
     }
 
-    fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
+    pub(super) fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
         let lit = self.parse_lit()?;
         debug!("checking if {:?} is unusuffixed", lit);
 

src/librustc_parse/parser/expr.rs

@@ -922,12 +922,11 @@ impl<'a> Parser<'a> {
                     // `!`, as an operator, is prefix, so we know this isn't that.
                     if self.eat(&token::Not) {
                         // MACRO INVOCATION expression
-                        let (delim, tts) = self.expect_delimited_token_tree()?;
+                        let args = self.parse_mac_args()?;
                         hi = self.prev_span;
                         ex = ExprKind::Mac(Mac {
                             path,
-                            tts,
-                            delim,
+                            args,
                             span: lo.to(hi),
                             prior_type_ascription: self.last_type_ascription,
                         });

src/librustc_parse/parser/item.rs

@@ -8,7 +8,7 @@ use syntax::ast::{ItemKind, ImplItem, ImplItemKind, TraitItem, TraitItemKind, Us
 use syntax::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness, Extern, StrLit};
 use syntax::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind};
 use syntax::ast::{Ty, TyKind, Generics, TraitRef, EnumDef, Variant, VariantData, StructField};
-use syntax::ast::{Mac, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
+use syntax::ast::{Mac, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::ThinVec;
@@ -437,16 +437,15 @@ impl<'a> Parser<'a> {
             // Item macro
             let path = self.parse_path(PathStyle::Mod)?;
             self.expect(&token::Not)?;
-            let (delim, tts) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace && !self.eat(&token::Semi) {
+            let args = self.parse_mac_args()?;
+            if args.need_semicolon() && !self.eat(&token::Semi) {
                 self.report_invalid_macro_expansion_item();
             }
 
             let hi = self.prev_span;
             let mac = Mac {
                 path,
-                tts,
-                delim,
+                args,
                 span: mac_lo.to(hi),
                 prior_type_ascription: self.last_type_ascription,
             };
@@ -518,15 +517,14 @@ impl<'a> Parser<'a> {
             *at_end = true;
 
             // eat a matched-delimiter token tree:
-            let (delim, tts) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace {
+            let args = self.parse_mac_args()?;
+            if args.need_semicolon() {
                 self.expect_semi()?;
             }
 
             Ok(Some(Mac {
                 path,
-                tts,
-                delim,
+                args,
                 span: lo.to(self.prev_span),
                 prior_type_ascription: self.last_type_ascription,
             }))
@@ -1660,12 +1658,12 @@ impl<'a> Parser<'a> {
             self.bump();
 
             let ident = self.parse_ident()?;
-            let (delim, tokens) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace && !self.eat(&token::Semi) {
+            let args = self.parse_mac_args()?;
+            if args.need_semicolon() && !self.eat(&token::Semi) {
                 self.report_invalid_macro_expansion_item();
             }
 
-            (ident, ast::MacroDef { tokens, legacy: true })
+            (ident, ast::MacroDef { tokens: args.inner_tokens(), legacy: true })
         } else {
             return Ok(None);
         };

src/librustc_parse/parser/mod.rs

@@ -16,7 +16,7 @@ use crate::lexer::UnmatchedBrace;
 
 use syntax::ast::{
     self, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Extern, Ident, StrLit,
-    IsAsync, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety,
+    IsAsync, MacArgs, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety,
 };
 
 use syntax::print::pprust;
@@ -1010,27 +1010,49 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> {
-        let delim = match self.token.kind {
-            token::OpenDelim(delim) => delim,
-            _ => {
-                let msg = "expected open delimiter";
-                let mut err = self.fatal(msg);
-                err.span_label(self.token.span, msg);
-                return Err(err)
+    fn parse_mac_args(&mut self) -> PResult<'a, P<MacArgs>> {
+        self.parse_mac_args_common(true)
+    }
+
+    #[allow(dead_code)]
+    fn parse_attr_args(&mut self) -> PResult<'a, P<MacArgs>> {
+        self.parse_mac_args_common(false)
+    }
+
+    fn parse_mac_args_common(&mut self, delimited_only: bool) -> PResult<'a, P<MacArgs>> {
+        Ok(P(if self.check(&token::OpenDelim(DelimToken::Paren)) ||
+                       self.check(&token::OpenDelim(DelimToken::Bracket)) ||
+                       self.check(&token::OpenDelim(DelimToken::Brace)) {
+            match self.parse_token_tree() {
+                TokenTree::Delimited(dspan, delim, tokens) =>
+                    MacArgs::Delimited(dspan, MacDelimiter::from_token(delim), tokens),
+                _ => unreachable!(),
             }
-        };
-        let tts = match self.parse_token_tree() {
-            TokenTree::Delimited(_, _, tts) => tts,
-            _ => unreachable!(),
-        };
-        let delim = match delim {
-            token::Paren => MacDelimiter::Parenthesis,
-            token::Bracket => MacDelimiter::Bracket,
-            token::Brace => MacDelimiter::Brace,
-            token::NoDelim => self.bug("unexpected no delimiter"),
-        };
-        Ok((delim, tts.into()))
+        } else if !delimited_only {
+            if self.eat(&token::Eq) {
+                let eq_span = self.prev_span;
+                let mut is_interpolated_expr = false;
+                if let token::Interpolated(nt) = &self.token.kind {
+                    if let token::NtExpr(..) = **nt {
+                        is_interpolated_expr = true;
+                    }
+                }
+                let token_tree = if is_interpolated_expr {
+                    // We need to accept arbitrary interpolated expressions to continue
+                    // supporting things like `doc = $expr` that work on stable.
+                    // Non-literal interpolated expressions are rejected after expansion.
+                    self.parse_token_tree()
+                } else {
+                    self.parse_unsuffixed_lit()?.token_tree()
+                };
+
+                MacArgs::Eq(eq_span, token_tree.into())
+            } else {
+                MacArgs::Empty
+            }
+        } else {
+            return self.unexpected();
+        }))
     }
 
     fn parse_or_use_outer_attributes(

src/librustc_parse/parser/pat.rs

@@ -595,11 +595,10 @@ impl<'a> Parser<'a> {
     /// Parse macro invocation
     fn parse_pat_mac_invoc(&mut self, lo: Span, path: Path) -> PResult<'a, PatKind> {
         self.bump();
-        let (delim, tts) = self.expect_delimited_token_tree()?;
+        let args = self.parse_mac_args()?;
         let mac = Mac {
             path,
-            tts,
-            delim,
+            args,
             span: lo.to(self.prev_span),
             prior_type_ascription: self.last_type_ascription,
         };

src/librustc_parse/parser/stmt.rs

@@ -10,7 +10,7 @@ use syntax::ThinVec;
 use syntax::ptr::P;
 use syntax::ast;
 use syntax::ast::{DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind};
-use syntax::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter};
+use syntax::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac};
 use syntax::util::classify;
 use syntax::token;
 use syntax::source_map::{respan, Span};
@@ -93,23 +93,23 @@ impl<'a> Parser<'a> {
                 }));
             }
 
-            let (delim, tts) = self.expect_delimited_token_tree()?;
+            let args = self.parse_mac_args()?;
+            let delim = args.delim();
             let hi = self.prev_span;
 
-            let style = if delim == MacDelimiter::Brace {
+            let style = if delim == token::Brace {
                 MacStmtStyle::Braces
             } else {
                 MacStmtStyle::NoBraces
             };
 
             let mac = Mac {
                 path,
-                tts,
-                delim,
+                args,
                 span: lo.to(hi),
                 prior_type_ascription: self.last_type_ascription,
             };
-            let kind = if delim == MacDelimiter::Brace ||
+            let kind = if delim == token::Brace ||
                           self.token == token::Semi || self.token == token::Eof {
                 StmtKind::Mac(P((mac, style, attrs.into())))
             }

src/librustc_parse/parser/ty.rs

@@ -177,11 +177,10 @@ impl<'a> Parser<'a> {
             let path = self.parse_path(PathStyle::Type)?;
             if self.eat(&token::Not) {
                 // Macro invocation in type position
-                let (delim, tts) = self.expect_delimited_token_tree()?;
+                let args = self.parse_mac_args()?;
                 let mac = Mac {
                     path,
-                    tts,
-                    delim,
+                    args,
                     span: lo.to(self.prev_span),
                     prior_type_ascription: self.last_type_ascription,
                 };

src/libsyntax/ast.rs

@@ -27,7 +27,7 @@ pub use syntax_pos::symbol::{Ident, Symbol as Name};
 use crate::ptr::P;
 use crate::source_map::{dummy_spanned, respan, Spanned};
 use crate::token::{self, DelimToken};
-use crate::tokenstream::TokenStream;
+use crate::tokenstream::{TokenStream, TokenTree, DelimSpan};
 
 use syntax_pos::symbol::{kw, sym, Symbol};
 use syntax_pos::{Span, DUMMY_SP, ExpnId};
@@ -40,6 +40,7 @@ use rustc_index::vec::Idx;
 use rustc_serialize::{self, Decoder, Encoder};
 use rustc_macros::HashStable_Generic;
 
+use std::iter;
 use std::fmt;
 
 #[cfg(test)]
@@ -1372,34 +1373,78 @@ pub enum Movability {
     Movable,
 }
 
-/// Represents a macro invocation. The `Path` indicates which macro
-/// is being invoked, and the vector of token-trees contains the source
-/// of the macro invocation.
-///
-/// N.B., the additional ident for a `macro_rules`-style macro is actually
-/// stored in the enclosing item.
+/// Represents a macro invocation. The `path` indicates which macro
+/// is being invoked, and the `args` are arguments passed to it.
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
 pub struct Mac {
     pub path: Path,
-    pub delim: MacDelimiter,
-    pub tts: TokenStream,
+    pub args: P<MacArgs>,
     pub span: Span,
     pub prior_type_ascription: Option<(Span, bool)>,
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Debug)]
-pub enum MacDelimiter {
-    Parenthesis,
-    Bracket,
-    Brace,
+/// Arguments passed to an attribute or a function-like macro.
+#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
+pub enum MacArgs {
+    /// No arguments - `#[attr]`.
+    Empty,
+    /// Delimited arguments - `#[attr()/[]/{}]` or `mac!()/[]/{}`.
+    Delimited(DelimSpan, MacDelimiter, TokenStream),
+    /// Arguments of a key-value attribute - `#[attr = "value"]`.
+    /// Span belongs to the `=` token, token stream is the "value".
+    Eq(Span, TokenStream),
+}
+
+impl MacArgs {
+    pub fn delim(&self) -> DelimToken {
+        match self {
+            MacArgs::Delimited(_, delim, _) => delim.to_token(),
+            MacArgs::Empty | MacArgs::Eq(..) => token::NoDelim,
+        }
+    }
+
+    /// Tokens inside the delimiters or after `=`.
+    /// Proc macros see these tokens, for example.
+    pub fn inner_tokens(&self) -> TokenStream {
+        match self {
+            MacArgs::Empty => TokenStream::default(),
+            MacArgs::Delimited(.., tokens) => tokens.clone(),
+            MacArgs::Eq(.., tokens) => tokens.clone(),
+        }
+    }
+
+    /// Tokens together with the delimiters or `=`.
+    /// Use of this functions generally means that something suspicious or hacky is happening.
+    pub fn outer_tokens(&self) -> TokenStream {
+        match *self {
+            MacArgs::Empty => TokenStream::default(),
+            MacArgs::Delimited(dspan, delim, ref tokens) =>
+                TokenTree::Delimited(dspan, delim.to_token(), tokens.clone()).into(),
+            MacArgs::Eq(eq_span, ref tokens) => iter::once(TokenTree::token(token::Eq, eq_span))
+                                                .chain(tokens.trees()).collect(),
+        }
+    }
+
+    /// Whether a macro with these arguments needs a semicolon
+    /// when used as a standalone item or statement.
+    pub fn need_semicolon(&self) -> bool {
+        !matches!(self, MacArgs::Delimited(_, MacDelimiter::Brace ,_))
+    }
 }
 
 impl Mac {
     pub fn stream(&self) -> TokenStream {
-        self.tts.clone()
+        self.args.inner_tokens()
     }
 }
 
+#[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Debug)]
+pub enum MacDelimiter {
+    Parenthesis,
+    Bracket,
+    Brace,
+}
+
 impl MacDelimiter {
     crate fn to_token(self) -> DelimToken {
         match self {
@@ -1408,6 +1453,15 @@ impl MacDelimiter {
             MacDelimiter::Brace => DelimToken::Brace,
         }
     }
+
+    pub fn from_token(delim: DelimToken) -> MacDelimiter {
+        match delim {
+            token::Paren => MacDelimiter::Parenthesis,
+            token::Bracket => MacDelimiter::Bracket,
+            token::Brace => MacDelimiter::Brace,
+            token::NoDelim => panic!("expected a delimiter"),
+        }
+    }
 }
 
 /// Represents a macro definition.

src/libsyntax/lib.rs

@@ -12,6 +12,7 @@
 #![feature(const_transmute)]
 #![feature(crate_visibility_modifier)]
 #![feature(label_break_value)]
+#![feature(matches_macro)]
 #![feature(nll)]
 #![feature(try_trait)]
 #![feature(slice_patterns)]