Attach tokens to NtMeta (ast::AttrItem)

An `AttrItem` does not have outer attributes, so we only capture tokens
when parsing a `macro_rules!` matcher

Author: Aaron1011

compiler/rustc_ast/src/ast.rs

@@ -2419,6 +2419,7 @@ impl<D: Decoder> rustc_serialize::Decodable<D> for AttrId {
 pub struct AttrItem {
     pub path: Path,
     pub args: MacArgs,
+    pub tokens: Option<TokenStream>,
 }
 
 /// A list of attributes.

compiler/rustc_ast/src/attr/mod.rs

@@ -330,7 +330,7 @@ crate fn mk_attr_id() -> AttrId {
 }
 
 pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attribute {
-    mk_attr_from_item(style, AttrItem { path, args }, span)
+    mk_attr_from_item(style, AttrItem { path, args, tokens: None }, span)
 }
 
 pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute {

compiler/rustc_ast/src/mut_visit.rs

@@ -579,7 +579,7 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
 pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
     let Attribute { kind, id: _, style: _, span } = attr;
     match kind {
-        AttrKind::Normal(AttrItem { path, args }) => {
+        AttrKind::Normal(AttrItem { path, args, tokens: _ }) => {
             vis.visit_path(path);
             visit_mac_args(args, vis);
         }
@@ -709,7 +709,7 @@ pub fn noop_visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis:
         token::NtLifetime(ident) => vis.visit_ident(ident),
         token::NtLiteral(expr) => vis.visit_expr(expr),
         token::NtMeta(item) => {
-            let AttrItem { path, args } = item.deref_mut();
+            let AttrItem { path, args, tokens: _ } = item.deref_mut();
             vis.visit_path(path);
             visit_mac_args(args, vis);
         }

compiler/rustc_ast_lowering/src/lib.rs

@@ -967,6 +967,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             AttrKind::Normal(ref item) => AttrKind::Normal(AttrItem {
                 path: item.path.clone(),
                 args: self.lower_mac_args(&item.args),
+                tokens: None,
             }),
             AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data),
         };

compiler/rustc_builtin_macros/src/cmdline_attrs.rs

@@ -15,7 +15,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         );
 
         let start_span = parser.token.span;
-        let AttrItem { path, args } = match parser.parse_attr_item() {
+        let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item() {
             Ok(ai) => ai,
             Err(mut err) => {
                 err.emit();

compiler/rustc_expand/src/expand.rs

@@ -1777,6 +1777,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
                 kind: ast::AttrKind::Normal(AttrItem {
                     path: meta.path,
                     args: meta.kind.mac_args(meta.span),
+                    tokens: None,
                 }),
                 span: at.span,
                 id: at.id,

compiler/rustc_parse/src/lib.rs

@@ -277,6 +277,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
         Nonterminal::NtLifetime(ident) => {
             Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())
         }
+        Nonterminal::NtMeta(ref attr) => attr.tokens.clone(),
         Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
         Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
             if expr.tokens.is_none() {

compiler/rustc_parse/src/parser/attr.rs

@@ -162,7 +162,7 @@ impl<'a> Parser<'a> {
         } else {
             let path = self.parse_path(PathStyle::Mod)?;
             let args = self.parse_attr_args()?;
-            ast::AttrItem { path, args }
+            ast::AttrItem { path, args, tokens: None }
         })
     }
 

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -169,7 +169,14 @@ impl<'a> Parser<'a> {
                 }
             }
             NonterminalKind::Path => token::NtPath(self.parse_path(PathStyle::Type)?),
-            NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item()?)),
+            NonterminalKind::Meta => {
+                let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item())?;
+                // We may have eaten a nonterminal, which could already have tokens
+                if attr.tokens.is_none() {
+                    attr.tokens = Some(tokens);
+                }
+                token::NtMeta(P(attr))
+            }
             NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
             NonterminalKind::Vis => token::NtVis(self.parse_visibility(FollowedByType::Yes)?),
             NonterminalKind::Lifetime => {