Auto merge of #63674 - petrochenkov:meta2, r=<try>

syntax: Support modern attribute syntax in the `meta` matcher

Where "modern" means #57367:
```
PATH
PATH `(` TOKEN_STREAM `)`
PATH `[` TOKEN_STREAM `]`
PATH `{` TOKEN_STREAM `}`
```

Unfortunately, `meta` wasn't future-proofed using the `FOLLOW` token set like other matchers (#34011), so code like `$meta:meta {` or `$meta:meta [` may break, and we need a crater run to find out how often this happens in practice.

Closes #49629 (by fully supporting `meta` rather than removing it.)

Author: bors

src/librustc/hir/lowering.rs

@@ -986,10 +986,12 @@ impl<'a> LoweringContext<'a> {
         // lower attributes (we use the AST version) there is nowhere to keep
         // the `HirId`s. We don't actually need HIR version of attributes anyway.
         Attribute {
+            item: AttrItem {
+                path: attr.path.clone(),
+                tokens: self.lower_token_stream(attr.tokens.clone()),
+            },
             id: attr.id,
             style: attr.style,
-            path: attr.path.clone(),
-            tokens: self.lower_token_stream(attr.tokens.clone()),
             is_sugared_doc: attr.is_sugared_doc,
             span: attr.span,
         }

src/librustc/ich/impls_syntax.rs

@@ -227,6 +227,11 @@ impl<'a> HashStable<StableHashingContext<'a>> for ast::Path {
     }
 }
 
+impl_stable_hash_for!(struct ::syntax::ast::AttrItem {
+    path,
+    tokens,
+});
+
 impl<'a> HashStable<StableHashingContext<'a>> for ast::Attribute {
     fn hash_stable<W: StableHasherResult>(&self,
                                           hcx: &mut StableHashingContext<'a>,
@@ -236,19 +241,15 @@ impl<'a> HashStable<StableHashingContext<'a>> for ast::Attribute {
         debug_assert!(!self.is_sugared_doc);
 
         let ast::Attribute {
+            ref item,
             id: _,
             style,
-            ref path,
-            ref tokens,
             is_sugared_doc: _,
             span,
         } = *self;
 
+        item.hash_stable(hcx, hasher);
         style.hash_stable(hcx, hasher);
-        path.hash_stable(hcx, hasher);
-        for tt in tokens.trees() {
-            tt.hash_stable(hcx, hasher);
-        }
         span.hash_stable(hcx, hasher);
     }
 }

src/libsyntax/ast.rs

@@ -2124,18 +2124,28 @@ impl rustc_serialize::Decodable for AttrId {
     }
 }
 
+#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
+pub struct AttrItem {
+    pub path: Path,
+    pub tokens: TokenStream,
+}
+
 /// Metadata associated with an item.
 /// Doc-comments are promoted to attributes that have `is_sugared_doc = true`.
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
 pub struct Attribute {
+    pub item: AttrItem,
     pub id: AttrId,
     pub style: AttrStyle,
-    pub path: Path,
-    pub tokens: TokenStream,
     pub is_sugared_doc: bool,
     pub span: Span,
 }
 
+impl std::ops::Deref for Attribute {
+    type Target = AttrItem;
+    fn deref(&self) -> &Self::Target { &self.item }
+}
+
 /// `TraitRef`s appear in impls.
 ///
 /// Resolution maps each `TraitRef`'s `ref_id` to its defining trait; that's all

src/libsyntax/attr/mod.rs

@@ -9,7 +9,7 @@ pub use StabilityLevel::*;
 pub use crate::ast::Attribute;
 
 use crate::ast;
-use crate::ast::{AttrId, AttrStyle, Name, Ident, Path, PathSegment};
+use crate::ast::{AttrItem, AttrId, AttrStyle, Name, Ident, Path, PathSegment};
 use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem};
 use crate::ast::{Lit, LitKind, Expr, Item, Local, Stmt, StmtKind, GenericParam};
 use crate::mut_visit::visit_clobber;
@@ -254,9 +254,8 @@ impl MetaItem {
     }
 }
 
-impl Attribute {
-    /// Extracts the MetaItem from inside this Attribute.
-    pub fn meta(&self) -> Option<MetaItem> {
+impl AttrItem {
+    crate fn meta(&self, span: Span) -> Option<MetaItem> {
         let mut tokens = self.tokens.trees().peekable();
         Some(MetaItem {
             path: self.path.clone(),
@@ -268,9 +267,16 @@ impl Attribute {
             } else {
                 return None;
             },
-            span: self.span,
+            span,
         })
     }
+}
+
+impl Attribute {
+    /// Extracts the MetaItem from inside this Attribute.
+    pub fn meta(&self) -> Option<MetaItem> {
+        self.item.meta(self.span)
+    }
 
     pub fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T>
         where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>,
@@ -332,10 +338,9 @@ impl Attribute {
                 DUMMY_SP,
             );
             f(&Attribute {
+                item: AttrItem { path: meta.path, tokens: meta.node.tokens(meta.span) },
                 id: self.id,
                 style: self.style,
-                path: meta.path,
-                tokens: meta.node.tokens(meta.span),
                 is_sugared_doc: true,
                 span: self.span,
             })
@@ -384,10 +389,9 @@ crate fn mk_attr_id() -> AttrId {
 /// Returns an inner attribute with the given value and span.
 pub fn mk_attr_inner(item: MetaItem) -> Attribute {
     Attribute {
+        item: AttrItem { path: item.path, tokens: item.node.tokens(item.span) },
         id: mk_attr_id(),
         style: ast::AttrStyle::Inner,
-        path: item.path,
-        tokens: item.node.tokens(item.span),
         is_sugared_doc: false,
         span: item.span,
     }
@@ -396,10 +400,9 @@ pub fn mk_attr_inner(item: MetaItem) -> Attribute {
 /// Returns an outer attribute with the given value and span.
 pub fn mk_attr_outer(item: MetaItem) -> Attribute {
     Attribute {
+        item: AttrItem { path: item.path, tokens: item.node.tokens(item.span) },
         id: mk_attr_id(),
         style: ast::AttrStyle::Outer,
-        path: item.path,
-        tokens: item.node.tokens(item.span),
         is_sugared_doc: false,
         span: item.span,
     }
@@ -410,10 +413,12 @@ pub fn mk_sugared_doc_attr(text: Symbol, span: Span) -> Attribute {
     let lit_kind = LitKind::Str(text, ast::StrStyle::Cooked);
     let lit = Lit::from_lit_kind(lit_kind, span);
     Attribute {
+        item: AttrItem {
+            path: Path::from_ident(Ident::with_dummy_span(sym::doc).with_span_pos(span)),
+            tokens: MetaItemKind::NameValue(lit).tokens(span),
+        },
         id: mk_attr_id(),
         style,
-        path: Path::from_ident(Ident::with_dummy_span(sym::doc).with_span_pos(span)),
-        tokens: MetaItemKind::NameValue(lit).tokens(span),
         is_sugared_doc: true,
         span,
     }
@@ -502,7 +507,7 @@ impl MetaItem {
             }
             Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
                 token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
-                token::Nonterminal::NtMeta(ref meta) => return Some(meta.clone()),
+                token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
                 token::Nonterminal::NtPath(ref path) => path.clone(),
                 _ => return None,
             },
@@ -726,7 +731,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         );
 
         let start_span = parser.token.span;
-        let (path, tokens) = panictry!(parser.parse_meta_item_unrestricted());
+        let item = panictry!(parser.parse_attr_item());
         let end_span = parser.token.span;
         if parser.token != token::Eof {
             parse_sess.span_diagnostic
@@ -735,10 +740,9 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         }
 
         krate.attrs.push(Attribute {
+            item,
             id: mk_attr_id(),
             style: AttrStyle::Inner,
-            path,
-            tokens,
             is_sugared_doc: false,
             span: start_span.to(end_span),
         });

src/libsyntax/config.rs

@@ -122,8 +122,8 @@ impl<'a> StripUnconfigured<'a> {
 
             while !parser.check(&token::CloseDelim(token::Paren)) {
                 let lo = parser.token.span.lo();
-                let (path, tokens) = parser.parse_meta_item_unrestricted()?;
-                expanded_attrs.push((path, tokens, parser.prev_span.with_lo(lo)));
+                let item = parser.parse_attr_item()?;
+                expanded_attrs.push((item, parser.prev_span.with_lo(lo)));
                 parser.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Paren)])?;
             }
 
@@ -150,11 +150,10 @@ impl<'a> StripUnconfigured<'a> {
             // `cfg_attr` inside of another `cfg_attr`. E.g.
             //  `#[cfg_attr(false, cfg_attr(true, some_attr))]`.
             expanded_attrs.into_iter()
-            .flat_map(|(path, tokens, span)| self.process_cfg_attr(ast::Attribute {
+            .flat_map(|(item, span)| self.process_cfg_attr(ast::Attribute {
+                item,
                 id: attr::mk_attr_id(),
                 style: attr.style,
-                path,
-                tokens,
                 is_sugared_doc: false,
                 span,
             }))

src/libsyntax/ext/expand.rs

@@ -1,4 +1,4 @@
-use crate::ast::{self, Block, Ident, LitKind, NodeId, PatKind, Path};
+use crate::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path};
 use crate::ast::{MacStmtStyle, StmtKind, ItemKind};
 use crate::attr::{self, HasAttrs};
 use crate::source_map::respan;
@@ -526,9 +526,10 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                         Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
                         Annotatable::Expr(expr) => token::NtExpr(expr),
                     })), DUMMY_SP).into();
-                    let input = self.extract_proc_macro_attr_input(attr.tokens, span);
+                    let input = self.extract_proc_macro_attr_input(attr.item.tokens, span);
                     let tok_result = expander.expand(self.cx, span, input, item_tok);
-                    let res = self.parse_ast_fragment(tok_result, fragment_kind, &attr.path, span);
+                    let res =
+                        self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span);
                     self.gate_proc_macro_expansion(span, &res);
                     res
                 }
@@ -1327,11 +1328,10 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
 
             let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items);
             *at = attr::Attribute {
+                item: AttrItem { path: meta.path, tokens: meta.node.tokens(meta.span) },
                 span: at.span,
                 id: at.id,
                 style: at.style,
-                path: meta.path,
-                tokens: meta.node.tokens(meta.span),
                 is_sugared_doc: false,
             };
         } else {

src/libsyntax/ext/tt/macro_parser.rs

@@ -936,7 +936,7 @@ fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal {
             FatalError.raise()
         }
         sym::path => token::NtPath(panictry!(p.parse_path(PathStyle::Type))),
-        sym::meta => token::NtMeta(panictry!(p.parse_meta_item())),
+        sym::meta => token::NtMeta(panictry!(p.parse_attr_item())),
         sym::vis => token::NtVis(panictry!(p.parse_visibility(true))),
         sym::lifetime => if p.check_lifetime() {
             token::NtLifetime(p.expect_lifetime().ident)

src/libsyntax/mut_visit.rs

@@ -527,7 +527,8 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
 }
 
 pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
-    let Attribute { id: _, style: _, path, tokens, is_sugared_doc: _, span } = attr;
+    let Attribute { item: AttrItem { path, tokens }, id: _, style: _, is_sugared_doc: _, span }
+        = attr;
     vis.visit_path(path);
     vis.visit_tts(tokens);
     vis.visit_span(span);
@@ -656,7 +657,10 @@ pub fn noop_visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis:
         token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
         token::NtLifetime(ident) => vis.visit_ident(ident),
         token::NtLiteral(expr) => vis.visit_expr(expr),
-        token::NtMeta(meta) => vis.visit_meta_item(meta),
+        token::NtMeta(AttrItem { path, tokens }) => {
+            vis.visit_path(path);
+            vis.visit_tts(tokens);
+        }
         token::NtPath(path) => vis.visit_path(path),
         token::NtTT(tt) => vis.visit_tt(tt),
         token::NtImplItem(item) =>

src/libsyntax/parse/attr.rs

@@ -98,7 +98,7 @@ impl<'a> Parser<'a> {
         debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}",
                inner_parse_policy,
                self.token);
-        let (span, path, tokens, style) = match self.token.kind {
+        let (span, item, style) = match self.token.kind {
             token::Pound => {
                 let lo = self.token.span;
                 self.bump();
@@ -115,7 +115,7 @@ impl<'a> Parser<'a> {
                 };
 
                 self.expect(&token::OpenDelim(token::Bracket))?;
-                let (path, tokens) = self.parse_meta_item_unrestricted()?;
+                let item = self.parse_attr_item()?;
                 self.expect(&token::CloseDelim(token::Bracket))?;
                 let hi = self.prev_span;
 
@@ -150,7 +150,7 @@ impl<'a> Parser<'a> {
                     }
                 }
 
-                (attr_sp, path, tokens, style)
+                (attr_sp, item, style)
             }
             _ => {
                 let token_str = self.this_token_to_string();
@@ -159,10 +159,9 @@ impl<'a> Parser<'a> {
         };
 
         Ok(ast::Attribute {
+            item,
             id: attr::mk_attr_id(),
             style,
-            path,
-            tokens,
             is_sugared_doc: false,
             span,
         })
@@ -177,17 +176,17 @@ impl<'a> Parser<'a> {
     /// PATH
     /// PATH `=` TOKEN_TREE
     /// The delimiters or `=` are still put into the resulting token stream.
-    crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
-        let meta = match self.token.kind {
+    crate fn parse_attr_item(&mut self) -> PResult<'a, ast::AttrItem> {
+        let item = match self.token.kind {
             token::Interpolated(ref nt) => match **nt {
-                Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
+                Nonterminal::NtMeta(ref item) => Some(item.clone()),
                 _ => None,
             },
             _ => None,
         };
-        Ok(if let Some(meta) = meta {
+        Ok(if let Some(item) = item {
             self.bump();
-            (meta.path, meta.node.tokens(meta.span))
+            item
         } else {
             let path = self.parse_path(PathStyle::Mod)?;
             let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) ||
@@ -214,7 +213,7 @@ impl<'a> Parser<'a> {
             } else {
                 TokenStream::empty()
             };
-            (path, tokens)
+            ast::AttrItem { path, tokens }
         })
     }
 
@@ -282,9 +281,14 @@ impl<'a> Parser<'a> {
             _ => None,
         };
 
-        if let Some(meta) = nt_meta {
-            self.bump();
-            return Ok(meta);
+        if let Some(item) = nt_meta {
+            return match item.meta(item.path.span) {
+                Some(meta) => {
+                    self.bump();
+                    Ok(meta)
+                }
+                None => self.unexpected(),
+            }
         }
 
         let lo = self.token.span;

src/libsyntax/parse/parser/path.rs

@@ -114,9 +114,9 @@ impl<'a> Parser<'a> {
     pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, Path> {
         let meta_ident = match self.token.kind {
             token::Interpolated(ref nt) => match **nt {
-                token::NtMeta(ref meta) => match meta.node {
-                    ast::MetaItemKind::Word => Some(meta.path.clone()),
-                    _ => None,
+                token::NtMeta(ref item) => match item.tokens.is_empty() {
+                    true => Some(item.path.clone()),
+                    false => None,
                 },
                 _ => None,
             },