Don't create an empty LazyTokenStream

Aaron1011 · Aaron1011 · commit 920bed121337 · 2020-10-22T10:09:08.000-04:00
diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs
@@ -72,7 +72,7 @@ impl<'a> Parser<'a> {
                 }
             })?;
             if let Some(mut attr) = attr {
-                attr.tokens = Some(tokens);
+                attr.tokens = tokens;
                 attrs.push(attr);
             } else {
                 break;
@@ -176,7 +176,7 @@ impl<'a> Parser<'a> {
             };
             if capture_tokens {
                 let (mut item, tokens) = self.collect_tokens(do_parse)?;
-                item.tokens = Some(tokens);
+                item.tokens = tokens;
                 item
             } else {
                 do_parse(self)?
@@ -213,7 +213,7 @@ impl<'a> Parser<'a> {
                 }
             })?;
             if let Some(mut attr) = attr {
-                attr.tokens = Some(tokens);
+                attr.tokens = tokens;
                 attrs.push(attr);
             } else {
                 break;
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
@@ -1116,7 +1116,7 @@ impl<'a> Parser<'a> {
     ) -> PResult<'a, P<Expr>> {
         if needs_tokens {
             let (mut expr, tokens) = self.collect_tokens(f)?;
-            expr.tokens = Some(tokens);
+            expr.tokens = tokens;
             Ok(expr)
         } else {
             f(self)
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
@@ -151,7 +151,7 @@ impl<'a> Parser<'a> {
         if let Some(tokens) = tokens {
             if let Some(item) = &mut item {
                 if !item.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) {
-                    item.tokens = Some(tokens);
+                    item.tokens = tokens;
                 }
             }
         }
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
@@ -1178,8 +1178,9 @@ impl<'a> Parser<'a> {
 
     /// Records all tokens consumed by the provided callback,
     /// including the current token. These tokens are collected
-    /// into a `TokenStream`, and returned along with the result
-    /// of the callback.
+    /// into a `LazyTokenStream`, and returned along with the result
+    /// of the callback. The returned `LazyTokenStream` will be `None`
+    /// if not tokens were captured.
     ///
     /// Note: If your callback consumes an opening delimiter
     /// (including the case where you call `collect_tokens`
@@ -1195,7 +1196,7 @@ impl<'a> Parser<'a> {
     pub fn collect_tokens<R>(
         &mut self,
         f: impl FnOnce(&mut Self) -> PResult<'a, R>,
-    ) -> PResult<'a, (R, LazyTokenStream)> {
+    ) -> PResult<'a, (R, Option<LazyTokenStream>)> {
         let start_token = (self.token.clone(), self.token_spacing);
         let mut cursor_snapshot = self.token_cursor.clone();
 
@@ -1205,6 +1206,11 @@ impl<'a> Parser<'a> {
         let num_calls = new_calls - cursor_snapshot.num_next_calls;
         let desugar_doc_comments = self.desugar_doc_comments;
 
+        // We didn't capture any tokens
+        if num_calls == 0 {
+            return Ok((ret, None));
+        }
+
         // Produces a `TokenStream` on-demand. Using `cursor_snapshot`
         // and `num_calls`, we can reconstruct the `TokenStream` seen
         // by the callback. This allows us to avoid producing a `TokenStream`
@@ -1233,7 +1239,7 @@ impl<'a> Parser<'a> {
         };
         let stream = LazyTokenStream::new(LazyTokenStreamInner::Lazy(Box::new(lazy_cb)));
 
-        Ok((ret, stream))
+        Ok((ret, Some(stream)))
     }
 
     /// `::{` or `::*`
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -103,7 +103,7 @@ impl<'a> Parser<'a> {
                     // If we captured tokens during parsing (due to outer attributes),
                     // use those.
                     if item.tokens.is_none() {
-                        item.tokens = Some(tokens);
+                        item.tokens = tokens;
                     }
                     token::NtItem(item)
                 }
@@ -115,7 +115,7 @@ impl<'a> Parser<'a> {
                 let (mut block, tokens) = self.collect_tokens(|this| this.parse_block())?;
                 // We have have eaten an NtBlock, which could already have tokens
                 if block.tokens.is_none() {
-                    block.tokens = Some(tokens);
+                    block.tokens = tokens;
                 }
                 token::NtBlock(block)
             }
@@ -124,7 +124,7 @@ impl<'a> Parser<'a> {
                 match stmt {
                     Some(mut s) => {
                         if s.tokens.is_none() {
-                            s.tokens = Some(tokens);
+                            s.tokens = tokens;
                         }
                         token::NtStmt(s)
                     }
@@ -137,7 +137,7 @@ impl<'a> Parser<'a> {
                 let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?;
                 // We have have eaten an NtPat, which could already have tokens
                 if pat.tokens.is_none() {
-                    pat.tokens = Some(tokens);
+                    pat.tokens = tokens;
                 }
                 token::NtPat(pat)
             }
@@ -146,7 +146,7 @@ impl<'a> Parser<'a> {
                 // If we captured tokens during parsing (due to outer attributes),
                 // use those.
                 if expr.tokens.is_none() {
-                    expr.tokens = Some(tokens);
+                    expr.tokens = tokens;
                 }
                 token::NtExpr(expr)
             }
@@ -155,15 +155,15 @@ impl<'a> Parser<'a> {
                     self.collect_tokens(|this| this.parse_literal_maybe_minus())?;
                 // We have have eaten a nonterminal, which  could already have tokens
                 if lit.tokens.is_none() {
-                    lit.tokens = Some(tokens);
+                    lit.tokens = tokens;
                 }
                 token::NtLiteral(lit)
             }
             NonterminalKind::Ty => {
                 let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?;
                 // We have an eaten an NtTy, which could already have tokens
                 if ty.tokens.is_none() {
-                    ty.tokens = Some(tokens);
+                    ty.tokens = tokens;
                 }
                 token::NtTy(ty)
             }
@@ -183,15 +183,15 @@ impl<'a> Parser<'a> {
                     self.collect_tokens(|this| this.parse_path(PathStyle::Type))?;
                 // We have have eaten an NtPath, which could already have tokens
                 if path.tokens.is_none() {
-                    path.tokens = Some(tokens);
+                    path.tokens = tokens;
                 }
                 token::NtPath(path)
             }
             NonterminalKind::Meta => {
                 let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item(false))?;
                 // We may have eaten a nonterminal, which could already have tokens
                 if attr.tokens.is_none() {
-                    attr.tokens = Some(tokens);
+                    attr.tokens = tokens;
                 }
                 token::NtMeta(P(attr))
             }
@@ -201,7 +201,7 @@ impl<'a> Parser<'a> {
                     self.collect_tokens(|this| this.parse_visibility(FollowedByType::Yes))?;
                 // We may have etan an `NtVis`, which could already have tokens
                 if vis.tokens.is_none() {
-                    vis.tokens = Some(tokens);
+                    vis.tokens = tokens;
                 }
                 token::NtVis(vis)
             }

Original file line number	Diff line number	Diff line change
`@@ -151,7 +151,7 @@ impl<'a> Parser<'a> {`
`151`	`151`	`if let Some(tokens) = tokens {`
`152`	`152`	`if let Some(item) = &mut item {`
`153`	`153`	`if !item.attrs.iter().any(\|attr\| attr.style == AttrStyle::Inner) {`
`154`		`- item.tokens = Some(tokens);`
	`154`	`+ item.tokens = tokens;`
`155`	`155`	`}`
`156`	`156`	`}`
`157`	`157`	`}`
Original file line number	Diff line number	Diff line change
`@@ -103,7 +103,7 @@ impl<'a> Parser<'a> {`
`103`	`103`	`// If we captured tokens during parsing (due to outer attributes),`
`104`	`104`	`// use those.`
`105`	`105`	`if item.tokens.is_none() {`
`106`		`- item.tokens = Some(tokens);`
	`106`	`+ item.tokens = tokens;`
`107`	`107`	`}`
`108`	`108`	`token::NtItem(item)`
`109`	`109`	`}`
`@@ -115,7 +115,7 @@ impl<'a> Parser<'a> {`
`115`	`115`	`let (mut block, tokens) = self.collect_tokens(\|this\| this.parse_block())?;`
`116`	`116`	`// We have have eaten an NtBlock, which could already have tokens`
`117`	`117`	`if block.tokens.is_none() {`
`118`		`- block.tokens = Some(tokens);`
	`118`	`+ block.tokens = tokens;`
`119`	`119`	`}`
`120`	`120`	`token::NtBlock(block)`
`121`	`121`	`}`
`@@ -124,7 +124,7 @@ impl<'a> Parser<'a> {`
`124`	`124`	`match stmt {`
`125`	`125`	`Some(mut s) => {`
`126`	`126`	`if s.tokens.is_none() {`
`127`		`- s.tokens = Some(tokens);`
	`127`	`+ s.tokens = tokens;`
`128`	`128`	`}`
`129`	`129`	`token::NtStmt(s)`
`130`	`130`	`}`
`@@ -137,7 +137,7 @@ impl<'a> Parser<'a> {`
`137`	`137`	`let (mut pat, tokens) = self.collect_tokens(\|this\| this.parse_pat(None))?;`
`138`	`138`	`// We have have eaten an NtPat, which could already have tokens`
`139`	`139`	`if pat.tokens.is_none() {`
`140`		`- pat.tokens = Some(tokens);`
	`140`	`+ pat.tokens = tokens;`
`141`	`141`	`}`
`142`	`142`	`token::NtPat(pat)`
`143`	`143`	`}`
`@@ -146,7 +146,7 @@ impl<'a> Parser<'a> {`
`146`	`146`	`// If we captured tokens during parsing (due to outer attributes),`
`147`	`147`	`// use those.`
`148`	`148`	`if expr.tokens.is_none() {`
`149`		`- expr.tokens = Some(tokens);`
	`149`	`+ expr.tokens = tokens;`
`150`	`150`	`}`
`151`	`151`	`token::NtExpr(expr)`
`152`	`152`	`}`
`@@ -155,15 +155,15 @@ impl<'a> Parser<'a> {`
`155`	`155`	`self.collect_tokens(\|this\| this.parse_literal_maybe_minus())?;`
`156`	`156`	`// We have have eaten a nonterminal, which could already have tokens`
`157`	`157`	`if lit.tokens.is_none() {`
`158`		`- lit.tokens = Some(tokens);`
	`158`	`+ lit.tokens = tokens;`
`159`	`159`	`}`
`160`	`160`	`token::NtLiteral(lit)`
`161`	`161`	`}`
`162`	`162`	`NonterminalKind::Ty => {`
`163`	`163`	`let (mut ty, tokens) = self.collect_tokens(\|this\| this.parse_ty())?;`
`164`	`164`	`// We have an eaten an NtTy, which could already have tokens`
`165`	`165`	`if ty.tokens.is_none() {`
`166`		`- ty.tokens = Some(tokens);`
	`166`	`+ ty.tokens = tokens;`
`167`	`167`	`}`
`168`	`168`	`token::NtTy(ty)`
`169`	`169`	`}`
`@@ -183,15 +183,15 @@ impl<'a> Parser<'a> {`
`183`	`183`	`self.collect_tokens(\|this\| this.parse_path(PathStyle::Type))?;`
`184`	`184`	`// We have have eaten an NtPath, which could already have tokens`
`185`	`185`	`if path.tokens.is_none() {`
`186`		`- path.tokens = Some(tokens);`
	`186`	`+ path.tokens = tokens;`
`187`	`187`	`}`
`188`	`188`	`token::NtPath(path)`
`189`	`189`	`}`
`190`	`190`	`NonterminalKind::Meta => {`
`191`	`191`	`let (mut attr, tokens) = self.collect_tokens(\|this\| this.parse_attr_item(false))?;`
`192`	`192`	`// We may have eaten a nonterminal, which could already have tokens`
`193`	`193`	`if attr.tokens.is_none() {`
`194`		`- attr.tokens = Some(tokens);`
	`194`	`+ attr.tokens = tokens;`
`195`	`195`	`}`
`196`	`196`	`token::NtMeta(P(attr))`
`197`	`197`	`}`
`@@ -201,7 +201,7 @@ impl<'a> Parser<'a> {`
`201`	`201`	`self.collect_tokens(\|this\| this.parse_visibility(FollowedByType::Yes))?;`
`202`	`202`	// We may have etan an `NtVis`, which could already have tokens
`203`	`203`	`if vis.tokens.is_none() {`
`204`		`- vis.tokens = Some(tokens);`
	`204`	`+ vis.tokens = tokens;`
`205`	`205`	`}`
`206`	`206`	`token::NtVis(vis)`
`207`	`207`	`}`