Make clear that TokenTree::Token shouldn't contain a delimiter.

Author: nnethercote

compiler/rustc_ast/src/tokenstream.rs

@@ -41,7 +41,8 @@ use std::{fmt, iter};
 /// Nothing special happens to misnamed or misplaced `SubstNt`s.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum TokenTree {
-    /// A single token.
+    /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
+    /// delimiters are implicitly represented by `Delimited`.
     Token(Token, Spacing),
     /// A delimited sequence of token trees.
     Delimited(DelimSpan, Delimiter, TokenStream),

compiler/rustc_parse/src/parser/mod.rs

@@ -290,7 +290,13 @@ impl TokenCursor {
                             self.frame.tree_cursor.replace_prev_and_rewind(desugared);
                             // Continue to get the first token of the desugared doc comment.
                         }
-                        _ => return (token.clone(), spacing),
+                        _ => {
+                            debug_assert!(!matches!(
+                                token.kind,
+                                token::OpenDelim(_) | token::CloseDelim(_)
+                            ));
+                            return (token.clone(), spacing);
+                        }
                     },
                     &TokenTree::Delimited(sp, delim, ref tts) => {
                         // Set `open_delim` to true here because we deal with it immediately.