Fix DebugParser.

It currently doesn't work at all. This commit changes it to a simpler
imperative style that produces a valid `tokens` vec.

(An aside: I find `Iterator::scan` to be a pretty wretched function,
that produces code which is very hard to understand. Probably why this
is just one of two uses of it in the entire compiler.)

Author: nnethercote

compiler/rustc_parse/src/parser/mod.rs

@@ -1537,14 +1537,16 @@ impl<'a> Parser<'a> {
 
                 // we don't need N spans, but we want at least one, so print all of prev_token
                 dbg_fmt.field("prev_token", &parser.prev_token);
-                // make it easier to peek farther ahead by taking TokenKinds only until EOF
-                let tokens = (0..*lookahead)
-                    .map(|i| parser.look_ahead(i, |tok| tok.kind.clone()))
-                    .scan(parser.prev_token == TokenKind::Eof, |eof, tok| {
-                        let current = eof.then_some(tok.clone()); // include a trailing EOF token
-                        *eof |= &tok == &TokenKind::Eof;
-                        current
-                    });
+                let mut tokens = vec![];
+                for i in 0..*lookahead {
+                    let tok = parser.look_ahead(i, |tok| tok.kind.clone());
+                    let is_eof = tok == TokenKind::Eof;
+                    tokens.push(tok);
+                    if is_eof {
+                        // Don't look ahead past EOF.
+                        break;
+                    }
+                }
                 dbg_fmt.field_with("tokens", |field| field.debug_list().entries(tokens).finish());
                 dbg_fmt.field("approx_token_stream_pos", &parser.num_bump_calls);
 

compiler/rustc_parse/src/parser/tests.rs

@@ -1541,11 +1541,36 @@ fn debug_lookahead() {
             ctxt: #0,
         },
     },
-    tokens: [],
+    tokens: [
+        Ident(
+            \"fn\",
+            No,
+        ),
+        Ident(
+            \"f\",
+            No,
+        ),
+        OpenDelim(
+            Parenthesis,
+        ),
+        Ident(
+            \"x\",
+            No,
+        ),
+        Colon,
+        Ident(
+            \"u32\",
+            No,
+        ),
+        CloseDelim(
+            Parenthesis,
+        ),
+    ],
     approx_token_stream_pos: 1,
     ..
 }"
         );
+        // There are 13 tokens. We request 15, get 14; the last one is `Eof`.
         assert_eq!(
             &format!("{:#?}", p.debug_lookahead(15)),
             "Parser {
@@ -1561,7 +1586,51 @@ fn debug_lookahead() {
             ctxt: #0,
         },
     },
-    tokens: [],
+    tokens: [
+        Ident(
+            \"fn\",
+            No,
+        ),
+        Ident(
+            \"f\",
+            No,
+        ),
+        OpenDelim(
+            Parenthesis,
+        ),
+        Ident(
+            \"x\",
+            No,
+        ),
+        Colon,
+        Ident(
+            \"u32\",
+            No,
+        ),
+        CloseDelim(
+            Parenthesis,
+        ),
+        OpenDelim(
+            Brace,
+        ),
+        Ident(
+            \"x\",
+            No,
+        ),
+        CloseDelim(
+            Brace,
+        ),
+        Ident(
+            \"struct\",
+            No,
+        ),
+        Ident(
+            \"S\",
+            No,
+        ),
+        Semi,
+        Eof,
+    ],
     approx_token_stream_pos: 1,
     ..
 }"
@@ -1588,7 +1657,12 @@ fn debug_lookahead() {
             ctxt: #0,
         },
     },
-    tokens: [],
+    tokens: [
+        Ident(
+            \"x\",
+            No,
+        ),
+    ],
     approx_token_stream_pos: 9,
     ..
 }"
@@ -1610,7 +1684,23 @@ fn debug_lookahead() {
             ctxt: #0,
         },
     },
-    tokens: [],
+    tokens: [
+        Ident(
+            \"x\",
+            No,
+        ),
+        CloseDelim(
+            Brace,
+        ),
+        Ident(
+            \"struct\",
+            No,
+        ),
+        Ident(
+            \"S\",
+            No,
+        ),
+    ],
     approx_token_stream_pos: 9,
     ..
 }"
@@ -1637,8 +1727,6 @@ fn debug_lookahead() {
     },
     tokens: [
         Eof,
-        Eof,
-        Eof,
     ],
     approx_token_stream_pos: 15,
     ..