Rename Cursor/CursorRef as TokenTreeCursor/RefTokenTreeCursor.

This makes it clear they return token trees, and makes for a nice
comparison against `TokenCursor` which returns tokens.

Author: nnethercote

compiler/rustc_ast/src/tokenstream.rs

@@ -389,12 +389,12 @@ impl TokenStream {
         self.0.len()
     }
 
-    pub fn trees(&self) -> CursorRef<'_> {
-        CursorRef::new(self)
+    pub fn trees(&self) -> RefTokenTreeCursor<'_> {
+        RefTokenTreeCursor::new(self)
     }
 
-    pub fn into_trees(self) -> Cursor {
-        Cursor::new(self)
+    pub fn into_trees(self) -> TokenTreeCursor {
+        TokenTreeCursor::new(self)
     }
 
     /// Compares two `TokenStream`s, checking equality without regarding span information.
@@ -552,24 +552,25 @@ impl TokenStream {
     }
 }
 
-/// By-reference iterator over a [`TokenStream`].
+/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`
+/// items.
 #[derive(Clone)]
-pub struct CursorRef<'t> {
+pub struct RefTokenTreeCursor<'t> {
     stream: &'t TokenStream,
     index: usize,
 }
 
-impl<'t> CursorRef<'t> {
+impl<'t> RefTokenTreeCursor<'t> {
     fn new(stream: &'t TokenStream) -> Self {
-        CursorRef { stream, index: 0 }
+        RefTokenTreeCursor { stream, index: 0 }
     }
 
     pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
         self.stream.0.get(self.index + n)
     }
 }
 
-impl<'t> Iterator for CursorRef<'t> {
+impl<'t> Iterator for RefTokenTreeCursor<'t> {
     type Item = &'t TokenTree;
 
     fn next(&mut self) -> Option<&'t TokenTree> {
@@ -580,15 +581,16 @@ impl<'t> Iterator for CursorRef<'t> {
     }
 }
 
-/// Owning by-value iterator over a [`TokenStream`].
+/// Owning by-value iterator over a [`TokenStream`], that produces `TokenTree`
+/// items.
 // FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
 #[derive(Clone)]
-pub struct Cursor {
+pub struct TokenTreeCursor {
     pub stream: TokenStream,
     index: usize,
 }
 
-impl Iterator for Cursor {
+impl Iterator for TokenTreeCursor {
     type Item = TokenTree;
 
     fn next(&mut self) -> Option<TokenTree> {
@@ -599,9 +601,9 @@ impl Iterator for Cursor {
     }
 }
 
-impl Cursor {
+impl TokenTreeCursor {
     fn new(stream: TokenStream) -> Self {
-        Cursor { stream, index: 0 }
+        TokenTreeCursor { stream, index: 0 }
     }
 
     #[inline]

compiler/rustc_expand/src/mbe/metavar_expr.rs

@@ -1,5 +1,5 @@
 use rustc_ast::token::{self, Delimiter};
-use rustc_ast::tokenstream::{CursorRef, TokenStream, TokenTree};
+use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
 use rustc_ast::{LitIntType, LitKind};
 use rustc_ast_pretty::pprust;
 use rustc_errors::{Applicability, PResult};
@@ -72,7 +72,7 @@ impl MetaVarExpr {
 
 // Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
 fn check_trailing_token<'sess>(
-    iter: &mut CursorRef<'_>,
+    iter: &mut RefTokenTreeCursor<'_>,
     sess: &'sess ParseSess,
 ) -> PResult<'sess, ()> {
     if let Some(tt) = iter.next() {
@@ -88,7 +88,7 @@ fn check_trailing_token<'sess>(
 
 /// Parse a meta-variable `count` expression: `count(ident[, depth])`
 fn parse_count<'sess>(
-    iter: &mut CursorRef<'_>,
+    iter: &mut RefTokenTreeCursor<'_>,
     sess: &'sess ParseSess,
     span: Span,
 ) -> PResult<'sess, MetaVarExpr> {
@@ -99,7 +99,7 @@ fn parse_count<'sess>(
 
 /// Parses the depth used by index(depth) and length(depth).
 fn parse_depth<'sess>(
-    iter: &mut CursorRef<'_>,
+    iter: &mut RefTokenTreeCursor<'_>,
     sess: &'sess ParseSess,
     span: Span,
 ) -> PResult<'sess, usize> {
@@ -126,7 +126,7 @@ fn parse_depth<'sess>(
 
 /// Parses an generic ident
 fn parse_ident<'sess>(
-    iter: &mut CursorRef<'_>,
+    iter: &mut RefTokenTreeCursor<'_>,
     sess: &'sess ParseSess,
     span: Span,
 ) -> PResult<'sess, Ident> {
@@ -152,7 +152,7 @@ fn parse_ident<'sess>(
 
 /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
 /// iterator is not modified and the result is `false`.
-fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool {
+fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
     if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
         let _ = iter.next();
         return true;

compiler/rustc_parse/src/parser/mod.rs

@@ -19,9 +19,8 @@ pub use path::PathStyle;
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Nonterminal, Token, TokenKind};
-use rustc_ast::tokenstream::AttributesData;
-use rustc_ast::tokenstream::{self, DelimSpan, Spacing};
-use rustc_ast::tokenstream::{TokenStream, TokenTree};
+use rustc_ast::tokenstream::{AttributesData, DelimSpan, Spacing};
+use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor};
 use rustc_ast::util::case::Case;
 use rustc_ast::AttrId;
 use rustc_ast::DUMMY_NODE_ID;
@@ -221,17 +220,21 @@ impl<'a> Drop for Parser<'a> {
     }
 }
 
+/// Iterator over a `TokenStream` that produces `Token`s. It's a bit odd that
+/// we (a) lex tokens into a nice tree structure (`TokenStream`), and then (b)
+/// use this type to emit them as a linear sequence. But a linear sequence is
+/// what the parser expects, for the most part.
 #[derive(Clone)]
 struct TokenCursor {
     // Cursor for the current (innermost) token stream. The delimiters for this
     // token stream are found in `self.stack.last()`; when that is `None` then
     // we are in the outermost token stream which never has delimiters.
-    tree_cursor: tokenstream::Cursor,
+    tree_cursor: TokenTreeCursor,
 
     // Token streams surrounding the current one. The delimiters for stack[n]'s
     // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters
     // because it's the outermost token stream which never has delimiters.
-    stack: Vec<(tokenstream::Cursor, Delimiter, DelimSpan)>,
+    stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>,
 
     desugar_doc_comments: bool,
 

src/tools/rustfmt/src/macros.rs

@@ -13,7 +13,7 @@ use std::collections::HashMap;
 use std::panic::{catch_unwind, AssertUnwindSafe};
 
 use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind};
-use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree};
+use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor};
 use rustc_ast::{ast, ptr};
 use rustc_ast_pretty::pprust;
 use rustc_span::{
@@ -736,7 +736,7 @@ impl MacroArgParser {
         self.buf.clear();
     }
 
-    fn add_meta_variable(&mut self, iter: &mut Cursor) -> Option<()> {
+    fn add_meta_variable(&mut self, iter: &mut TokenTreeCursor) -> Option<()> {
         match iter.next() {
             Some(TokenTree::Token(
                 Token {
@@ -768,7 +768,7 @@ impl MacroArgParser {
         &mut self,
         inner: Vec<ParsedMacroArg>,
         delim: Delimiter,
-        iter: &mut Cursor,
+        iter: &mut TokenTreeCursor,
     ) -> Option<()> {
         let mut buffer = String::new();
         let mut first = true;
@@ -1121,7 +1121,7 @@ pub(crate) fn macro_style(mac: &ast::MacCall, context: &RewriteContext<'_>) -> D
 // Currently we do not attempt to parse any further than that.
 #[derive(new)]
 struct MacroParser {
-    toks: Cursor,
+    toks: TokenTreeCursor,
 }
 
 impl MacroParser {