Address review comments.

Author: nnethercote

compiler/rustc_lexer/src/lib.rs

@@ -23,15 +23,17 @@
 // We want to be able to build this crate with a stable compiler, so no
 // `#![feature]` attributes should be added.
 
-pub mod cursor;
+mod cursor;
 pub mod unescape;
 
 #[cfg(test)]
 mod tests;
 
+pub use crate::cursor::Cursor;
+
 use self::LiteralKind::*;
 use self::TokenKind::*;
-use crate::cursor::{Cursor, EOF_CHAR};
+use crate::cursor::EOF_CHAR;
 use std::convert::TryFrom;
 
 /// Parsed token.

compiler/rustc_parse/src/lexer/mod.rs

@@ -4,8 +4,8 @@ use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::util::unicode::contains_text_flow_control_chars;
 use rustc_errors::{error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult};
-use rustc_lexer::cursor::Cursor;
 use rustc_lexer::unescape::{self, Mode};
+use rustc_lexer::Cursor;
 use rustc_lexer::{Base, DocStyle, RawStrError};
 use rustc_session::lint::builtin::{
     RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, TEXT_DIRECTION_CODEPOINT_IN_COMMENT,

compiler/rustc_parse/src/lexer/tokentrees.rs

@@ -53,7 +53,7 @@ impl<'a> TokenTreesReader<'a> {
                 token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
                 token::CloseDelim(delim) => return Err(self.close_delim_err(delim)),
                 token::Eof => return Ok(buf.into_token_stream()),
-                _ => buf.push(self.parse_token_tree_other()),
+                _ => buf.push(self.parse_token_tree_non_delim_non_eof()),
             }
         }
     }
@@ -66,11 +66,10 @@ impl<'a> TokenTreesReader<'a> {
                 token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
                 token::CloseDelim(..) => return buf.into_token_stream(),
                 token::Eof => {
-                    let mut err = self.eof_err();
-                    err.emit();
+                    self.eof_err().emit();
                     return buf.into_token_stream();
                 }
-                _ => buf.push(self.parse_token_tree_other()),
+                _ => buf.push(self.parse_token_tree_non_delim_non_eof()),
             }
         }
     }
@@ -245,9 +244,10 @@ impl<'a> TokenTreesReader<'a> {
     }
 
     #[inline]
-    fn parse_token_tree_other(&mut self) -> TokenTree {
-        // `spacing` for the returned token is determined by the next token:
-        // its kind and its `preceded_by_whitespace` status.
+    fn parse_token_tree_non_delim_non_eof(&mut self) -> TokenTree {
+        // `this_spacing` for the returned token refers to whether the token is
+        // immediately followed by another op token. It is determined by the
+        // next token: its kind and its `preceded_by_whitespace` status.
         let (next_tok, is_next_tok_preceded_by_whitespace) = self.string_reader.next_token();
         let this_spacing = if is_next_tok_preceded_by_whitespace || !next_tok.is_op() {
             Spacing::Alone

src/librustdoc/html/highlight.rs

@@ -13,7 +13,7 @@ use std::collections::VecDeque;
 use std::fmt::{Display, Write};
 
 use rustc_data_structures::fx::FxHashMap;
-use rustc_lexer::cursor::Cursor;
+use rustc_lexer::Cursor;
 use rustc_lexer::{LiteralKind, TokenKind};
 use rustc_span::edition::Edition;
 use rustc_span::symbol::Symbol;