Rollup merge of rust-lang#65849 - popzxc:document-librustc_lexer, r=petrochenkov

librustc_lexer: Enhance documentation

This PR enhances documentation state of the `librustc_lexer` (as initiative caused by [rustc-guide#474](rust-lang/rustc-dev-guide#474)), by adding:

- Module documentation.
- Doc-comments (and a bit of usual comments) in non-obvious (as for me) places.

r? @petrochenkov

cc @Centril

Author: Centril

src/librustc_lexer/src/cursor.rs

@@ -1,5 +1,9 @@
 use std::str::Chars;
 
+/// Peekable iterator over a char sequence.
+///
+/// Next characters can be peeked via `nth_char` method,
+/// and position can be shifted forward via `bump` method.
 pub(crate) struct Cursor<'a> {
     initial_len: usize,
     chars: Chars<'a>,
@@ -18,7 +22,9 @@ impl<'a> Cursor<'a> {
             prev: EOF_CHAR,
         }
     }
+
     /// For debug assertions only
+    /// Returns the last eaten symbol (or '\0' in release builds).
     pub(crate) fn prev(&self) -> char {
         #[cfg(debug_assertions)]
         {
@@ -30,19 +36,30 @@ impl<'a> Cursor<'a> {
             '\0'
         }
     }
+
+    /// Returns nth character relative to the current cursor position.
+    /// If requested position doesn't exist, `EOF_CHAR` is returned.
+    /// However, getting `EOF_CHAR` doesn't always mean actual end of file,
+    /// it should be checked with `is_eof` method.
     pub(crate) fn nth_char(&self, n: usize) -> char {
         self.chars().nth(n).unwrap_or(EOF_CHAR)
     }
+
+    /// Checks if there is nothing more to consume.
     pub(crate) fn is_eof(&self) -> bool {
         self.chars.as_str().is_empty()
     }
+
+    /// Returns amount of already consumed symbols.
     pub(crate) fn len_consumed(&self) -> usize {
         self.initial_len - self.chars.as_str().len()
     }
-    /// Returns an iterator over the remaining characters.
+
+    /// Returns a `Chars` iterator over the remaining characters.
     fn chars(&self) -> Chars<'a> {
         self.chars.clone()
     }
+
     /// Moves to the next character.
     pub(crate) fn bump(&mut self) -> Option<char> {
         let c = self.chars.next()?;