Move the eat_*_digits() methods to literals.rs.

Author: Julian Wollersberger

compiler/rustc_lexer/src/cursor.rs

@@ -81,4 +81,11 @@ impl<'a> Cursor<'a> {
 
         Some(c)
     }
+
+    /// Eats symbols while predicate returns true or until the end of file is reached.
+    pub(crate) fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
+        while predicate(self.first()) && !self.is_eof() {
+            self.bump();
+        }
+    }
 }

compiler/rustc_lexer/src/lib.rs

@@ -31,8 +31,8 @@ mod tests;
 use self::TokenKind::*;
 use crate::cursor::Cursor;
 use crate::literals::{
-    double_quoted_string, lifetime_or_char, number, raw_double_quoted_string, single_quoted_string,
-    LiteralKind,
+    double_quoted_string, eat_literal_suffix, lifetime_or_char, number, raw_double_quoted_string,
+    single_quoted_string, LiteralKind,
 };
 
 /// Parsed token.
@@ -272,7 +272,7 @@ impl Cursor<'_> {
                     let (n_hashes, err) = raw_double_quoted_string(self, 1);
                     let suffix_start = self.len_consumed();
                     if err.is_none() {
-                        self.eat_literal_suffix();
+                        eat_literal_suffix(self);
                     }
                     let kind = LiteralKind::RawStr { n_hashes, err };
                     Literal { kind, suffix_start }
@@ -287,7 +287,7 @@ impl Cursor<'_> {
                     let terminated = single_quoted_string(self);
                     let suffix_start = self.len_consumed();
                     if terminated {
-                        self.eat_literal_suffix();
+                        eat_literal_suffix(self);
                     }
                     let kind = LiteralKind::Byte { terminated };
                     Literal { kind, suffix_start }
@@ -297,7 +297,7 @@ impl Cursor<'_> {
                     let terminated = double_quoted_string(self);
                     let suffix_start = self.len_consumed();
                     if terminated {
-                        self.eat_literal_suffix();
+                        eat_literal_suffix(self);
                     }
                     let kind = LiteralKind::ByteStr { terminated };
                     Literal { kind, suffix_start }
@@ -307,7 +307,7 @@ impl Cursor<'_> {
                     let (n_hashes, err) = raw_double_quoted_string(self, 2);
                     let suffix_start = self.len_consumed();
                     if err.is_none() {
-                        self.eat_literal_suffix();
+                        eat_literal_suffix(self);
                     }
                     let kind = LiteralKind::RawByteStr { n_hashes, err };
                     Literal { kind, suffix_start }
@@ -323,7 +323,7 @@ impl Cursor<'_> {
             c @ '0'..='9' => {
                 let literal_kind = number(self, c);
                 let suffix_start = self.len_consumed();
-                self.eat_literal_suffix();
+                eat_literal_suffix(self);
                 TokenKind::Literal { kind: literal_kind, suffix_start }
             }
 
@@ -363,7 +363,7 @@ impl Cursor<'_> {
                 let terminated = double_quoted_string(self);
                 let suffix_start = self.len_consumed();
                 if terminated {
-                    self.eat_literal_suffix();
+                    eat_literal_suffix(self);
                 }
                 let kind = LiteralKind::Str { terminated };
                 Literal { kind, suffix_start }
@@ -448,56 +448,7 @@ impl Cursor<'_> {
         Ident
     }
 
-    fn eat_decimal_digits(&mut self) -> bool {
-        let mut has_digits = false;
-        loop {
-            match self.first() {
-                '_' => {
-                    self.bump();
-                }
-                '0'..='9' => {
-                    has_digits = true;
-                    self.bump();
-                }
-                _ => break,
-            }
-        }
-        has_digits
-    }
-
-    fn eat_hexadecimal_digits(&mut self) -> bool {
-        let mut has_digits = false;
-        loop {
-            match self.first() {
-                '_' => {
-                    self.bump();
-                }
-                '0'..='9' | 'a'..='f' | 'A'..='F' => {
-                    has_digits = true;
-                    self.bump();
-                }
-                _ => break,
-            }
-        }
-        has_digits
-    }
-
-    /// Eats the float exponent. Returns true if at least one digit was met,
-    /// and returns false otherwise.
-    fn eat_float_exponent(&mut self) -> bool {
-        debug_assert!(self.prev() == 'e' || self.prev() == 'E');
-        if self.first() == '-' || self.first() == '+' {
-            self.bump();
-        }
-        self.eat_decimal_digits()
-    }
-
-    // Eats the suffix of the literal, e.g. "_u8".
-    fn eat_literal_suffix(&mut self) {
-        self.eat_identifier();
-    }
-
-    // Eats the identifier.
+    /// Eats one identifier.
     fn eat_identifier(&mut self) {
         if !is_id_start(self.first()) {
             return;
@@ -506,11 +457,4 @@ impl Cursor<'_> {
 
         self.eat_while(is_id_continue);
     }
-
-    /// Eats symbols while predicate returns true or until the end of file is reached.
-    fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
-        while predicate(self.first()) && !self.is_eof() {
-            self.bump();
-        }
-    }
 }

compiler/rustc_lexer/src/literals.rs

@@ -60,21 +60,21 @@ pub(crate) fn number(cursor: &mut Cursor, first_digit: char) -> LiteralKind {
             'b' => {
                 base = Base::Binary;
                 cursor.bump();
-                cursor.eat_decimal_digits()
+                eat_decimal_digits(cursor)
             }
             'o' => {
                 base = Base::Octal;
                 cursor.bump();
-                cursor.eat_decimal_digits()
+                eat_decimal_digits(cursor)
             }
             'x' => {
                 base = Base::Hexadecimal;
                 cursor.bump();
-                cursor.eat_hexadecimal_digits()
+                eat_hexadecimal_digits(cursor)
             }
             // Not a base prefix.
             '0'..='9' | '_' | '.' | 'e' | 'E' => {
-                cursor.eat_decimal_digits();
+                eat_decimal_digits(cursor);
                 true
             }
             // Just a 0.
@@ -87,7 +87,7 @@ pub(crate) fn number(cursor: &mut Cursor, first_digit: char) -> LiteralKind {
         }
     } else {
         // No base prefix, parse number in the usual way.
-        cursor.eat_decimal_digits();
+        eat_decimal_digits(cursor);
     };
 
     match cursor.first() {
@@ -100,11 +100,11 @@ pub(crate) fn number(cursor: &mut Cursor, first_digit: char) -> LiteralKind {
             cursor.bump();
             let mut empty_exponent = false;
             if cursor.first().is_digit(10) {
-                cursor.eat_decimal_digits();
+                eat_decimal_digits(cursor);
                 match cursor.first() {
                     'e' | 'E' => {
                         cursor.bump();
-                        empty_exponent = !cursor.eat_float_exponent();
+                        empty_exponent = !eat_float_exponent(cursor);
                     }
                     _ => (),
                 }
@@ -113,13 +113,57 @@ pub(crate) fn number(cursor: &mut Cursor, first_digit: char) -> LiteralKind {
         }
         'e' | 'E' => {
             cursor.bump();
-            let empty_exponent = !cursor.eat_float_exponent();
+            let empty_exponent = !eat_float_exponent(cursor);
             LiteralKind::Float { base, empty_exponent }
         }
         _ => LiteralKind::Int { base, empty_int: false },
     }
 }
 
+pub(crate) fn eat_decimal_digits(cursor: &mut Cursor) -> bool {
+    let mut has_digits = false;
+    loop {
+        match cursor.first() {
+            '_' => {
+                cursor.bump();
+            }
+            '0'..='9' => {
+                has_digits = true;
+                cursor.bump();
+            }
+            _ => break,
+        }
+    }
+    has_digits
+}
+
+pub(crate) fn eat_hexadecimal_digits(cursor: &mut Cursor) -> bool {
+    let mut has_digits = false;
+    loop {
+        match cursor.first() {
+            '_' => {
+                cursor.bump();
+            }
+            '0'..='9' | 'a'..='f' | 'A'..='F' => {
+                has_digits = true;
+                cursor.bump();
+            }
+            _ => break,
+        }
+    }
+    has_digits
+}
+
+/// Eats the float exponent. Returns true if at least one digit was met,
+/// and returns false otherwise.
+fn eat_float_exponent(cursor: &mut Cursor) -> bool {
+    debug_assert!(cursor.prev() == 'e' || cursor.prev() == 'E');
+    if cursor.first() == '-' || cursor.first() == '+' {
+        cursor.bump();
+    }
+    eat_decimal_digits(cursor)
+}
+
 pub(crate) fn lifetime_or_char(cursor: &mut Cursor) -> TokenKind {
     debug_assert!(cursor.prev() == '\'');
 
@@ -137,7 +181,7 @@ pub(crate) fn lifetime_or_char(cursor: &mut Cursor) -> TokenKind {
         let terminated = single_quoted_string(cursor);
         let suffix_start = cursor.len_consumed();
         if terminated {
-            cursor.eat_literal_suffix();
+            eat_literal_suffix(cursor);
         }
         let kind = LiteralKind::Char { terminated };
         return TokenKind::Literal { kind, suffix_start };
@@ -307,3 +351,8 @@ fn raw_string_unvalidated(cursor: &mut Cursor, prefix_len: usize) -> (usize, Opt
         }
     }
 }
+
+/// Eats the suffix of a literal, e.g. "_u8".
+pub(crate) fn eat_literal_suffix(cursor: &mut Cursor) {
+    cursor.eat_identifier();
+}