clippy_dev: Capture token patterns by position and length.

Author: Jarcho

clippy_dev/src/new_lint.rs

@@ -1,4 +1,4 @@
-use crate::parse::cursor::{self, Cursor};
+use crate::parse::cursor::{self, Capture, Cursor};
 use crate::utils::Version;
 use clap::ValueEnum;
 use indoc::{formatdoc, writedoc};
@@ -522,6 +522,7 @@ fn parse_mod_file(path: &Path, contents: &str) -> (&'static str, usize) {
     let mut context = None;
     let mut decl_end = None;
     let mut cursor = Cursor::new(contents);
+    let mut captures = [Capture::EMPTY];
     while let Some(name) = cursor.find_capture_pat(CaptureIdent) {
         match name {
             "declare_clippy_lint" => {
@@ -530,9 +531,8 @@ fn parse_mod_file(path: &Path, contents: &str) -> (&'static str, usize) {
                 }
             },
             "impl" => {
-                let mut capture = "";
-                if cursor.match_all(&[Lt, Lifetime, Gt, CaptureIdent], &mut [&mut capture]) {
-                    match capture {
+                if cursor.match_all(&[Lt, Lifetime, Gt, CaptureIdent], &mut captures) {
+                    match cursor.get_text(captures[0]) {
                         "LateLintPass" => context = Some("LateContext"),
                         "EarlyLintPass" => context = Some("EarlyContext"),
                         _ => {},

clippy_dev/src/parse.rs

@@ -1,6 +1,6 @@
 pub mod cursor;
 
-use self::cursor::Cursor;
+use self::cursor::{Capture, Cursor};
 use crate::utils::{ErrAction, File, expect_action};
 use core::range::Range;
 use std::fs;
@@ -102,13 +102,13 @@ fn parse_clippy_lint_decls(path: &Path, contents: &str, module: &str, lints: &mu
     ];
 
     let mut cursor = Cursor::new(contents);
+    let mut captures = [Capture::EMPTY; 2];
     while cursor.find_pat(Ident("declare_clippy_lint")) {
         let start = cursor.pos() as usize - "declare_clippy_lint".len();
-        let (mut name, mut group) = ("", "");
-        if cursor.match_all(DECL_TOKENS, &mut [&mut name, &mut group]) && cursor.find_pat(CloseBrace) {
+        if cursor.match_all(DECL_TOKENS, &mut captures) && cursor.find_pat(CloseBrace) {
             lints.push(Lint {
-                name: name.to_lowercase(),
-                group: group.into(),
+                name: cursor.get_text(captures[0]).to_lowercase(),
+                group: cursor.get_text(captures[1]).into(),
                 module: module.into(),
                 path: path.into(),
                 declaration_range: start..cursor.pos() as usize,
@@ -146,6 +146,7 @@ pub fn read_deprecated_lints() -> (Vec<DeprecatedLint>, Vec<RenamedLint>) {
     File::open_read_to_cleared_string(path, &mut contents);
 
     let mut cursor = Cursor::new(&contents);
+    let mut captures = [Capture::EMPTY; 3];
 
     // First instance is the macro definition.
     assert!(
@@ -154,29 +155,23 @@ pub fn read_deprecated_lints() -> (Vec<DeprecatedLint>, Vec<RenamedLint>) {
     );
 
     if cursor.find_pat(Ident("declare_with_version")) && cursor.match_all(DEPRECATED_TOKENS, &mut []) {
-        let mut version = "";
-        let mut name = "";
-        let mut reason = "";
-        while cursor.match_all(DECL_TOKENS, &mut [&mut version, &mut name, &mut reason]) {
+        while cursor.match_all(DECL_TOKENS, &mut captures) {
             deprecated.push(DeprecatedLint {
-                name: parse_str_single_line(path.as_ref(), name),
-                reason: parse_str_single_line(path.as_ref(), reason),
-                version: parse_str_single_line(path.as_ref(), version),
+                name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])),
+                reason: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])),
+                version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])),
             });
         }
     } else {
         panic!("error reading deprecated lints");
     }
 
     if cursor.find_pat(Ident("declare_with_version")) && cursor.match_all(RENAMED_TOKENS, &mut []) {
-        let mut version = "";
-        let mut old_name = "";
-        let mut new_name = "";
-        while cursor.match_all(DECL_TOKENS, &mut [&mut version, &mut old_name, &mut new_name]) {
+        while cursor.match_all(DECL_TOKENS, &mut captures) {
             renamed.push(RenamedLint {
-                old_name: parse_str_single_line(path.as_ref(), old_name),
-                new_name: parse_str_single_line(path.as_ref(), new_name),
-                version: parse_str_single_line(path.as_ref(), version),
+                old_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])),
+                new_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])),
+                version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])),
             });
         }
     } else {

clippy_dev/src/parse/cursor.rs

@@ -32,6 +32,15 @@ pub enum Pat<'a> {
     Semi,
 }
 
+#[derive(Clone, Copy)]
+pub struct Capture {
+    pub pos: u32,
+    pub len: u32,
+}
+impl Capture {
+    pub const EMPTY: Self = Self { pos: 0, len: 0 };
+}
+
 /// A unidirectional cursor over a token stream that is lexed on demand.
 pub struct Cursor<'txt> {
     next_token: Token,
@@ -51,6 +60,12 @@ impl<'txt> Cursor<'txt> {
         }
     }
 
+    /// Gets the text of the captured token assuming it came from this cursor.
+    #[must_use]
+    pub fn get_text(&self, capture: Capture) -> &'txt str {
+        &self.text[capture.pos as usize..(capture.pos + capture.len) as usize]
+    }
+
     /// Gets the text that makes up the next token in the stream, or the empty string if
     /// stream is exhausted.
     #[must_use]
@@ -97,7 +112,7 @@ impl<'txt> Cursor<'txt> {
     ///
     /// For each capture made by the pattern one item will be taken from the capture
     /// sequence with the result placed inside.
-    fn match_pat(&mut self, pat: Pat<'_>, captures: &mut slice::IterMut<'_, &mut &'txt str>) -> bool {
+    fn match_impl(&mut self, pat: Pat<'_>, captures: &mut slice::IterMut<'_, Capture>) -> bool {
         loop {
             match (pat, self.next_token.kind) {
                 #[rustfmt::skip] // rustfmt bug: https://github.com/rust-lang/rustfmt/issues/6697
@@ -154,7 +169,7 @@ impl<'txt> Cursor<'txt> {
                     },
                 )
                 | (Pat::CaptureIdent, TokenKind::Ident) => {
-                    **captures.next().unwrap() = self.peek_text();
+                    *captures.next().unwrap() = Capture { pos: self.pos, len: self.next_token.len };
                     self.step();
                     return true;
                 },
@@ -169,9 +184,9 @@ impl<'txt> Cursor<'txt> {
     /// Not generally suitable for multi-token patterns or patterns that can match
     /// nothing.
     #[must_use]
-    pub fn find_pat(&mut self, token: Pat<'_>) -> bool {
+    pub fn find_pat(&mut self, pat: Pat<'_>) -> bool {
         let mut capture = [].iter_mut();
-        while !self.match_pat(token, &mut capture) {
+        while !self.match_impl(pat, &mut capture) {
             self.step();
             if self.at_end() {
                 return false;
@@ -182,21 +197,20 @@ impl<'txt> Cursor<'txt> {
 
     /// The same as [`Self::find_pat`], but returns a capture as well.
     #[must_use]
-    pub fn find_capture_pat(&mut self, token: Pat<'_>) -> Option<&'txt str> {
-        let mut res = "";
-        let mut capture = &mut res;
-        let mut capture = slice::from_mut(&mut capture).iter_mut();
-        while !self.match_pat(token, &mut capture) {
+    pub fn find_capture_pat(&mut self, pat: Pat<'_>) -> Option<&'txt str> {
+        let mut capture = Capture::EMPTY;
+        let mut captures = slice::from_mut(&mut capture).iter_mut();
+        while !self.match_impl(pat, &mut captures) {
             self.step();
             if self.at_end() {
                 return None;
             }
         }
-        Some(res)
+        Some(self.get_text(capture))
     }
 
     /// Attempts to match a sequence of patterns at the current position. Returns whether
-    /// the match is successful.
+    /// all patterns were successfully matched.
     ///
     /// Captures will be written to the given slice in the order they're matched. If a
     /// capture is matched, but there are no more capture slots this will panic. If the
@@ -205,8 +219,8 @@ impl<'txt> Cursor<'txt> {
     ///
     /// If the match fails the cursor will be positioned at the first failing token.
     #[must_use]
-    pub fn match_all(&mut self, tokens: &[Pat<'_>], captures: &mut [&mut &'txt str]) -> bool {
+    pub fn match_all(&mut self, pats: &[Pat<'_>], captures: &mut [Capture]) -> bool {
         let mut captures = captures.iter_mut();
-        tokens.iter().all(|&t| self.match_pat(t, &mut captures))
+        pats.iter().all(|&t| self.match_impl(t, &mut captures))
     }
 }

clippy_dev/src/rename_lint.rs

@@ -1,4 +1,4 @@
-use crate::parse::cursor::{self, Cursor};
+use crate::parse::cursor::{self, Capture, Cursor};
 use crate::parse::{RenamedLint, find_lint_decls, read_deprecated_lints};
 use crate::update_lints::generate_lint_files;
 use crate::utils::{
@@ -281,7 +281,7 @@ fn file_update_fn<'a, 'b>(
         let mut copy_pos = 0u32;
         let mut changed = false;
         let mut cursor = Cursor::new(src);
-        let mut capture = "";
+        let mut captures = [Capture::EMPTY];
         loop {
             match cursor.peek() {
                 TokenKind::Eof => break,
@@ -292,12 +292,10 @@ fn file_update_fn<'a, 'b>(
                     match text {
                         // clippy::line_name or clippy::lint-name
                         "clippy" => {
-                            if cursor.match_all(
-                                &[cursor::Pat::DoubleColon, cursor::Pat::CaptureIdent],
-                                &mut [&mut capture],
-                            ) && capture == old_name
+                            if cursor.match_all(&[cursor::Pat::DoubleColon, cursor::Pat::CaptureIdent], &mut captures)
+                                && cursor.get_text(captures[0]) == old_name
                             {
-                                dst.push_str(&src[copy_pos as usize..cursor.pos() as usize - capture.len()]);
+                                dst.push_str(&src[copy_pos as usize..captures[0].pos as usize]);
                                 dst.push_str(new_name);
                                 copy_pos = cursor.pos();
                                 changed = true;
@@ -306,12 +304,12 @@ fn file_update_fn<'a, 'b>(
                         // mod lint_name
                         "mod" => {
                             if !matches!(mod_edit, ModEdit::None)
-                                && cursor.match_all(&[cursor::Pat::CaptureIdent], &mut [&mut capture])
-                                && capture == old_name
+                                && cursor.match_all(&[cursor::Pat::CaptureIdent], &mut captures)
+                                && cursor.get_text(captures[0]) == old_name
                             {
                                 match mod_edit {
                                     ModEdit::Rename => {
-                                        dst.push_str(&src[copy_pos as usize..cursor.pos() as usize - capture.len()]);
+                                        dst.push_str(&src[copy_pos as usize..captures[0].pos as usize]);
                                         dst.push_str(new_name);
                                         copy_pos = cursor.pos();
                                         changed = true;
@@ -374,12 +372,10 @@ fn file_update_fn<'a, 'b>(
                 },
                 // ::lint_name
                 TokenKind::Colon
-                    if cursor.match_all(
-                        &[cursor::Pat::DoubleColon, cursor::Pat::CaptureIdent],
-                        &mut [&mut capture],
-                    ) && capture == old_name =>
+                    if cursor.match_all(&[cursor::Pat::DoubleColon, cursor::Pat::CaptureIdent], &mut captures)
+                        && cursor.get_text(captures[0]) == old_name =>
                 {
-                    dst.push_str(&src[copy_pos as usize..cursor.pos() as usize - capture.len()]);
+                    dst.push_str(&src[copy_pos as usize..captures[0].pos as usize]);
                     dst.push_str(new_name);
                     copy_pos = cursor.pos();
                     changed = true;