Factor out token ranking

Author: Veykril

crates/ide-db/src/lib.rs

@@ -293,3 +293,33 @@ impl SnippetCap {
         }
     }
 }
+
+pub struct Ranker<'a> {
+    pub kind: parser::SyntaxKind,
+    pub text: &'a str,
+    pub ident_kind: bool,
+}
+
+impl<'a> Ranker<'a> {
+    pub fn from_token(token: &'a syntax::SyntaxToken) -> Self {
+        let kind = token.kind();
+        Ranker { kind, text: token.text(), ident_kind: kind.is_any_identifier() }
+    }
+
+    /// A utility function that ranks a token again a given kind and text, returning a number that
+    /// represents how close the token is to the given kind and text.
+    pub fn rank_token(&self, tok: &syntax::SyntaxToken) -> usize {
+        let tok_kind = tok.kind();
+
+        let exact_same_kind = tok_kind == self.kind;
+        let both_idents = exact_same_kind || (tok_kind.is_any_identifier() && self.ident_kind);
+        let same_text = tok.text() == self.text;
+        // anything that mapped into a token tree has likely no semantic information
+        let no_tt_parent =
+            tok.parent().map_or(false, |it| it.kind() != parser::SyntaxKind::TOKEN_TREE);
+        !((both_idents as usize)
+            | ((exact_same_kind as usize) << 1)
+            | ((same_text as usize) << 2)
+            | ((no_tt_parent as usize) << 3))
+    }
+}

crates/ide/src/hover.rs

@@ -11,7 +11,7 @@ use ide_db::{
     defs::{Definition, IdentClass, NameRefClass, OperatorClass},
     famous_defs::FamousDefs,
     helpers::pick_best_token,
-    FileRange, FxIndexSet, RootDatabase,
+    FileRange, FxIndexSet, Ranker, RootDatabase,
 };
 use itertools::{multizip, Itertools};
 use span::Edition;
@@ -182,27 +182,13 @@ fn hover_offset(
     // equivalency is more important
     let mut descended = sema.descend_into_macros(original_token.clone());
 
-    let kind = original_token.kind();
-    let text = original_token.text();
-    let ident_kind = kind.is_any_identifier();
-
-    descended.sort_by_cached_key(|tok| {
-        let tok_kind = tok.kind();
-
-        let exact_same_kind = tok_kind == kind;
-        let both_idents = exact_same_kind || (tok_kind.is_any_identifier() && ident_kind);
-        let same_text = tok.text() == text;
-        // anything that mapped into a token tree has likely no semantic information
-        let no_tt_parent = tok.parent().map_or(false, |it| it.kind() != TOKEN_TREE);
-        !((both_idents as usize)
-            | ((exact_same_kind as usize) << 1)
-            | ((same_text as usize) << 2)
-            | ((no_tt_parent as usize) << 3))
-    });
+    let ranker = Ranker::from_token(&original_token);
+
+    descended.sort_by_cached_key(|tok| ranker.rank_token(tok));
 
     let mut res = vec![];
     for token in descended {
-        let is_same_kind = token.kind() == kind;
+        let is_same_kind = token.kind() == ranker.kind;
         let lint_hover = (|| {
             // FIXME: Definition should include known lints and the like instead of having this special case here
             let attr = token.parent_ancestors().find_map(ast::Attr::cast)?;

crates/ide/src/syntax_highlighting.rs

@@ -16,7 +16,7 @@ mod tests;
 use std::ops::ControlFlow;
 
 use hir::{InRealFile, Name, Semantics};
-use ide_db::{FxHashMap, RootDatabase, SymbolKind};
+use ide_db::{FxHashMap, Ranker, RootDatabase, SymbolKind};
 use span::EditionedFileId;
 use syntax::{
     ast::{self, IsString},
@@ -401,9 +401,7 @@ fn traverse(
             // Attempt to descend tokens into macro-calls.
             let res = match element {
                 NodeOrToken::Token(token) if token.kind() != COMMENT => {
-                    let kind = token.kind();
-                    let text = token.text();
-                    let ident_kind = kind.is_any_identifier();
+                    let ranker = Ranker::from_token(&token);
 
                     let mut t = None;
                     let mut r = 0;
@@ -412,19 +410,7 @@ fn traverse(
                         |tok, _ctx| {
                             // FIXME: Consider checking ctx transparency for being opaque?
                             let tok = tok.value;
-                            let tok_kind = tok.kind();
-
-                            let exact_same_kind = tok_kind == kind;
-                            let both_idents =
-                                exact_same_kind || (tok_kind.is_any_identifier() && ident_kind);
-                            let same_text = tok.text() == text;
-                            // anything that mapped into a token tree has likely no semantic information
-                            let no_tt_parent =
-                                tok.parent().map_or(false, |it| it.kind() != TOKEN_TREE);
-                            let my_rank = (both_idents as usize)
-                                | ((exact_same_kind as usize) << 1)
-                                | ((same_text as usize) << 2)
-                                | ((no_tt_parent as usize) << 3);
+                            let my_rank = ranker.rank_token(&tok);
 
                             if my_rank > 0b1110 {
                                 // a rank of 0b1110 means that we have found a maximally interesting