tokenize some regexp patterns (#554)

Author: atuchin-m

src/filters/network.rs

@@ -311,6 +311,7 @@ pub enum FilterPart {
     AnyOf(Vec<String>),
 }
 
+#[derive(Debug, PartialEq)]
 pub enum FilterTokens {
     Empty,
     OptDomains(Vec<Hash>),
@@ -929,6 +930,12 @@ impl NetworkFilter {
             if let Some(hostname) = self.hostname.as_ref() {
                 utils::tokenize_to(hostname, &mut tokens);
             }
+        } else if let Some(hostname) = self.hostname.as_ref() {
+            // Find last dot to tokenize the prefix
+            let last_dot_pos = hostname.rfind('.');
+            if let Some(last_dot_pos) = last_dot_pos {
+                utils::tokenize_to(&hostname[..last_dot_pos], &mut tokens);
+            }
         }
 
         if tokens.is_empty() && self.mask.contains(NetworkFilterMask::IS_REMOVEPARAM) {

tests/unit/engine.rs

@@ -237,9 +237,9 @@ mod tests {
             );
         }
         let expected_hash: u64 = if cfg!(feature = "css-validation") {
-            9439492009815519037
+            15545091389304905433
         } else {
-            14803842039735157685
+            543362704487480180
         };
 
         assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}");

tests/unit/filters/network.rs

@@ -1186,4 +1186,17 @@ mod parse_tests {
         defaults.opt_domains = Some(vec![utils::fast_hash("auth.wi-fi.ru")]);
         assert_eq!(defaults, NetworkFilterBreakdown::from(&filter));
     }
+
+    #[test]
+    fn test_simple_pattern_tokenization() {
+        let rule = "||some.primewire.c*/sw$script,1p";
+        let filter = NetworkFilter::parse(rule, true, ParseOptions::default()).unwrap();
+        assert_eq!(
+            filter.get_tokens_optimized(),
+            FilterTokens::Other(vec![
+                utils::fast_hash("some"),
+                utils::fast_hash("primewire")
+            ])
+        );
+    }
 }