Optimize vec[] creation (#545)

Author: atuchin-m

src/filters/network.rs

@@ -917,10 +917,7 @@ impl NetworkFilter {
                         (self.is_plain() || self.is_regex()) && !self.is_right_anchor();
                     let skip_first_token = self.is_right_anchor();
 
-                    let filter_tokens =
-                        utils::tokenize_filter(f, skip_first_token, skip_last_token);
-
-                    tokens.extend(filter_tokens);
+                    utils::tokenize_filter_to(f, skip_first_token, skip_last_token, &mut tokens);
                 }
             }
             FilterPart::AnyOf(_) => (), // across AnyOf set of filters no single token is guaranteed to match to a request
@@ -930,16 +927,14 @@ impl NetworkFilter {
         // Append tokens from hostname, if any
         if !self.mask.contains(NetworkFilterMask::IS_HOSTNAME_REGEX) {
             if let Some(hostname) = self.hostname.as_ref() {
-                let hostname_tokens = utils::tokenize(hostname);
-                tokens.extend(hostname_tokens);
+                utils::tokenize_to(hostname, &mut tokens);
             }
         }
 
         if tokens.is_empty() && self.mask.contains(NetworkFilterMask::IS_REMOVEPARAM) {
             if let Some(removeparam) = &self.modifier_option {
                 if VALID_PARAM.is_match(removeparam) {
-                    let param_tokens = utils::tokenize(&removeparam.to_ascii_lowercase());
-                    tokens.extend(param_tokens);
+                    utils::tokenize_to(&removeparam.to_ascii_lowercase(), &mut tokens);
                 }
             }
         }

src/utils.rs

@@ -81,32 +81,45 @@ pub(crate) fn tokenize_pooled(pattern: &str, tokens_buffer: &mut Vec<Hash>) {
 
 pub fn tokenize(pattern: &str) -> Vec<Hash> {
     let mut tokens_buffer: Vec<Hash> = Vec::with_capacity(TOKENS_BUFFER_SIZE);
-    fast_tokenizer_no_regex(
-        pattern,
-        &is_allowed_filter,
-        false,
-        false,
-        &mut tokens_buffer,
-    );
+    tokenize_to(pattern, &mut tokens_buffer);
     tokens_buffer
 }
 
+pub(crate) fn tokenize_to(pattern: &str, tokens_buffer: &mut Vec<Hash>) {
+    fast_tokenizer_no_regex(pattern, &is_allowed_filter, false, false, tokens_buffer);
+}
+
+#[cfg(test)]
 pub(crate) fn tokenize_filter(
     pattern: &str,
     skip_first_token: bool,
     skip_last_token: bool,
 ) -> Vec<Hash> {
     let mut tokens_buffer: Vec<Hash> = Vec::with_capacity(TOKENS_BUFFER_SIZE);
-    fast_tokenizer_no_regex(
+    tokenize_filter_to(
         pattern,
-        &is_allowed_filter,
         skip_first_token,
         skip_last_token,
         &mut tokens_buffer,
     );
     tokens_buffer
 }
 
+pub(crate) fn tokenize_filter_to(
+    pattern: &str,
+    skip_first_token: bool,
+    skip_last_token: bool,
+    tokens_buffer: &mut Vec<Hash>,
+) {
+    fast_tokenizer_no_regex(
+        pattern,
+        &is_allowed_filter,
+        skip_first_token,
+        skip_last_token,
+        tokens_buffer,
+    );
+}
+
 pub(crate) fn bin_lookup<T: Ord>(arr: &[T], elt: T) -> bool {
     arr.binary_search(&elt).is_ok()
 }