Restore logic to reserve a free slot for zero token

Author: atuchin-m

src/flatbuffers/unsafe_tools.rs

@@ -130,6 +130,10 @@ impl<T: Default, const MAX_SIZE: usize> StackVector<T, MAX_SIZE> {
         self.size == 0
     }
 
+    pub fn get_free_capacity(&self) -> usize {
+        MAX_SIZE - self.size
+    }
+
     pub fn clear(&mut self) {
         self.size = 0;
     }

src/utils.rs

@@ -44,6 +44,9 @@ fn fast_tokenizer_no_regex(
     let mut preceding_ch: Option<char> = None; // Used to check if a '*' is not just before a token
 
     for (i, c) in pattern.char_indices() {
+        if tokens_buffer.get_free_capacity() <= 1 {
+            return; // reserve one free slot for the zero token
+        }
         if is_allowed_code(c) {
             if !inside {
                 inside = true;