optimizations

Author: bwdGitHub

+bert/+tokenizer/+internal/BasicTokenizer.m

@@ -34,12 +34,11 @@
             u = this.cleanText(u);
             u = this.tokenizeCJK(u);
             text = u.string();
-            origTokens = this.whiteSpaceTokenize(text);
             if this.IgnoreCase
-                origTokens = lower(origTokens);
-                origTokens = textanalytics.unicode.nfd(origTokens);
+                text = lower(text);
+                text = textanalytics.unicode.nfd(text);
             end
-            u = textanalytics.unicode.UTF32(origTokens);
+            u = textanalytics.unicode.UTF32(text);
             cats = u.characterCategories('Granularity','detailed');
             if this.IgnoreCase
                 [u,cats] = this.stripAccents(u,cats);

+bert/+tokenizer/+internal/WordPieceTokenizer.m

@@ -63,8 +63,9 @@
                         if start>1
                             sub.Data = [uint32('##'),sub.Data];
                         end
-                        if this.Vocab.isVocabularyWord(sub.string())
-                            currentSub = sub.string();
+                        strForm = sub.string();
+                        if this.Vocab.isVocabularyWord(strForm)
+                            currentSub = strForm;
                             break
                         end
                         finish = finish-1;