more possible optimizations

Author: bwdGitHub

+bert/+tokenizer/+internal/FullTokenizer.m

@@ -85,9 +85,10 @@
             %   tokens = tokenize(tokenizer,text) tokenizes the input
             %   string text using the FullTokenizer specified by tokenizer.
             basicToks = this.Basic.tokenize(txt);
+            basicToksUnicode = textanalytics.unicode.UTF32(basicToks);
             subToks = cell(numel(basicToks),1);
             for i = 1:numel(basicToks)
-                subToks{i} = this.WordPiece.tokenize(basicToks{i});
+                subToks{i} = this.WordPiece.tokenize(basicToksUnicode(i));
             end
             toks = cat(2,subToks{:});
         end

+bert/+tokenizer/+internal/WordPieceTokenizer.m

@@ -37,16 +37,15 @@
             this.Vocab = this.parseVocab(vocab);
         end
 
-        function tokens = tokenize(this,text)
+        function tokens = tokenize(this,utext)
             arguments
                 this
-                text (1,1) string
+                utext
             end
             tokens = string.empty();
-            wsTokens = this.WhitespaceTokenizer.tokenize(text);
-            wsTokensU = textanalytics.unicode.UTF32(wsTokens);
-            for i = 1:numel(wsTokensU)
-                token = wsTokensU(i);
+            sub = textanalytics.unicode.UTF32();
+            for i = 1:numel(utext)
+                token = utext(i);
                 if numel(token.Data)>this.MaxChar
                     tokens = [tokens,this.Unk]; %#ok
                     continue
@@ -57,8 +56,7 @@
                 while start<(numel(token.Data)+1)
                     finish = numel(token.Data);
                     currentSub = [];
-                    while start<finish+1
-                        sub = textanalytics.unicode.UTF32();
+                    while start<finish+1                        
                         sub.Data = token.Data(start:finish);
                         if start>1
                             sub.Data = [uint32('##'),sub.Data];