Skip to content

Commit 47484a1

Browse files
committed
this split call wasn't doing anything
1 parent ea9fed4 commit 47484a1

File tree

1 file changed

+4
-1
lines changed

1 file changed

+4
-1
lines changed

+bert/+tokenizer/+internal/WordPieceTokenizer.m

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@
3535
this.Unk = nvp.UnknownToken;
3636
this.MaxChar = nvp.MaxTokenLength;
3737
this.Vocab = this.parseVocab(vocab);
38+
39+
40+
3841
end
3942

4043
function tokens = tokenize(this,text)
@@ -98,7 +101,7 @@
98101
c = fread(fid,Inf);
99102
fclose(fid);
100103
c = native2unicode(c,'utf-8');%#ok
101-
words = split(splitlines(c')).';
104+
words = splitlines(c').';
102105
empties = cellfun(@isempty,words);
103106
words(empties) = [];
104107
vocab = wordEncoding(words);

0 commit comments

Comments
 (0)