We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent f000aa7 commit 11d0d0cCopy full SHA for 11d0d0c
Sources/Tokenizers/Tokenizer.swift
@@ -321,7 +321,6 @@ public class PreTrainedTokenizer: Tokenizer {
321
}.joined(separator: "|")
322
addedTokensRegex = try? NSRegularExpression(pattern: addedTokensRegexString, options: [])
323
324
- // TODO: specialTokens are stored but never used
325
self.specialTokens = specialTokens
326
self.addedTokens = Set(addedTokens.keys)
327
0 commit comments