File tree Expand file tree Collapse file tree 1 file changed +3
-3
lines changed Expand file tree Collapse file tree 1 file changed +3
-3
lines changed Original file line number Diff line number Diff line change 1010/// We could make it generic so TokenLatticeNode stores an opaque type, but it's overkill right now.
1111/// Based on https://github.com/huggingface/tokenizers/blob/b58227c7f1ccf8b73ee2268354336da56d91e492/tokenizers/src/models/unigram/lattice.rs#L137
1212/// and https://github.com/xenova/transformers.js/blob/b07336d8f7ff57453cc164cc68aead2a79cbd57e/src/utils/data-structures.js#L269C28-L269C28
13- public struct TokenLattice {
13+ struct TokenLattice {
1414 let sentence : String
1515 let bosTokenId : Int
1616 let eosTokenId : Int
@@ -40,7 +40,7 @@ public struct TokenLattice {
4040 }
4141}
4242
43- public extension TokenLattice {
43+ extension TokenLattice {
4444 /// Insert a new token into the node lattice.
4545 ///
4646 /// - Parameters:
@@ -108,7 +108,7 @@ extension TokenLattice {
108108 }
109109}
110110
111- public extension TokenLattice {
111+ extension TokenLattice {
112112 var tokens : [ String ] {
113113 viterbi ( ) . map { String ( piece ( $0) ) }
114114 }
You can’t perform that action at this time.
0 commit comments