File tree Expand file tree Collapse file tree 2 files changed +1
-16
lines changed
src/main/java/com/example/model/format Expand file tree Collapse file tree 2 files changed +1
-16
lines changed Original file line number Diff line number Diff line change @@ -57,7 +57,7 @@ public List<Integer> encodeMessage(Message message) {
5757 public List <Integer > encodeDialogPrompt (boolean appendAssistantTurn , List <Message > dialog ) {
5858 List <Integer > tokens = new ArrayList <>();
5959 tokens .add (beginOfText );
60- for (LlamaChatFormat . Message message : dialog ) {
60+ for (Message message : dialog ) {
6161 tokens .addAll (encodeMessage (message ));
6262 }
6363 if (appendAssistantTurn ) {
Original file line number Diff line number Diff line change @@ -58,26 +58,11 @@ public List<Integer> encodeHeader(Message message) {
5858 @ Override
5959 public List <Integer > encodeMessage (Message message ) {
6060 List <Integer > tokens = encodeHeader (message );
61- //tokens.add(beginOfInstruction);
6261 tokens .addAll (tokenizer .encodeAsList (message .content ().strip ()));
6362 tokens .add (endOfInstruction );
6463 return tokens ;
6564 }
6665
67- public List <Integer > encodeMessage (String userMessage , boolean addHeader , boolean addFooter ) {
68- List <Integer > tokens = new ArrayList <>();
69- if (addHeader ) {
70- tokens .add (this .beginOfInstruction );
71- }
72- if (userMessage != null ) {
73- tokens .addAll (this .tokenizer .encodeAsList (userMessage .strip ()));
74- }
75- if (addFooter ) {
76- tokens .add (endOfInstruction );
77- }
78- return tokens ;
79- }
80-
8166 public List <Integer > encodeFillInTheMiddle (String prefix , String suffix ) {
8267 List <Integer > tokens = new ArrayList <>();
8368 // dummy - empty string set to comply with encode method signature.
You can’t perform that action at this time.
0 commit comments