Skip to content

Commit 34bc1bd

Browse files
Copilotmudler
Copilot
and
mudler
authored
fix(api): SSE streaming format to comply with specification (#7182)
* Initial plan * Fix SSE streaming format to comply with specification - Replace json.Encoder with json.Marshal for explicit formatting - Use explicit \n\n for all SSE messages (instead of relying on implicit newlines) - Change %v to %s format specifier for proper string formatting - Fix error message streaming to include proper SSE format - Ensure consistency between chat.go and completion.go endpoints Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Add proper error handling for JSON marshal failures in streaming - Handle json.Marshal errors explicitly in error response paths - Add fallback simple error message if marshal fails - Prevents sending 'data: <nil>' on marshal failures - Addresses code review feedback Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Fix SSE streaming format to comply with specification Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Fix finish_reason field to use pointer for proper null handling - Change FinishReason from string to *string in Choice schema - Streaming chunks now omit finish_reason (null) instead of empty string - Final chunks properly set finish_reason to "stop", "tool_calls", etc. - Remove empty content from initial streaming chunks (only send role) - Final streaming chunk sends empty delta with finish_reason - Addresses OpenAI API compliance issues causing client failures Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Improve code consistency for string pointer creation - Use consistent pattern: declare variable then take address - Remove inline anonymous function for better readability - Addresses code review feedback Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Move common finish reasons to constants - Create constants.go with FinishReasonStop, FinishReasonToolCalls, FinishReasonFunctionCall - Replace all string literals with constants in chat.go, completion.go, realtime.go - Improves code maintainability and prevents typos Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Make it build Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix finish_reason to always be present with null or string value - Remove omitempty from FinishReason field in Choice struct - Explicitly set FinishReason to nil for all streaming chunks - Ensures finish_reason appears as null in JSON for streaming chunks - Final chunks still properly set finish_reason to "stop", "tool_calls", etc. - Complies with OpenAI API specification example Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto <mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 01cd58a commit 34bc1bd

File tree

8 files changed

+95
-47
lines changed

8 files changed

+95
-47
lines changed

‎.github/gallery-agent/go.mod‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ require (
88
github.com/onsi/gomega v1.38.2
99
github.com/sashabaranov/go-openai v1.41.2
1010
github.com/tmc/langchaingo v0.1.13
11-
gopkg.in/yaml.v3 v3.0.1
1211
)
1312

1413
require (

‎core/http/endpoints/openai/chat.go‎

Lines changed: 43 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package openai
22

33
import (
44
"bufio"
5-
"bytes"
65
"context"
76
"encoding/json"
87
"fmt"
@@ -91,7 +90,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
9190
ID: id,
9291
Created: created,
9392
Model: req.Model, // we have to return what the user sent here, due to OpenAI spec.
94-
Choices: []schema.Choice{{Delta: &schema.Message{Role: "assistant", Content: &textContentToReturn}}},
93+
Choices: []schema.Choice{{Delta: &schema.Message{Role: "assistant"}, Index: 0, FinishReason: nil}},
9594
Object: "chat.completion.chunk",
9695
}
9796
responses <- initialMessage
@@ -111,7 +110,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
111110
ID: id,
112111
Created: created,
113112
Model: req.Model, // we have to return what the user sent here, due to OpenAI spec.
114-
Choices: []schema.Choice{{Delta: &schema.Message{Content: &s}, Index: 0}},
113+
Choices: []schema.Choice{{Delta: &schema.Message{Content: &s}, Index: 0, FinishReason: nil}},
115114
Object: "chat.completion.chunk",
116115
Usage: usage,
117116
}
@@ -145,7 +144,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
145144
ID: id,
146145
Created: created,
147146
Model: req.Model, // we have to return what the user sent here, due to OpenAI spec.
148-
Choices: []schema.Choice{{Delta: &schema.Message{Role: "assistant", Content: &textContentToReturn}}},
147+
Choices: []schema.Choice{{Delta: &schema.Message{Role: "assistant"}, Index: 0, FinishReason: nil}},
149148
Object: "chat.completion.chunk",
150149
}
151150
responses <- initialMessage
@@ -169,7 +168,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
169168
ID: id,
170169
Created: created,
171170
Model: req.Model, // we have to return what the user sent here, due to OpenAI spec.
172-
Choices: []schema.Choice{{Delta: &schema.Message{Content: &result}, Index: 0}},
171+
Choices: []schema.Choice{{Delta: &schema.Message{Content: &result}, Index: 0, FinishReason: nil}},
173172
Object: "chat.completion.chunk",
174173
Usage: usage,
175174
}
@@ -197,7 +196,10 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
197196
},
198197
},
199198
},
200-
}}},
199+
},
200+
Index: 0,
201+
FinishReason: nil,
202+
}},
201203
Object: "chat.completion.chunk",
202204
}
203205
responses <- initialMessage
@@ -220,7 +222,10 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
220222
},
221223
},
222224
},
223-
}}},
225+
},
226+
Index: 0,
227+
FinishReason: nil,
228+
}},
224229
Object: "chat.completion.chunk",
225230
}
226231
}
@@ -427,11 +432,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
427432
if len(ev.Choices[0].Delta.ToolCalls) > 0 {
428433
toolsCalled = true
429434
}
430-
var buf bytes.Buffer
431-
enc := json.NewEncoder(&buf)
432-
enc.Encode(ev)
433-
log.Debug().Msgf("Sending chunk: %s", buf.String())
434-
_, err := fmt.Fprintf(w, "data: %v\n", buf.String())
435+
respData, err := json.Marshal(ev)
436+
if err != nil {
437+
log.Debug().Msgf("Failed to marshal response: %v", err)
438+
input.Cancel()
439+
continue
440+
}
441+
log.Debug().Msgf("Sending chunk: %s", string(respData))
442+
_, err = fmt.Fprintf(w, "data: %s\n\n", string(respData))
435443
if err != nil {
436444
log.Debug().Msgf("Sending chunk failed: %v", err)
437445
input.Cancel()
@@ -443,34 +451,40 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
443451
}
444452
log.Error().Msgf("Stream ended with error: %v", err)
445453

454+
stopReason := FinishReasonStop
446455
resp := &schema.OpenAIResponse{
447456
ID: id,
448457
Created: created,
449458
Model: input.Model, // we have to return what the user sent here, due to OpenAI spec.
450459
Choices: []schema.Choice{
451460
{
452-
FinishReason: "stop",
461+
FinishReason: &stopReason,
453462
Index: 0,
454463
Delta: &schema.Message{Content: "Internal error: " + err.Error()},
455464
}},
456465
Object: "chat.completion.chunk",
457466
Usage: *usage,
458467
}
459-
respData, _ := json.Marshal(resp)
460-
461-
w.WriteString(fmt.Sprintf("data: %s\n\n", respData))
468+
respData, marshalErr := json.Marshal(resp)
469+
if marshalErr != nil {
470+
log.Error().Msgf("Failed to marshal error response: %v", marshalErr)
471+
// Send a simple error message as fallback
472+
w.WriteString("data: {\"error\":\"Internal error\"}\n\n")
473+
} else {
474+
w.WriteString(fmt.Sprintf("data: %s\n\n", respData))
475+
}
462476
w.WriteString("data: [DONE]\n\n")
463477
w.Flush()
464478

465479
return
466480
}
467481
}
468482

469-
finishReason := "stop"
483+
finishReason := FinishReasonStop
470484
if toolsCalled && len(input.Tools) > 0 {
471-
finishReason = "tool_calls"
485+
finishReason = FinishReasonToolCalls
472486
} else if toolsCalled {
473-
finishReason = "function_call"
487+
finishReason = FinishReasonFunctionCall
474488
}
475489

476490
resp := &schema.OpenAIResponse{
@@ -479,9 +493,9 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
479493
Model: input.Model, // we have to return what the user sent here, due to OpenAI spec.
480494
Choices: []schema.Choice{
481495
{
482-
FinishReason: finishReason,
496+
FinishReason: &finishReason,
483497
Index: 0,
484-
Delta: &schema.Message{Content: &textContentToReturn},
498+
Delta: &schema.Message{},
485499
}},
486500
Object: "chat.completion.chunk",
487501
Usage: *usage,
@@ -502,7 +516,8 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
502516
tokenCallback := func(s string, c *[]schema.Choice) {
503517
if !shouldUseFn {
504518
// no function is called, just reply and use stop as finish reason
505-
*c = append(*c, schema.Choice{FinishReason: "stop", Index: 0, Message: &schema.Message{Role: "assistant", Content: &s}})
519+
stopReason := FinishReasonStop
520+
*c = append(*c, schema.Choice{FinishReason: &stopReason, Index: 0, Message: &schema.Message{Role: "assistant", Content: &s}})
506521
return
507522
}
508523

@@ -520,12 +535,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
520535
return
521536
}
522537

538+
stopReason := FinishReasonStop
523539
*c = append(*c, schema.Choice{
524-
FinishReason: "stop",
540+
FinishReason: &stopReason,
525541
Message: &schema.Message{Role: "assistant", Content: &result}})
526542
default:
543+
toolCallsReason := FinishReasonToolCalls
527544
toolChoice := schema.Choice{
528-
FinishReason: "tool_calls",
545+
FinishReason: &toolCallsReason,
529546
Message: &schema.Message{
530547
Role: "assistant",
531548
},
@@ -549,8 +566,9 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
549566
)
550567
} else {
551568
// otherwise we return more choices directly (deprecated)
569+
functionCallReason := FinishReasonFunctionCall
552570
*c = append(*c, schema.Choice{
553-
FinishReason: "function_call",
571+
FinishReason: &functionCallReason,
554572
Message: &schema.Message{
555573
Role: "assistant",
556574
Content: &textContentToReturn,

‎core/http/endpoints/openai/completion.go‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package openai
22

33
import (
44
"bufio"
5-
"bytes"
65
"encoding/json"
76
"errors"
87
"fmt"
@@ -47,8 +46,9 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
4746
Model: req.Model, // we have to return what the user sent here, due to OpenAI spec.
4847
Choices: []schema.Choice{
4948
{
50-
Index: 0,
51-
Text: s,
49+
Index: 0,
50+
Text: s,
51+
FinishReason: nil,
5252
},
5353
},
5454
Object: "text_completion",
@@ -140,32 +140,57 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
140140
log.Debug().Msgf("No choices in the response, skipping")
141141
continue
142142
}
143-
var buf bytes.Buffer
144-
enc := json.NewEncoder(&buf)
145-
enc.Encode(ev)
143+
respData, err := json.Marshal(ev)
144+
if err != nil {
145+
log.Debug().Msgf("Failed to marshal response: %v", err)
146+
continue
147+
}
146148

147-
log.Debug().Msgf("Sending chunk: %s", buf.String())
148-
fmt.Fprintf(w, "data: %v\n", buf.String())
149+
log.Debug().Msgf("Sending chunk: %s", string(respData))
150+
fmt.Fprintf(w, "data: %s\n\n", string(respData))
149151
w.Flush()
150152
case err := <-ended:
151153
if err == nil {
152154
break LOOP
153155
}
154156
log.Error().Msgf("Stream ended with error: %v", err)
155-
fmt.Fprintf(w, "data: %v\n", "Internal error: "+err.Error())
157+
158+
stopReason := FinishReasonStop
159+
errorResp := schema.OpenAIResponse{
160+
ID: id,
161+
Created: created,
162+
Model: input.Model,
163+
Choices: []schema.Choice{
164+
{
165+
Index: 0,
166+
FinishReason: &stopReason,
167+
Text: "Internal error: " + err.Error(),
168+
},
169+
},
170+
Object: "text_completion",
171+
}
172+
errorData, marshalErr := json.Marshal(errorResp)
173+
if marshalErr != nil {
174+
log.Error().Msgf("Failed to marshal error response: %v", marshalErr)
175+
// Send a simple error message as fallback
176+
fmt.Fprintf(w, "data: {\"error\":\"Internal error\"}\n\n")
177+
} else {
178+
fmt.Fprintf(w, "data: %s\n\n", string(errorData))
179+
}
156180
w.Flush()
157181
break LOOP
158182
}
159183
}
160184

185+
stopReason := FinishReasonStop
161186
resp := &schema.OpenAIResponse{
162187
ID: id,
163188
Created: created,
164189
Model: input.Model, // we have to return what the user sent here, due to OpenAI spec.
165190
Choices: []schema.Choice{
166191
{
167192
Index: 0,
168-
FinishReason: "stop",
193+
FinishReason: &stopReason,
169194
},
170195
},
171196
Object: "text_completion",
@@ -197,7 +222,8 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
197222

198223
r, tokenUsage, err := ComputeChoices(
199224
input, i, config, cl, appConfig, ml, func(s string, c *[]schema.Choice) {
200-
*c = append(*c, schema.Choice{Text: s, FinishReason: "stop", Index: k})
225+
stopReason := FinishReasonStop
226+
*c = append(*c, schema.Choice{Text: s, FinishReason: &stopReason, Index: k})
201227
}, nil)
202228
if err != nil {
203229
return err

‎core/http/endpoints/openai/constants.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package openai
2+
3+
// Finish reason constants for OpenAI API responses
4+
const (
5+
FinishReasonStop = "stop"
6+
FinishReasonToolCalls = "tool_calls"
7+
FinishReasonFunctionCall = "function_call"
8+
)

‎core/http/endpoints/openai/realtime.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1072,7 +1072,8 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
10721072
result, tokenUsage, err := ComputeChoices(input, prompt, config, startupOptions, ml, func(s string, c *[]schema.Choice) {
10731073
if !shouldUseFn {
10741074
// no function is called, just reply and use stop as finish reason
1075-
*c = append(*c, schema.Choice{FinishReason: "stop", Index: 0, Message: &schema.Message{Role: "assistant", Content: &s}})
1075+
stopReason := FinishReasonStop
1076+
*c = append(*c, schema.Choice{FinishReason: &stopReason, Index: 0, Message: &schema.Message{Role: "assistant", Content: &s}})
10761077
return
10771078
}
10781079
@@ -1099,7 +1100,8 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
10991100
}
11001101
11011102
if len(input.Tools) > 0 {
1102-
toolChoice.FinishReason = "tool_calls"
1103+
toolCallsReason := FinishReasonToolCalls
1104+
toolChoice.FinishReason = &toolCallsReason
11031105
}
11041106
11051107
for _, ss := range results {
@@ -1120,8 +1122,9 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
11201122
)
11211123
} else {
11221124
// otherwise we return more choices directly
1125+
functionCallReason := FinishReasonFunctionCall
11231126
*c = append(*c, schema.Choice{
1124-
FinishReason: "function_call",
1127+
FinishReason: &functionCallReason,
11251128
Message: &schema.Message{
11261129
Role: "assistant",
11271130
Content: &textContentToReturn,

‎core/schema/openai.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ type OpenAIResponse struct {
5050

5151
type Choice struct {
5252
Index int `json:"index"`
53-
FinishReason string `json:"finish_reason"`
53+
FinishReason *string `json:"finish_reason"`
5454
Message *Message `json:"message,omitempty"`
5555
Delta *Message `json:"delta,omitempty"`
5656
Text string `json:"text,omitempty"`

‎docs/go.mod‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
11
module github.com/McShelby/hugo-theme-relearn.git
22

33
go 1.19
4-
5-
require github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20200 // indirect

‎docs/go.sum‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +0,0 @@
1-
github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20200 h1:SmpwwN3DNzJWbV+IT8gaFu07ENUFpCvKou5BHYUKuVs=
2-
github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20200/go.mod h1:kx8MBj9T7SFR8ZClWvKZPmmUxBaltkoXvnWlZZcSnYA=
3-
github.com/gohugoio/hugo-mod-jslibs-dist/popperjs/v2 v2.21100.20000/go.mod h1:mFberT6ZtcchrsDtfvJM7aAH2bDKLdOnruUHl0hlapI=
4-
github.com/twbs/bootstrap v5.3.2+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0=

0 commit comments

Comments
 (0)