CERBERUS/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2026-10-21 11:01:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

openai: keep image positions when merging tool result parts

Browse source 
Merging a tool result's parts collected its images separately from the
text, so renderers placed them all before the text: "Before: <A> After:
<B>" rendered as "<A><B>Before: After:". Write an [img] placeholder at
each image's position so renderContentWithImageTags and chatPrompt put
each image back where it was.
This commit is contained in:
Dong Chen 2026-10-01 19:42:33 -07:00 • committed by Dong Chen
parent 77cef85de4
commit bea35c8984
2 changed files with 16 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

5
openai/openai.go
View file

@ -684,11 +684,12 @@ func FromChatRequest(r ChatCompletionRequest, thinking ...*model.Thinking) (*api
}
}
// a tool message is a single result, so keep its parts together and
// preserve the tool call it answers
// preserve the tool call it answers. an [img] placeholder keeps each
// image at its position among the text parts
if strings.ToLower(msg.Role) == "tool" && (len(messages) > start || toolName != "" || msg.ToolCallID != "") {
tool := api.Message{Role: msg.Role, ToolName: toolName, ToolCallID: msg.ToolCallID}
for _, part := range messages[start:] {
tool.Content += part.Content
tool.Content += part.Content + strings.Repeat("[img]", len(part.Images))
tool.Images = append(tool.Images, part.Images...)
}
messages = append(messages[:start], tool)

19
openai/openai_test.go
View file

@ -158,7 +158,9 @@ func TestFromChatRequest_WithImage(t *testing.T) {
}
func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
const image2 = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==`
imgData, _ := base64.StdEncoding.DecodeString(image)
img2Data, _ := base64.StdEncoding.DecodeString(image2)
req := ChatCompletionRequest{
Model: "test-model",
@ -179,12 +181,16 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
Role: "tool",
ToolCallID: "call_1",
Content: []any{
map[string]any{"type": "text", "text": "Captured "},
map[string]any{"type": "text", "text": "Before: "},
map[string]any{
"type": "image_url",
"image_url": map[string]any{"url": prefix + image},
},
map[string]any{"type": "text", "text": "screen.png"},
map[string]any{"type": "text", "text": " After: "},
map[string]any{
"type": "image_url",
"image_url": map[string]any{"url": prefix + image2},
},
},
},
},
@ -203,11 +209,12 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
if tool.Role != "tool" || tool.ToolCallID != "call_1" || tool.ToolName != "screenshot" {
t.Errorf("expected tool message for call_1/screenshot, got role=%q id=%q name=%q", tool.Role, tool.ToolCallID, tool.ToolName)
}
if tool.Content != "Captured screen.png" {
t.Errorf("expected content 'Captured screen.png', got %q", tool.Content)
// each image keeps its position among the text parts
if tool.Content != "Before: [img] After: [img]" {
t.Errorf("expected content 'Before: [img] After: [img]', got %q", tool.Content)
}
if len(tool.Images) != 1 || string(tool.Images[0]) != string(imgData) {
t.Errorf("expected the tool image to be kept, got %d images", len(tool.Images))
if len(tool.Images) != 2 || string(tool.Images[0]) != string(imgData) || string(tool.Images[1]) != string(img2Data) {
t.Errorf("expected both tool images to be kept in order, got %d images", len(tool.Images))
}
}