From bea35c898464ed250e6067f69fdd98ce588a0055 Mon Sep 17 00:00:00 2001
From: Dong Chen <dongluochen@ollama.com>
Date: Thu, 1 Oct 2026 19:42:33 -0700
Subject: [PATCH] openai: keep image positions when merging tool result parts

Merging a tool result's parts collected its images separately from the
text, so renderers placed them all before the text: "Before: <A> After:
<B>" rendered as "<A><B>Before: After:". Write an [img] placeholder at
each image's position so renderContentWithImageTags and chatPrompt put
each image back where it was.
---
 openai/openai.go      |  5 +++--
 openai/openai_test.go | 19 +++++++++++++------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/openai/openai.go b/openai/openai.go
index a3cc19a65..8a8550977 100644
--- a/openai/openai.go
+++ b/openai/openai.go
@@ -684,11 +684,12 @@ func FromChatRequest(r ChatCompletionRequest, thinking ...*model.Thinking) (*api
 				}
 			}
 			// a tool message is a single result, so keep its parts together and
-			// preserve the tool call it answers
+			// preserve the tool call it answers. an [img] placeholder keeps each
+			// image at its position among the text parts
 			if strings.ToLower(msg.Role) == "tool" && (len(messages) > start || toolName != "" || msg.ToolCallID != "") {
 				tool := api.Message{Role: msg.Role, ToolName: toolName, ToolCallID: msg.ToolCallID}
 				for _, part := range messages[start:] {
-					tool.Content += part.Content
+					tool.Content += part.Content + strings.Repeat("[img]", len(part.Images))
 					tool.Images = append(tool.Images, part.Images...)
 				}
 				messages = append(messages[:start], tool)
diff --git a/openai/openai_test.go b/openai/openai_test.go
index 6d72a9907..1b9ec1e7c 100644
--- a/openai/openai_test.go
+++ b/openai/openai_test.go
@@ -158,7 +158,9 @@ func TestFromChatRequest_WithImage(t *testing.T) {
 }
 
 func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
+	const image2 = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==`
 	imgData, _ := base64.StdEncoding.DecodeString(image)
+	img2Data, _ := base64.StdEncoding.DecodeString(image2)
 
 	req := ChatCompletionRequest{
 		Model: "test-model",
@@ -179,12 +181,16 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
 				Role:       "tool",
 				ToolCallID: "call_1",
 				Content: []any{
-					map[string]any{"type": "text", "text": "Captured "},
+					map[string]any{"type": "text", "text": "Before: "},
 					map[string]any{
 						"type":      "image_url",
 						"image_url": map[string]any{"url": prefix + image},
 					},
-					map[string]any{"type": "text", "text": "screen.png"},
+					map[string]any{"type": "text", "text": " After: "},
+					map[string]any{
+						"type":      "image_url",
+						"image_url": map[string]any{"url": prefix + image2},
+					},
 				},
 			},
 		},
@@ -203,11 +209,12 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
 	if tool.Role != "tool" || tool.ToolCallID != "call_1" || tool.ToolName != "screenshot" {
 		t.Errorf("expected tool message for call_1/screenshot, got role=%q id=%q name=%q", tool.Role, tool.ToolCallID, tool.ToolName)
 	}
-	if tool.Content != "Captured screen.png" {
-		t.Errorf("expected content 'Captured screen.png', got %q", tool.Content)
+	// each image keeps its position among the text parts
+	if tool.Content != "Before: [img] After: [img]" {
+		t.Errorf("expected content 'Before: [img] After: [img]', got %q", tool.Content)
 	}
-	if len(tool.Images) != 1 || string(tool.Images[0]) != string(imgData) {
-		t.Errorf("expected the tool image to be kept, got %d images", len(tool.Images))
+	if len(tool.Images) != 2 || string(tool.Images[0]) != string(imgData) || string(tool.Images[1]) != string(img2Data) {
+		t.Errorf("expected both tool images to be kept in order, got %d images", len(tool.Images))
 	}
 }