launch/opencode: add image modalities for vision models (#15922)

cmd/launch/opencode.go

@@ -1,6 +1,7 @@
 package launch
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -8,9 +9,12 @@ import (
 	"path/filepath"
 	"runtime"
 	"slices"
+	"time"
 
+	"github.com/ollama/ollama/api"
 	"github.com/ollama/ollama/cmd/internal/fileutil"
 	"github.com/ollama/ollama/envconfig"
+	"github.com/ollama/ollama/types/model"
 )
 
 // OpenCode implements Runner and Editor for OpenCode integration.
@@ -20,6 +24,8 @@ type OpenCode struct {
 	configContent string // JSON config built by Edit, passed to Run via env var
 }
 
+const openCodeModelShowTimeout = 2 * time.Second
+
 func (o *OpenCode) String() string { return "OpenCode" }
 
 // findOpenCode returns the opencode binary path, checking PATH first then the
@@ -176,6 +182,12 @@ func buildInlineConfig(primary string, models []string) (string, error) {
 	if primary == "" || len(models) == 0 {
 		return "", fmt.Errorf("buildInlineConfig: primary and models are required")
 	}
+
+	client, err := api.ClientFromEnvironment()
+	if err != nil {
+		client = nil
+	}
+
 	config := map[string]any{
 		"$schema": "https://opencode.ai/config.json",
 		"provider": map[string]any{
@@ -185,7 +197,7 @@ func buildInlineConfig(primary string, models []string) (string, error) {
 				"options": map[string]any{
 					"baseURL": envconfig.Host().String() + "/v1",
 				},
-				"models": buildModelEntries(models),
+				"models": buildModelEntries(context.Background(), client, models),
 			},
 		},
 		"model": "ollama/" + primary,
@@ -228,21 +240,39 @@ func readModelJSONModels() []string {
 	return models
 }
 
-func buildModelEntries(modelList []string) map[string]any {
-	models := make(map[string]any)
-	for _, model := range modelList {
-		entry := map[string]any{
-			"name": model,
+func buildModelEntries(ctx context.Context, client *api.Client, modelList []string) map[string]any {
+	if client != nil {
+		var cancel context.CancelFunc
+		if _, hasDeadline := ctx.Deadline(); !hasDeadline {
+			ctx, cancel = context.WithTimeout(ctx, openCodeModelShowTimeout)
+			defer cancel()
 		}
-		if isCloudModelName(model) {
-			if l, ok := lookupCloudModelLimit(model); ok {
+	}
+
+	models := make(map[string]any)
+	for _, modelID := range modelList {
+		entry := map[string]any{
+			"name": modelID,
+		}
+		if client != nil {
+			if resp, err := client.Show(ctx, &api.ShowRequest{Model: modelID}); err == nil {
+				if slices.Contains(resp.Capabilities, model.CapabilityVision) {
+					entry["modalities"] = map[string]any{
+						"input":  []string{"text", "image"},
+						"output": []string{"text"},
+					}
+				}
+			}
+		}
+		if isCloudModelName(modelID) {
+			if l, ok := lookupCloudModelLimit(modelID); ok {
 				entry["limit"] = map[string]any{
 					"context": l.Context,
 					"output":  l.Output,
 				}
 			}
 		}
-		models[model] = entry
+		models[modelID] = entry
 	}
 	return models
 }

cmd/launch/opencode_test.go

@@ -1,12 +1,21 @@
 package launch
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
+	"sync"
 	"testing"
+	"time"
+
+	"github.com/ollama/ollama/api"
 )
 
 func TestOpenCodeIntegration(t *testing.T) {
@@ -157,6 +166,105 @@ func TestOpenCodeEdit(t *testing.T) {
 			t.Errorf("local model should not have limit, got %v", entry["limit"])
 		}
 	})
+
+	t.Run("vision model gets image input modalities", func(t *testing.T) {
+		u, err := url.Parse("http://ollama.example")
+		if err != nil {
+			t.Fatalf("parse test URL: %v", err)
+		}
+		client := api.NewClient(u, &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+			if req.URL.Path != "/api/show" {
+				return &http.Response{
+					StatusCode: http.StatusNotFound,
+					Body:       io.NopCloser(strings.NewReader("not found")),
+					Header:     make(http.Header),
+				}, nil
+			}
+
+			var body struct {
+				Model string `json:"model"`
+			}
+			if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
+				t.Fatalf("decode show request: %v", err)
+			}
+			if body.Model != "gemma4:26b" {
+				t.Fatalf("show request model = %q, want gemma4:26b", body.Model)
+			}
+
+			return &http.Response{
+				StatusCode: http.StatusOK,
+				Body:       io.NopCloser(strings.NewReader(`{"capabilities":["vision"],"model_info":{}}`)),
+				Header:     make(http.Header),
+			}, nil
+		})})
+
+		models := buildModelEntries(context.Background(), client, []string{"gemma4:26b"})
+		entry, _ := models["gemma4:26b"].(map[string]any)
+		modalities, _ := entry["modalities"].(map[string]any)
+		input, _ := modalities["input"].([]string)
+		output, _ := modalities["output"].([]string)
+
+		if len(input) != 2 || input[0] != "text" || input[1] != "image" {
+			t.Fatalf("modalities.input = %v, want [text image]", input)
+		}
+		if len(output) != 1 || output[0] != "text" {
+			t.Fatalf("modalities.output = %v, want [text]", output)
+		}
+	})
+}
+
+func TestBuildModelEntries(t *testing.T) {
+	t.Run("defaults to model name when capabilities cannot be probed", func(t *testing.T) {
+		models := buildModelEntries(context.Background(), nil, []string{"llama3.2"})
+		entry, _ := models["llama3.2"].(map[string]any)
+		if entry["name"] != "llama3.2" {
+			t.Fatalf("name = %v, want llama3.2", entry["name"])
+		}
+		if _, ok := entry["modalities"]; ok {
+			t.Fatalf("modalities should not be set without an API client, got %v", entry["modalities"])
+		}
+	})
+
+	t.Run("uses one timeout budget across capability probes", func(t *testing.T) {
+		u, err := url.Parse("http://ollama.example")
+		if err != nil {
+			t.Fatalf("parse test URL: %v", err)
+		}
+
+		var mu sync.Mutex
+		waited := 0
+
+		client := api.NewClient(u, &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+			mu.Lock()
+			if req.Context().Err() == nil {
+				waited++
+			}
+			mu.Unlock()
+
+			<-req.Context().Done()
+			return nil, req.Context().Err()
+		})})
+
+		ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond)
+		defer cancel()
+
+		models := buildModelEntries(ctx, client, []string{"slow-1", "slow-2"})
+		for _, modelID := range []string{"slow-1", "slow-2"} {
+			entry, _ := models[modelID].(map[string]any)
+			if entry["name"] != modelID {
+				t.Fatalf("name for %q = %v, want %q", modelID, entry["name"], modelID)
+			}
+			if _, ok := entry["modalities"]; ok {
+				t.Fatalf("modalities for %q should not be set after probe timeout, got %v", modelID, entry["modalities"])
+			}
+		}
+
+		mu.Lock()
+		defer mu.Unlock()
+		if waited != 1 {
+			t.Fatalf("expected shared timeout to block one probe, waited on %d probes", waited)
+		}
+	})
 }
 
 func TestOpenCodeModels_ReturnsNil(t *testing.T) {