cli: add decision model support to ollama run (#18962)

This commit is contained in:
Jeffrey Morgan 2026-10-13 12:39:05 -07:00 • committed by GitHub
parent 72f61ca412
commit 7a5ae12ead
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 749 additions and 65 deletions

View file

@ -443,6 +443,15 @@ func (c *Client) Heartbeat(ctx context.Context) error {
return nil
}
// SystemOne answers typed questions about a state with a decision model.
func (c *Client) SystemOne(ctx context.Context, req *SystemOneRequest) (*SystemOneResponse, error) {
var resp SystemOneResponse
if err := c.do(ctx, http.MethodPost, "/v1/systemone", req, &resp); err != nil {
return nil, err
}
return &resp, nil
}
// Embed generates embeddings from a model.
func (c *Client) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) {
var resp EmbedResponse

View file

@ -601,6 +601,48 @@ type Runner struct {
DraftNumPredict int `json:"draft_num_predict,omitempty"`
}
// SystemOneRequest is the request passed to [Client.SystemOne]. It asks a
// decision model typed questions about a state.
type SystemOneRequest struct {
Model string `json:"model"`
Runner string `json:"runner,omitempty"`
State json.RawMessage `json:"state"`
Images []ImageData `json:"images,omitempty"`
Videos []json.RawMessage `json:"videos,omitempty"`
Questions *SystemOneQuestions `json:"questions"`
KeepAlive *Duration `json:"keep_alive,omitempty"`
}
// SystemOneQuestions keeps questions in request order, which is the order of
// the answers.
type SystemOneQuestions = orderedmap.Map[string, SystemOneQuestion]
// SystemOneQuestion is a noul (yes/no), choice, or score question. Criteria is
// an object for noul and choice questions and an ordered array for score.
type SystemOneQuestion struct {
Type string `json:"type"`
Instructions json.RawMessage `json:"instructions"`
Criteria json.RawMessage `json:"criteria,omitempty"`
}
// SystemOneResponse is the response returned by [Client.SystemOne]. Each
// answer's type field says which answer fields it has.
type SystemOneResponse struct {
Model string `json:"model"`
Answers *SystemOneAnswers `json:"answers"`
Usage SystemOneUsage `json:"usage"`
PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`
}
// SystemOneAnswers keeps answers in request order.
type SystemOneAnswers = orderedmap.Map[string, json.RawMessage]
// SystemOneUsage counts the tokens evaluated and generated to answer.
type SystemOneUsage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
}
// EmbedRequest is the request passed to [Client.Embed].
type EmbedRequest struct {
// Model is the model name.

View file

@ -972,24 +972,15 @@ func RunHandler(cmd *cobra.Command, args []string) error {
return err
}
audioCapable := slices.Contains(info.Capabilities, model.CapabilityAudio)
opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || audioCapable
// TODO: remove the projector info and vision info checks below,
// these are left in for backwards compatibility with older servers
// that don't have the capabilities field in the model info
if len(info.ProjectorInfo) != 0 {
opts.MultiModal = true
}
for k := range info.ModelInfo {
if strings.Contains(k, ".vision.") {
opts.MultiModal = true
break
}
}
applyShowResponseToRunOptions(&opts, info)
if slices.Contains(info.Capabilities, model.CapabilityDecision) {
return runDecision(cmd, opts, interactive)
}
if cmd.Flags().Changed("ask") {
return fmt.Errorf("%s is not a decision model, so it can't answer --ask questions", name)
}
// Check if this is an embedding model
isEmbeddingModel := slices.Contains(info.Capabilities, model.CapabilityEmbedding)
@ -1824,6 +1815,8 @@ type runOptions struct {
Think *api.ThinkValue
HideThinking bool
ShowConnect bool
Decision bool
Questions *api.SystemOneQuestions
}
func (r runOptions) Copy() runOptions {
@ -1876,12 +1869,27 @@ func (r runOptions) Copy() runOptions {
Think: think,
HideThinking: r.HideThinking,
ShowConnect: r.ShowConnect,
Decision: r.Decision,
Questions: r.Questions,
}
}
func applyShowResponseToRunOptions(opts *runOptions, info *api.ShowResponse) {
opts.ParentModel = info.Details.ParentModel
opts.LoadedMessages = slices.Clone(info.Messages)
opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) ||
slices.Contains(info.Capabilities, model.CapabilityAudio)
// Keep media input working with older servers that do not report capabilities.
if len(info.ProjectorInfo) != 0 {
opts.MultiModal = true
}
for key := range info.ModelInfo {
if strings.Contains(key, ".vision.") {
opts.MultiModal = true
break
}
}
}
type displayResponseState struct {
@ -2363,18 +2371,6 @@ func launchInteractiveModel(cmd *cobra.Command, modelName string) error {
return err
}
audioCapable := slices.Contains(info.Capabilities, model.CapabilityAudio)
opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || audioCapable
if len(info.ProjectorInfo) != 0 {
opts.MultiModal = true
}
for key := range info.ModelInfo {
if strings.Contains(key, ".vision.") {
opts.MultiModal = true
break
}
}
applyShowResponseToRunOptions(&opts, info)
if err := loadOrUnloadModel(cmd, &opts); err != nil {
@ -2571,8 +2567,18 @@ func NewCLI() *cobra.Command {
showCmd.Flags().BoolP("verbose", "v", false, "Show detailed model information")
runCmd := &cobra.Command{
Use: "run MODEL [PROMPT]",
Short: "Run a model",
Use: "run MODEL [PROMPT]",
Short: "Run a model",
Example: ` ollama run gemma4
ollama run gemma4 "Why is the sky blue?"
ollama run nimble --ask "Is a refund being requested?" < ticket.txt
ollama run nimble --ask "Which team should handle this?" \
--choice billing="Payments and refunds" \
--choice technical="Software problems" < ticket.txt
ollama run nimble --ask "How urgent is this?" \
--level "Routine" \
--level "Soon" \
--level "Immediate" < ticket.txt`,
Args: cobra.MinimumNArgs(1),
PreRunE: checkServerHeartbeat,
RunE: RunHandler,
@ -2590,6 +2596,10 @@ func NewCLI() *cobra.Command {
runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)")
runCmd.Flags().Bool("truncate", false, "For embedding models: truncate inputs exceeding context length (default: true). Set --truncate=false to error instead")
runCmd.Flags().Int("dimensions", 0, "Truncate output embeddings to specified dimension (embedding models only)")
var asked []askedQuestion
runCmd.Flags().Var(questionFlag{"ask", &asked}, "ask", "Ask a decision model a question, answered yes or no unless --choice or --level follow")
runCmd.Flags().Var(questionFlag{"choice", &asked}, "choice", `Add a label for the preceding --ask to choose from, e.g. billing="Payments and refunds"`)
runCmd.Flags().Var(questionFlag{"level", &asked}, "level", "Add a level to score the preceding --ask on, from lowest to highest")
stopCmd := &cobra.Command{
Use: "stop MODEL",

View file

@ -2497,6 +2497,44 @@ func TestApplyShowResponseToRunOptions(t *testing.T) {
}
}
func TestApplyShowResponseRefreshesMultimodal(t *testing.T) {
for _, tt := range []struct {
name string
info api.ShowResponse
}{
{
name: "decision vision",
info: api.ShowResponse{Capabilities: []model.Capability{model.CapabilityDecision, model.CapabilityVision}},
},
{
name: "audio",
info: api.ShowResponse{Capabilities: []model.Capability{model.CapabilityAudio}},
},
{
name: "legacy projector",
info: api.ShowResponse{ProjectorInfo: map[string]any{"general.architecture": "clip"}},
},
{
name: "legacy vision",
info: api.ShowResponse{ModelInfo: map[string]any{"mllama.vision.block_count": 32}},
},
} {
t.Run(tt.name, func(t *testing.T) {
opts := runOptions{}
applyShowResponseToRunOptions(&opts, &tt.info)
if !opts.MultiModal {
t.Fatal("loading a multimodal model did not enable media input")
}
applyShowResponseToRunOptions(&opts, &api.ShowResponse{
Capabilities: []model.Capability{model.CapabilityDecision},
})
if opts.MultiModal {
t.Fatal("loading a text-only model did not disable media input")
}
})
}
}
func TestRunOptions_Copy(t *testing.T) {
// Setup test data
originalKeepAlive := &api.Duration{Duration: 5 * time.Minute}

227
cmd/decide.go Normal file
View file

@ -0,0 +1,227 @@
package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/signal"
"strconv"
"strings"
"text/tabwriter"
"time"
"github.com/spf13/cobra"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/internal/orderedmap"
"github.com/ollama/ollama/progress"
)
// runDecision runs a decision model, which answers the --ask questions about
// a state: the text after the model name, or stdin.
func runDecision(cmd *cobra.Command, opts runOptions, interactive bool) error {
opts.Questions = askedQuestions(cmd)
if interactive {
opts.Decision = true
return generateInteractive(cmd, opts)
}
if opts.Prompt == "" || opts.Questions.Len() == 0 {
return fmt.Errorf("%s answers questions about text, e.g. ollama run %s --ask \"Is a refund being requested?\" < ticket.txt", opts.Model, opts.Model)
}
input := api.Message{Role: "user", Content: opts.Prompt}
if opts.MultiModal {
var err error
if input.Content, input.Images, err = extractFileData(input.Content); err != nil {
return err
}
}
return decide(cmd, opts, input, opts.Questions)
}
// yesNoQuestions asks each text as a yes/no question keyed by the text itself.
func yesNoQuestions(texts ...string) *api.SystemOneQuestions {
questions := &api.SystemOneQuestions{}
for _, text := range texts {
instructions, _ := json.Marshal(text)
questions.Set(text, api.SystemOneQuestion{Type: "noul", Instructions: instructions})
}
return questions
}
// questionFlag is --ask, --choice or --level. The three share the questions
// being asked, so each --choice or --level answers the --ask before it.
type questionFlag struct {
name string
asked *[]askedQuestion
}
type askedQuestion struct {
text string
choices *orderedmap.Map[string, any]
levels []string
}
func (f questionFlag) String() string { return "" }
func (f questionFlag) Type() string { return "string" }
func (f questionFlag) Set(value string) error {
if f.name == "ask" {
*f.asked = append(*f.asked, askedQuestion{text: value})
return nil
}
if len(*f.asked) == 0 {
return errors.New("must come after the --ask it answers")
}
q := &(*f.asked)[len(*f.asked)-1]
if f.name == "level" {
q.levels = append(q.levels, value)
} else {
if q.choices == nil {
q.choices = orderedmap.New[string, any]()
}
// A label may have a description: billing="Payments and refunds"
if label, description, ok := strings.Cut(value, "="); ok {
q.choices.Set(label, description)
} else {
q.choices.Set(value, nil)
}
}
if q.choices != nil && q.levels != nil {
return errors.New("a question takes --choice or --level, not both")
}
return nil
}
// askedQuestions returns the --ask questions keyed by their text: yes/no, or
// answered with a --choice label or on a --level scale.
func askedQuestions(cmd *cobra.Command) *api.SystemOneQuestions {
questions := &api.SystemOneQuestions{}
flag := cmd.Flags().Lookup("ask")
if flag == nil {
return questions
}
for _, q := range *flag.Value.(questionFlag).asked {
question := api.SystemOneQuestion{Type: "noul"}
question.Instructions, _ = json.Marshal(q.text)
switch {
case q.choices != nil:
question.Type = "choice"
question.Criteria, _ = json.Marshal(q.choices)
case q.levels != nil:
question.Type = "score"
question.Criteria, _ = json.Marshal(q.levels)
}
questions.Set(q.text, question)
}
return questions
}
// decide asks a decision model questions about a state and prints each answer:
// the likelier answer to a yes/no question, the chosen label, or the most
// likely score level, with its probability, and for a score the score itself.
func decide(cmd *cobra.Command, opts runOptions, input api.Message, questions *api.SystemOneQuestions) error {
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
// Piped text and multiline messages end in a newline that isn't part of the state.
text := strings.TrimRight(input.Content, "\r\n")
state, _ := json.Marshal(text)
if s := strings.TrimSpace(text); strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[") {
if json.Valid([]byte(s)) {
state = json.RawMessage(s)
}
}
req := &api.SystemOneRequest{
Model: opts.Model,
Runner: opts.Runner,
State: state,
Images: input.Images,
Questions: questions,
KeepAlive: opts.KeepAlive,
}
ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt)
defer stop()
p := progress.NewProgress(os.Stderr)
p.Add("", progress.NewSpinner(""))
start := time.Now()
resp, err := client.SystemOne(ctx, req)
p.StopAndClear()
if errors.Is(err, context.Canceled) {
return nil
} else if err != nil {
return err
}
if opts.Format == "json" {
enc := json.NewEncoder(os.Stdout)
enc.SetEscapeHTML(false)
return enc.Encode(resp)
}
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
for name := range questions.All() {
raw, ok := resp.Answers.Get(name)
if !ok {
return fmt.Errorf("no answer to %q", name)
}
var a struct {
Type string `json:"type"`
Noul float64 `json:"noul"`
Choice string `json:"choice"`
Score float64 `json:"score"`
Legend map[string]any `json:"legend"`
Probabilities map[string]float64 `json:"probabilities"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return fmt.Errorf("answer to %q: %w", name, err)
}
var answer, measure string
switch a.Type {
case "noul":
answer, measure = "yes", fmt.Sprintf("%.0f%%", a.Noul*100)
if a.Noul < 0.5 {
answer, measure = "no", fmt.Sprintf("%.0f%%", (1-a.Noul)*100)
}
case "choice":
answer, measure = a.Choice, fmt.Sprintf("%.0f%%", a.Probabilities[a.Choice]*100)
case "score":
// Not the level nearest the score: an answer split between the
// ends of the scale scores in the middle.
level := "0"
for i := 1; i < len(a.Legend); i++ {
if k := strconv.Itoa(i); a.Probabilities[k] > a.Probabilities[level] {
level = k
}
}
if v, ok := a.Legend[level]; ok {
answer = fmt.Sprint(v)
}
measure = fmt.Sprintf("%.0f%%\t%.1f / %d", a.Probabilities[level]*100, a.Score, len(a.Legend)-1)
default:
return fmt.Errorf("answer to %q has unknown type %q", name, a.Type)
}
if questions.Len() > 1 {
fmt.Fprintf(tw, "%s\t", name)
}
fmt.Fprintf(tw, "%s\t%s\n", answer, measure)
}
if err := tw.Flush(); err != nil {
return err
}
if verbose, _ := cmd.Flags().GetBool("verbose"); verbose {
metrics := api.Metrics{
TotalDuration: time.Since(start),
PromptEvalCount: resp.Usage.InputTokens,
PromptEvalCachedCount: resp.PromptEvalCachedCount,
}
metrics.Summary()
}
return nil
}

204
cmd/decide_test.go Normal file
View file

@ -0,0 +1,204 @@
package cmd
import (
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/types/model"
)
// runDecide runs `ollama run args...` with stdin piped in. nimble is a
// decision model and any other model is a chat model.
func runDecide(t *testing.T, stdin string, handler http.HandlerFunc, args ...string) (string, error) {
t.Helper()
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/show" && r.Method == http.MethodPost {
var req api.ShowRequest
_ = json.NewDecoder(r.Body).Decode(&req)
capability := model.CapabilityCompletion
if req.Model == "nimble" {
capability = model.CapabilityDecision
}
_ = json.NewEncoder(w).Encode(api.ShowResponse{Capabilities: []model.Capability{capability}})
return
}
if r.URL.Path == "/v1/systemone" && r.Method == http.MethodPost {
handler(w, r)
return
}
http.NotFound(w, r)
}))
t.Setenv("OLLAMA_HOST", mockServer.URL)
t.Cleanup(mockServer.Close)
cmd, _, err := NewCLI().Find([]string{"run"})
if err != nil {
t.Fatal(err)
}
cmd.SetContext(t.Context())
if err := cmd.ParseFlags(args); err != nil {
return "", err
}
oldStdin := os.Stdin
stdinR, stdinW, _ := os.Pipe()
defer stdinR.Close()
_, _ = io.WriteString(stdinW, stdin)
stdinW.Close()
os.Stdin = stdinR
defer func() { os.Stdin = oldStdin }()
out := captureStdout(t, func() {
err = RunHandler(cmd, cmd.Flags().Args())
})
return out, err
}
func TestRunDecisionModel(t *testing.T) {
const refund = `{"Is a refund being requested?":{"type":"noul","instructions":"Is a refund being requested?"}}`
for _, tt := range []struct {
name, stdin string
args []string
state string
questions string
answers string
want string
}{
{
name: "yes/no",
stdin: "I was charged twice\n",
args: []string{"--ask", "Is a refund being requested?"},
questions: refund,
answers: `{"Is a refund being requested?":{"type":"noul","noul":0.97}}`,
want: "yes 97%\n",
},
{
name: "state as an argument",
args: []string{"I was charged twice", "--ask", "Is a refund being requested?"},
questions: refund,
answers: `{"Is a refund being requested?":{"type":"noul","noul":0.2}}`,
want: "no 80%\n",
},
{
name: "JSON state",
args: []string{"--ask", "Is a refund being requested?", `{"ticket": "I was charged twice"}`},
state: `{"ticket":"I was charged twice"}`,
questions: refund,
answers: `{"Is a refund being requested?":{"type":"noul","noul":0.97}}`,
want: "yes 97%\n",
},
{
name: "labels",
stdin: "I was charged twice",
args: []string{"--ask", "Which team should handle this?", "--choice", "billing=Payments and refunds", "--choice", "technical"},
questions: `{"Which team should handle this?":{"type":"choice","instructions":"Which team should handle this?","criteria":{"billing":"Payments and refunds","technical":null}}}`,
answers: `{"Which team should handle this?":{"type":"choice","choice":"billing","probabilities":{"billing":0.91,"technical":0.09},"confidence":0.6}}`,
want: "billing 91%\n",
},
{
name: "levels",
stdin: "I was charged twice",
args: []string{"--ask", "How urgent is this?", "--level", "Routine", "--level", "Soon", "--level", "Immediate"},
questions: `{"How urgent is this?":{"type":"score","instructions":"How urgent is this?","criteria":["Routine","Soon","Immediate"]}}`,
answers: `{"How urgent is this?":{"type":"score","score":1.83,"legend":{"0":"Routine","1":"Soon","2":"Immediate"},"probabilities":{"0":0.04,"1":0.09,"2":0.87},"confidence":0.7}}`,
want: "Immediate 87% 1.8 / 2\n",
},
{
name: "most likely level, not the nearest",
stdin: "I was charged twice",
args: []string{"--ask", "How urgent is this?", "--level", "Routine", "--level", "Soon", "--level", "Immediate"},
questions: `{"How urgent is this?":{"type":"score","instructions":"How urgent is this?","criteria":["Routine","Soon","Immediate"]}}`,
answers: `{"How urgent is this?":{"type":"score","score":1.1,"legend":{"0":"Routine","1":"Soon","2":"Immediate"},"probabilities":{"0":0.45,"1":0,"2":0.55}}}`,
want: "Immediate 55% 1.1 / 2\n",
},
{
name: "questions in order",
stdin: "I was charged twice",
args: []string{
"--ask", "Is a refund being requested?",
"--ask", "Which team?", "--choice", "billing", "--choice", "technical",
"--ask", "How urgent?", "--level", "routine", "--level", "soon", "--level", "immediate",
},
questions: `{"Is a refund being requested?":{"type":"noul","instructions":"Is a refund being requested?"},"Which team?":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}},"How urgent?":{"type":"score","instructions":"How urgent?","criteria":["routine","soon","immediate"]}}`,
answers: `{"How urgent?":{"type":"score","score":1.2,"legend":{"0":"routine","1":"soon","2":"immediate"},"probabilities":{"0":0.1,"1":0.6,"2":0.3}},"Which team?":{"type":"choice","choice":"billing","probabilities":{"billing":0.91,"technical":0.09}},"Is a refund being requested?":{"type":"noul","noul":0.97}}`,
want: "Is a refund being requested? yes 97%\nWhich team? billing 91%\nHow urgent? soon 60% 1.2 / 2\n",
},
} {
t.Run(tt.name, func(t *testing.T) {
var got struct {
Model string `json:"model"`
Runner string `json:"runner"`
State json.RawMessage `json:"state"`
Questions json.RawMessage `json:"questions"`
}
out, err := runDecide(t, tt.stdin, func(w http.ResponseWriter, r *http.Request) {
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
t.Error(err)
}
_, _ = io.WriteString(w, `{"model":"nimble","answers":`+tt.answers+`}`)
}, append([]string{"nimble", "--runner", "llamacpp"}, tt.args...)...)
if err != nil {
t.Fatal(err)
}
if tt.state == "" {
tt.state = `"I was charged twice"`
}
if got.Model != "nimble" || got.Runner != "llamacpp" || string(got.State) != tt.state || string(got.Questions) != tt.questions {
t.Fatalf("unexpected request: %+v", got)
}
if out != tt.want {
t.Fatalf("got output %q, want %q", out, tt.want)
}
})
}
const response = `{"model":"nimble","answers":{"Is this urgent?":{"type":"noul","noul":0.9}},"usage":{"input_tokens":12,"output_tokens":1}}`
out, err := runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, response)
}, "nimble", "hello", "--ask", "Is this urgent?", "--format", "json")
if err != nil || out != response+"\n" {
t.Fatalf("json output = %q, %v", out, err)
}
}
func TestRunDecisionModelErrors(t *testing.T) {
const tooLong = "text and schema must not exceed 64 KiB"
_, err := runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusRequestEntityTooLarge)
_, _ = io.WriteString(w, `{"error":"`+tooLong+`"}`)
}, "nimble", "hello", "--ask", "Is this urgent?")
var status api.StatusError
if !errors.As(err, &status) || status.StatusCode != http.StatusRequestEntityTooLarge || status.ErrorMessage != tooLong {
t.Fatalf("lost server error: %v", err)
}
_, err = runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, `{"model":"nimble","answers":{}}`)
}, "nimble", "hello", "--ask", "Is this urgent?")
if err == nil || !strings.Contains(err.Error(), `no answer to "Is this urgent?"`) {
t.Fatalf("missing answer: %v", err)
}
unexpected := func(w http.ResponseWriter, r *http.Request) { t.Error("asked without text and a question") }
for _, tt := range []struct {
args []string
err string
}{
{[]string{"nimble"}, "answers questions about text"},
{[]string{"nimble", "hello"}, "answers questions about text"},
{[]string{"nimble", "--ask", "Is this urgent?"}, "answers questions about text"},
{[]string{"nimble", "hello", "--choice", "billing", "--ask", "Which team?"}, "must come after the --ask it answers"},
{[]string{"nimble", "hello", "--ask", "How urgent?", "--choice", "billing", "--level", "low"}, "--choice or --level, not both"},
{[]string{"gemma4", "Win a free cruise", "--ask", "Is this spam?"}, "gemma4 is not a decision model"},
} {
if _, err := runDecide(t, "", unexpected, tt.args...); err == nil || !strings.Contains(err.Error(), tt.err) {
t.Errorf("args %q: got %v", tt.args, err)
}
}
}

View file

@ -118,7 +118,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
scanner, err := readline.New(readline.Prompt{
Prompt: ">>> ",
AltPrompt: "... ",
Placeholder: "Send a message (/? for help)",
AltPlaceholder: "Press Enter to send",
})
if err != nil {
@ -135,8 +134,20 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
var sb strings.Builder
var multiline MultilineState
var thinkExplicitlySet bool = opts.Think != nil
// Decision models retain the first message and answer questions about it.
var decisionState *api.Message
for {
switch {
case !opts.Decision:
scanner.Prompt.Placeholder = "Send a message (/? for help)"
case opts.Questions.Len() > 0:
scanner.Prompt.Placeholder = "Enter text to evaluate (/? for help)"
case decisionState != nil:
scanner.Prompt.Placeholder = "Ask yes/no questions about the text, one per line (/? for help)"
default:
scanner.Prompt.Placeholder = "Enter text to evaluate, then ask yes/no questions (/? for help)"
}
line, err := scanner.Readline()
switch {
case errors.Is(err, io.EOF):
@ -234,6 +245,12 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
return err
}
applyShowResponseToRunOptions(&opts, info)
opts.Decision = slices.Contains(info.Capabilities, model.CapabilityDecision)
decisionState = nil
if opts.Decision {
// Decision models can't generate; the first answer loads the model.
continue
}
opts.Think, err = inferThinkingOption(&info.Capabilities, &opts, thinkExplicitlySet)
if err != nil {
return err
@ -277,6 +294,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
fmt.Printf("Created new model '%s'\n", args[1])
continue
case strings.HasPrefix(line, "/clear"):
decisionState = nil
opts.Messages = []api.Message{}
if opts.System != "" {
newMessage := api.Message{Role: "system", Content: opts.System}
@ -513,18 +531,46 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
}
if sb.Len() > 0 && multiline == MultilineNone {
newMessage := api.Message{Role: "user", Content: sb.String()}
if opts.MultiModal {
msg, images, err := extractFileData(sb.String())
if err != nil {
return err
input := sb.String()
sb.Reset()
if opts.Decision && decisionState != nil {
// Each line is a question, so pasting several asks them together.
var questions []string
for _, line := range strings.Split(input, "\n") {
if line = strings.TrimSpace(line); line != "" {
questions = append(questions, line)
}
}
newMessage.Content = msg
newMessage.Images = images
if len(questions) > 0 {
if err := decide(cmd, opts, *decisionState, yesNoQuestions(questions...)); err != nil {
fmt.Printf("error: %v\n", err)
}
}
continue
}
newMessage := api.Message{Role: "user", Content: input}
if opts.MultiModal {
if newMessage.Content, newMessage.Images, err = extractFileData(input); err != nil {
return err
}
}
if opts.Decision {
newMessage.Content = strings.TrimSpace(newMessage.Content)
switch {
case newMessage.Content == "" && len(newMessage.Images) == 0:
case opts.Questions.Len() > 0:
// With --ask, every message is a state to answer the questions about.
if err := decide(cmd, opts, newMessage, opts.Questions); err != nil {
fmt.Printf("error: %v\n", err)
}
default:
decisionState = &newMessage
}
continue
}
opts.Messages = append(opts.Messages, newMessage)
assistant, err := chat(cmd, opts)
@ -532,7 +578,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
if strings.Contains(err.Error(), "does not support thinking") ||
strings.Contains(err.Error(), "invalid think value") {
fmt.Printf("error: %v\n", err)
sb.Reset()
continue
}
return err
@ -540,8 +585,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
if assistant != nil {
opts.Messages = append(opts.Messages, *assistant)
}
sb.Reset()
}
}
}

View file

@ -1,30 +1,18 @@
package decision
import (
"encoding/json"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/internal/orderedmap"
)
// Questions preserves request order, which determines the order of scored fields.
// Its zero value is ready to use with Set.
type Questions = orderedmap.Map[string, Question]
type Questions = api.SystemOneQuestions
type Request struct {
Model string `json:"model"`
State json.RawMessage `json:"state"`
Images []api.ImageData `json:"images,omitempty"`
Videos []json.RawMessage `json:"videos,omitempty"`
Questions *Questions `json:"questions"`
KeepAlive *api.Duration `json:"keep_alive,omitempty"`
}
type Question struct {
Type string `json:"type"`
Instructions json.RawMessage `json:"instructions"`
Criteria json.RawMessage `json:"criteria"`
}
type (
Request = api.SystemOneRequest
Question = api.SystemOneQuestion
)
// Choice maps a single-token answer code to a value in the question's schema.
type Choice struct {
@ -53,10 +41,7 @@ type Response struct {
PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`
}
type Usage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
}
type Usage = api.SystemOneUsage
type NoulAnswer struct {
Type string `json:"type"`

View file

@ -23,6 +23,55 @@ Nimble, Tev1, [Clef](https://ollama.com/library/clef), and [Clef Flash](https://
Local requests do not need an API key.
## Ask from the terminal
`ollama run` answers questions about a state: text or JSON, piped in or given after the model name. Ask each question with `--ask`. It is answered yes or no, unless you follow it with labels to choose from (`--choice`) or levels to score it on (`--level`).
```shell examples.sh
# Ask a yes/no question
ollama run nimble --ask "Is a refund being requested?" < ticket.txt
# Choose a label
ollama run nimble --ask "Which team should handle this?" \
--choice billing="Payments and refunds" \
--choice technical="Software problems" < ticket.txt
# Score on a scale, from the lowest level to the highest
ollama run nimble --ask "How urgent is this?" \
--level "Routine" \
--level "Soon" \
--level "Immediate" < ticket.txt
# Ask several questions at once
ollama run nimble --ask "Is a refund being requested?" \
--ask "Which team should handle this?" \
--choice billing="Payments and refunds" \
--choice technical="Software problems" \
--ask "How urgent is this?" \
--level "Routine" \
--level "Soon" \
--level "Immediate" < ticket.txt
# Use a JSON file as the state, and refer to its fields in backticks
ollama run nimble --ask 'Is a refund being requested in `body`?' < ticket.json
# Judge an image with Clef
ollama run clef ./receipt.png --ask "Is this a restaurant receipt?"
# Print the full response
ollama run nimble --ask "Is a refund being requested?" --format json < ticket.txt
```
Each answer prints on its own line. A yes/no answer shows the likelier answer and its probability, a label shows its probability, and a score shows the most likely level, its probability, and the score on the scale:
```text
Is a refund being requested? yes 97%
Which team should handle this? billing 91%
How urgent is this? Soon 64% 1.2 / 2
```
Without a state, `ollama run nimble` opens an interactive prompt. Your first message is the state, and after that each line is a yes/no question about it, so you can paste several at once; `/clear` starts over. With `--ask`, each message is a new state to answer those questions about.
## Choose a label
Put your input in `state` and describe the possible answers in `criteria`:

View file

@ -79,6 +79,23 @@ Output is a JSON array:
echo "Hello world" | ollama run nomic-embed-text
```
### Ask a decision model
```
ollama run nimble --ask "Is a refund being requested?" < ticket.txt
ollama run nimble --ask "Which team should handle this?" \
--choice billing="Payments and refunds" \
--choice technical="Software problems" < ticket.txt
ollama run nimble --ask "How urgent is this?" \
--level "Routine" \
--level "Soon" \
--level "Immediate" < ticket.txt
```
The state is piped in or given after the model name, as text or JSON. Each `--ask` question is answered yes or no, unless it's followed by labels to choose from (`--choice`) or levels to score it on (`--level`). See [Decision](/capabilities/decision#ask-from-the-terminal).
### Download a model
```

View file

@ -990,7 +990,13 @@ func (s *Server) SystemOneHandler(c *gin.Context) {
c.JSON(http.StatusBadRequest, gin.H{"error": "System One requires a local decision model"})
return
}
m, err := s.getExistingModelForRunner(ref.Name.String(), "")
if runner, err := normalizeRunner(req.Runner); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
} else {
req.Runner = runner
}
m, err := s.getExistingModelForRunner(ref.Name.String(), req.Runner)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})

View file

@ -106,6 +106,59 @@ func TestDecisionModelRejectsCompletion(t *testing.T) {
}
}
func TestSystemOneHandlerRunnerSelection(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("OLLAMA_MODELS", t.TempDir())
for _, variant := range []struct{ runner, format string }{
{manifest.RunnerLlamaCPP, manifest.FormatGGUF},
{manifest.RunnerMLX, manifest.FormatSafetensors},
} {
writeShowManifestVariant(t, "decision-"+variant.runner, variant.runner, variant.format, model.ConfigV2{
ModelFormat: variant.format, Renderer: "clef", Capabilities: []string{"decision"},
}, map[string]any{"general.architecture": "clef", "clef.decision.type": "clef"})
}
writeManifestListFixture(t, "decision-list",
manifestListFixtureChild{name: "decision-llamacpp", runner: manifest.RunnerLlamaCPP, format: manifest.FormatGGUF},
manifestListFixtureChild{name: "decision-mlx", runner: manifest.RunnerMLX, format: manifest.FormatSafetensors},
)
defaultModel, err := GetModel("decision-list")
if err != nil {
t.Fatal(err)
}
for _, tt := range []struct{ runner, want string }{
{"", defaultModel.Runner},
{"llamacpp", manifest.RunnerLlamaCPP},
{"mlx", manifest.RunnerMLX},
{" llama.cpp ", manifest.RunnerLlamaCPP},
} {
t.Run(tt.runner, func(t *testing.T) {
runner := &systemOneTestRunner{}
s := newServerWithMockRunner(t, &runner.mockRunner)
selectedRunner := make(chan string, 1)
s.sched.loadFn = func(req *LlmRequest, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {
selectedRunner <- req.model.Runner
req.successCh <- &runnerRef{llama: runner}
return false
}
w := createRequest(t, s.SystemOneHandler, map[string]any{
"model": "decision-list", "runner": tt.runner, "state": "refund please",
"questions": map[string]any{"refund": map[string]any{"type": "noul", "instructions": "Refund requested?"}},
})
if w.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", w.Code, w.Body)
}
select {
case got := <-selectedRunner:
if got != tt.want {
t.Fatalf("scheduled runner=%q, want %q", got, tt.want)
}
default:
t.Fatal("no runner was scheduled")
}
})
}
}
func TestSystemOneHandler(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("OLLAMA_MODELS", t.TempDir())
@ -190,6 +243,7 @@ func TestSystemOneHandler(t *testing.T) {
}
tests := []testCase{
{"success", `{"model":"decision-test","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
{"unknown runner", `{"model":"decision-test","runner":"invalid","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
{"Clef image above text body limit", `{"model":"renamed-clef","state":"x","images":["` + base64.StdEncoding.EncodeToString(make([]byte, 70<<10)) + `"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},
{"over image body limit", `{"model":"renamed-clef","images":["` + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false},
{"images require a supported encoding", `{"model":"decision-test","state":"x","images":["aW1hZ2U="],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},