From 7a5ae12ead68342df402177cfabf4e7fc95aaca5 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Tue, 13 Oct 2026 12:39:05 -0700 Subject: [PATCH] cli: add decision model support to `ollama run` (#18962) --- api/client.go | 9 ++ api/types.go | 42 ++++++ cmd/cmd.go | 70 +++++----- cmd/cmd_test.go | 38 ++++++ cmd/decide.go | 227 +++++++++++++++++++++++++++++++++ cmd/decide_test.go | 204 +++++++++++++++++++++++++++++ cmd/interactive.go | 69 ++++++++-- decision/types.go | 27 +--- docs/capabilities/decision.mdx | 49 +++++++ docs/cli.mdx | 17 +++ server/routes.go | 8 +- server/systemone_test.go | 54 ++++++++ 12 files changed, 749 insertions(+), 65 deletions(-) create mode 100644 cmd/decide.go create mode 100644 cmd/decide_test.go diff --git a/api/client.go b/api/client.go index 7decf3d0f..2eb66cb9b 100644 --- a/api/client.go +++ b/api/client.go @@ -443,6 +443,15 @@ func (c *Client) Heartbeat(ctx context.Context) error { return nil } +// SystemOne answers typed questions about a state with a decision model. +func (c *Client) SystemOne(ctx context.Context, req *SystemOneRequest) (*SystemOneResponse, error) { + var resp SystemOneResponse + if err := c.do(ctx, http.MethodPost, "/v1/systemone", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + // Embed generates embeddings from a model. func (c *Client) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) { var resp EmbedResponse diff --git a/api/types.go b/api/types.go index e19de536f..66b6e15fa 100644 --- a/api/types.go +++ b/api/types.go @@ -601,6 +601,48 @@ type Runner struct { DraftNumPredict int `json:"draft_num_predict,omitempty"` } +// SystemOneRequest is the request passed to [Client.SystemOne]. It asks a +// decision model typed questions about a state. +type SystemOneRequest struct { + Model string `json:"model"` + Runner string `json:"runner,omitempty"` + State json.RawMessage `json:"state"` + Images []ImageData `json:"images,omitempty"` + Videos []json.RawMessage `json:"videos,omitempty"` + Questions *SystemOneQuestions `json:"questions"` + KeepAlive *Duration `json:"keep_alive,omitempty"` +} + +// SystemOneQuestions keeps questions in request order, which is the order of +// the answers. +type SystemOneQuestions = orderedmap.Map[string, SystemOneQuestion] + +// SystemOneQuestion is a noul (yes/no), choice, or score question. Criteria is +// an object for noul and choice questions and an ordered array for score. +type SystemOneQuestion struct { + Type string `json:"type"` + Instructions json.RawMessage `json:"instructions"` + Criteria json.RawMessage `json:"criteria,omitempty"` +} + +// SystemOneResponse is the response returned by [Client.SystemOne]. Each +// answer's type field says which answer fields it has. +type SystemOneResponse struct { + Model string `json:"model"` + Answers *SystemOneAnswers `json:"answers"` + Usage SystemOneUsage `json:"usage"` + PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"` +} + +// SystemOneAnswers keeps answers in request order. +type SystemOneAnswers = orderedmap.Map[string, json.RawMessage] + +// SystemOneUsage counts the tokens evaluated and generated to answer. +type SystemOneUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + // EmbedRequest is the request passed to [Client.Embed]. type EmbedRequest struct { // Model is the model name. diff --git a/cmd/cmd.go b/cmd/cmd.go index 9cd618613..8660c7f71 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -972,24 +972,15 @@ func RunHandler(cmd *cobra.Command, args []string) error { return err } - audioCapable := slices.Contains(info.Capabilities, model.CapabilityAudio) - opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || audioCapable - - // TODO: remove the projector info and vision info checks below, - // these are left in for backwards compatibility with older servers - // that don't have the capabilities field in the model info - if len(info.ProjectorInfo) != 0 { - opts.MultiModal = true - } - for k := range info.ModelInfo { - if strings.Contains(k, ".vision.") { - opts.MultiModal = true - break - } - } - applyShowResponseToRunOptions(&opts, info) + if slices.Contains(info.Capabilities, model.CapabilityDecision) { + return runDecision(cmd, opts, interactive) + } + if cmd.Flags().Changed("ask") { + return fmt.Errorf("%s is not a decision model, so it can't answer --ask questions", name) + } + // Check if this is an embedding model isEmbeddingModel := slices.Contains(info.Capabilities, model.CapabilityEmbedding) @@ -1824,6 +1815,8 @@ type runOptions struct { Think *api.ThinkValue HideThinking bool ShowConnect bool + Decision bool + Questions *api.SystemOneQuestions } func (r runOptions) Copy() runOptions { @@ -1876,12 +1869,27 @@ func (r runOptions) Copy() runOptions { Think: think, HideThinking: r.HideThinking, ShowConnect: r.ShowConnect, + Decision: r.Decision, + Questions: r.Questions, } } func applyShowResponseToRunOptions(opts *runOptions, info *api.ShowResponse) { opts.ParentModel = info.Details.ParentModel opts.LoadedMessages = slices.Clone(info.Messages) + opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || + slices.Contains(info.Capabilities, model.CapabilityAudio) + + // Keep media input working with older servers that do not report capabilities. + if len(info.ProjectorInfo) != 0 { + opts.MultiModal = true + } + for key := range info.ModelInfo { + if strings.Contains(key, ".vision.") { + opts.MultiModal = true + break + } + } } type displayResponseState struct { @@ -2363,18 +2371,6 @@ func launchInteractiveModel(cmd *cobra.Command, modelName string) error { return err } - audioCapable := slices.Contains(info.Capabilities, model.CapabilityAudio) - opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || audioCapable - if len(info.ProjectorInfo) != 0 { - opts.MultiModal = true - } - for key := range info.ModelInfo { - if strings.Contains(key, ".vision.") { - opts.MultiModal = true - break - } - } - applyShowResponseToRunOptions(&opts, info) if err := loadOrUnloadModel(cmd, &opts); err != nil { @@ -2571,8 +2567,18 @@ func NewCLI() *cobra.Command { showCmd.Flags().BoolP("verbose", "v", false, "Show detailed model information") runCmd := &cobra.Command{ - Use: "run MODEL [PROMPT]", - Short: "Run a model", + Use: "run MODEL [PROMPT]", + Short: "Run a model", + Example: ` ollama run gemma4 + ollama run gemma4 "Why is the sky blue?" + ollama run nimble --ask "Is a refund being requested?" < ticket.txt + ollama run nimble --ask "Which team should handle this?" \ + --choice billing="Payments and refunds" \ + --choice technical="Software problems" < ticket.txt + ollama run nimble --ask "How urgent is this?" \ + --level "Routine" \ + --level "Soon" \ + --level "Immediate" < ticket.txt`, Args: cobra.MinimumNArgs(1), PreRunE: checkServerHeartbeat, RunE: RunHandler, @@ -2590,6 +2596,10 @@ func NewCLI() *cobra.Command { runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)") runCmd.Flags().Bool("truncate", false, "For embedding models: truncate inputs exceeding context length (default: true). Set --truncate=false to error instead") runCmd.Flags().Int("dimensions", 0, "Truncate output embeddings to specified dimension (embedding models only)") + var asked []askedQuestion + runCmd.Flags().Var(questionFlag{"ask", &asked}, "ask", "Ask a decision model a question, answered yes or no unless --choice or --level follow") + runCmd.Flags().Var(questionFlag{"choice", &asked}, "choice", `Add a label for the preceding --ask to choose from, e.g. billing="Payments and refunds"`) + runCmd.Flags().Var(questionFlag{"level", &asked}, "level", "Add a level to score the preceding --ask on, from lowest to highest") stopCmd := &cobra.Command{ Use: "stop MODEL", diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 463ea1f1f..433e7c303 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -2497,6 +2497,44 @@ func TestApplyShowResponseToRunOptions(t *testing.T) { } } +func TestApplyShowResponseRefreshesMultimodal(t *testing.T) { + for _, tt := range []struct { + name string + info api.ShowResponse + }{ + { + name: "decision vision", + info: api.ShowResponse{Capabilities: []model.Capability{model.CapabilityDecision, model.CapabilityVision}}, + }, + { + name: "audio", + info: api.ShowResponse{Capabilities: []model.Capability{model.CapabilityAudio}}, + }, + { + name: "legacy projector", + info: api.ShowResponse{ProjectorInfo: map[string]any{"general.architecture": "clip"}}, + }, + { + name: "legacy vision", + info: api.ShowResponse{ModelInfo: map[string]any{"mllama.vision.block_count": 32}}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + opts := runOptions{} + applyShowResponseToRunOptions(&opts, &tt.info) + if !opts.MultiModal { + t.Fatal("loading a multimodal model did not enable media input") + } + applyShowResponseToRunOptions(&opts, &api.ShowResponse{ + Capabilities: []model.Capability{model.CapabilityDecision}, + }) + if opts.MultiModal { + t.Fatal("loading a text-only model did not disable media input") + } + }) + } +} + func TestRunOptions_Copy(t *testing.T) { // Setup test data originalKeepAlive := &api.Duration{Duration: 5 * time.Minute} diff --git a/cmd/decide.go b/cmd/decide.go new file mode 100644 index 000000000..37ba126d2 --- /dev/null +++ b/cmd/decide.go @@ -0,0 +1,227 @@ +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/signal" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/internal/orderedmap" + "github.com/ollama/ollama/progress" +) + +// runDecision runs a decision model, which answers the --ask questions about +// a state: the text after the model name, or stdin. +func runDecision(cmd *cobra.Command, opts runOptions, interactive bool) error { + opts.Questions = askedQuestions(cmd) + if interactive { + opts.Decision = true + return generateInteractive(cmd, opts) + } + if opts.Prompt == "" || opts.Questions.Len() == 0 { + return fmt.Errorf("%s answers questions about text, e.g. ollama run %s --ask \"Is a refund being requested?\" < ticket.txt", opts.Model, opts.Model) + } + + input := api.Message{Role: "user", Content: opts.Prompt} + if opts.MultiModal { + var err error + if input.Content, input.Images, err = extractFileData(input.Content); err != nil { + return err + } + } + return decide(cmd, opts, input, opts.Questions) +} + +// yesNoQuestions asks each text as a yes/no question keyed by the text itself. +func yesNoQuestions(texts ...string) *api.SystemOneQuestions { + questions := &api.SystemOneQuestions{} + for _, text := range texts { + instructions, _ := json.Marshal(text) + questions.Set(text, api.SystemOneQuestion{Type: "noul", Instructions: instructions}) + } + return questions +} + +// questionFlag is --ask, --choice or --level. The three share the questions +// being asked, so each --choice or --level answers the --ask before it. +type questionFlag struct { + name string + asked *[]askedQuestion +} + +type askedQuestion struct { + text string + choices *orderedmap.Map[string, any] + levels []string +} + +func (f questionFlag) String() string { return "" } +func (f questionFlag) Type() string { return "string" } + +func (f questionFlag) Set(value string) error { + if f.name == "ask" { + *f.asked = append(*f.asked, askedQuestion{text: value}) + return nil + } + if len(*f.asked) == 0 { + return errors.New("must come after the --ask it answers") + } + q := &(*f.asked)[len(*f.asked)-1] + if f.name == "level" { + q.levels = append(q.levels, value) + } else { + if q.choices == nil { + q.choices = orderedmap.New[string, any]() + } + // A label may have a description: billing="Payments and refunds" + if label, description, ok := strings.Cut(value, "="); ok { + q.choices.Set(label, description) + } else { + q.choices.Set(value, nil) + } + } + if q.choices != nil && q.levels != nil { + return errors.New("a question takes --choice or --level, not both") + } + return nil +} + +// askedQuestions returns the --ask questions keyed by their text: yes/no, or +// answered with a --choice label or on a --level scale. +func askedQuestions(cmd *cobra.Command) *api.SystemOneQuestions { + questions := &api.SystemOneQuestions{} + flag := cmd.Flags().Lookup("ask") + if flag == nil { + return questions + } + for _, q := range *flag.Value.(questionFlag).asked { + question := api.SystemOneQuestion{Type: "noul"} + question.Instructions, _ = json.Marshal(q.text) + switch { + case q.choices != nil: + question.Type = "choice" + question.Criteria, _ = json.Marshal(q.choices) + case q.levels != nil: + question.Type = "score" + question.Criteria, _ = json.Marshal(q.levels) + } + questions.Set(q.text, question) + } + return questions +} + +// decide asks a decision model questions about a state and prints each answer: +// the likelier answer to a yes/no question, the chosen label, or the most +// likely score level, with its probability, and for a score the score itself. +func decide(cmd *cobra.Command, opts runOptions, input api.Message, questions *api.SystemOneQuestions) error { + client, err := api.ClientFromEnvironment() + if err != nil { + return err + } + // Piped text and multiline messages end in a newline that isn't part of the state. + text := strings.TrimRight(input.Content, "\r\n") + state, _ := json.Marshal(text) + if s := strings.TrimSpace(text); strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[") { + if json.Valid([]byte(s)) { + state = json.RawMessage(s) + } + } + req := &api.SystemOneRequest{ + Model: opts.Model, + Runner: opts.Runner, + State: state, + Images: input.Images, + Questions: questions, + KeepAlive: opts.KeepAlive, + } + + ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt) + defer stop() + + p := progress.NewProgress(os.Stderr) + p.Add("", progress.NewSpinner("")) + start := time.Now() + resp, err := client.SystemOne(ctx, req) + p.StopAndClear() + if errors.Is(err, context.Canceled) { + return nil + } else if err != nil { + return err + } + + if opts.Format == "json" { + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + return enc.Encode(resp) + } + + tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for name := range questions.All() { + raw, ok := resp.Answers.Get(name) + if !ok { + return fmt.Errorf("no answer to %q", name) + } + var a struct { + Type string `json:"type"` + Noul float64 `json:"noul"` + Choice string `json:"choice"` + Score float64 `json:"score"` + Legend map[string]any `json:"legend"` + Probabilities map[string]float64 `json:"probabilities"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return fmt.Errorf("answer to %q: %w", name, err) + } + var answer, measure string + switch a.Type { + case "noul": + answer, measure = "yes", fmt.Sprintf("%.0f%%", a.Noul*100) + if a.Noul < 0.5 { + answer, measure = "no", fmt.Sprintf("%.0f%%", (1-a.Noul)*100) + } + case "choice": + answer, measure = a.Choice, fmt.Sprintf("%.0f%%", a.Probabilities[a.Choice]*100) + case "score": + // Not the level nearest the score: an answer split between the + // ends of the scale scores in the middle. + level := "0" + for i := 1; i < len(a.Legend); i++ { + if k := strconv.Itoa(i); a.Probabilities[k] > a.Probabilities[level] { + level = k + } + } + if v, ok := a.Legend[level]; ok { + answer = fmt.Sprint(v) + } + measure = fmt.Sprintf("%.0f%%\t%.1f / %d", a.Probabilities[level]*100, a.Score, len(a.Legend)-1) + default: + return fmt.Errorf("answer to %q has unknown type %q", name, a.Type) + } + if questions.Len() > 1 { + fmt.Fprintf(tw, "%s\t", name) + } + fmt.Fprintf(tw, "%s\t%s\n", answer, measure) + } + if err := tw.Flush(); err != nil { + return err + } + + if verbose, _ := cmd.Flags().GetBool("verbose"); verbose { + metrics := api.Metrics{ + TotalDuration: time.Since(start), + PromptEvalCount: resp.Usage.InputTokens, + PromptEvalCachedCount: resp.PromptEvalCachedCount, + } + metrics.Summary() + } + return nil +} diff --git a/cmd/decide_test.go b/cmd/decide_test.go new file mode 100644 index 000000000..938a142e7 --- /dev/null +++ b/cmd/decide_test.go @@ -0,0 +1,204 @@ +package cmd + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/types/model" +) + +// runDecide runs `ollama run args...` with stdin piped in. nimble is a +// decision model and any other model is a chat model. +func runDecide(t *testing.T, stdin string, handler http.HandlerFunc, args ...string) (string, error) { + t.Helper() + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/show" && r.Method == http.MethodPost { + var req api.ShowRequest + _ = json.NewDecoder(r.Body).Decode(&req) + capability := model.CapabilityCompletion + if req.Model == "nimble" { + capability = model.CapabilityDecision + } + _ = json.NewEncoder(w).Encode(api.ShowResponse{Capabilities: []model.Capability{capability}}) + return + } + if r.URL.Path == "/v1/systemone" && r.Method == http.MethodPost { + handler(w, r) + return + } + http.NotFound(w, r) + })) + t.Setenv("OLLAMA_HOST", mockServer.URL) + t.Cleanup(mockServer.Close) + + cmd, _, err := NewCLI().Find([]string{"run"}) + if err != nil { + t.Fatal(err) + } + cmd.SetContext(t.Context()) + if err := cmd.ParseFlags(args); err != nil { + return "", err + } + + oldStdin := os.Stdin + stdinR, stdinW, _ := os.Pipe() + defer stdinR.Close() + _, _ = io.WriteString(stdinW, stdin) + stdinW.Close() + os.Stdin = stdinR + defer func() { os.Stdin = oldStdin }() + out := captureStdout(t, func() { + err = RunHandler(cmd, cmd.Flags().Args()) + }) + return out, err +} + +func TestRunDecisionModel(t *testing.T) { + const refund = `{"Is a refund being requested?":{"type":"noul","instructions":"Is a refund being requested?"}}` + for _, tt := range []struct { + name, stdin string + args []string + state string + questions string + answers string + want string + }{ + { + name: "yes/no", + stdin: "I was charged twice\n", + args: []string{"--ask", "Is a refund being requested?"}, + questions: refund, + answers: `{"Is a refund being requested?":{"type":"noul","noul":0.97}}`, + want: "yes 97%\n", + }, + { + name: "state as an argument", + args: []string{"I was charged twice", "--ask", "Is a refund being requested?"}, + questions: refund, + answers: `{"Is a refund being requested?":{"type":"noul","noul":0.2}}`, + want: "no 80%\n", + }, + { + name: "JSON state", + args: []string{"--ask", "Is a refund being requested?", `{"ticket": "I was charged twice"}`}, + state: `{"ticket":"I was charged twice"}`, + questions: refund, + answers: `{"Is a refund being requested?":{"type":"noul","noul":0.97}}`, + want: "yes 97%\n", + }, + { + name: "labels", + stdin: "I was charged twice", + args: []string{"--ask", "Which team should handle this?", "--choice", "billing=Payments and refunds", "--choice", "technical"}, + questions: `{"Which team should handle this?":{"type":"choice","instructions":"Which team should handle this?","criteria":{"billing":"Payments and refunds","technical":null}}}`, + answers: `{"Which team should handle this?":{"type":"choice","choice":"billing","probabilities":{"billing":0.91,"technical":0.09},"confidence":0.6}}`, + want: "billing 91%\n", + }, + { + name: "levels", + stdin: "I was charged twice", + args: []string{"--ask", "How urgent is this?", "--level", "Routine", "--level", "Soon", "--level", "Immediate"}, + questions: `{"How urgent is this?":{"type":"score","instructions":"How urgent is this?","criteria":["Routine","Soon","Immediate"]}}`, + answers: `{"How urgent is this?":{"type":"score","score":1.83,"legend":{"0":"Routine","1":"Soon","2":"Immediate"},"probabilities":{"0":0.04,"1":0.09,"2":0.87},"confidence":0.7}}`, + want: "Immediate 87% 1.8 / 2\n", + }, + { + name: "most likely level, not the nearest", + stdin: "I was charged twice", + args: []string{"--ask", "How urgent is this?", "--level", "Routine", "--level", "Soon", "--level", "Immediate"}, + questions: `{"How urgent is this?":{"type":"score","instructions":"How urgent is this?","criteria":["Routine","Soon","Immediate"]}}`, + answers: `{"How urgent is this?":{"type":"score","score":1.1,"legend":{"0":"Routine","1":"Soon","2":"Immediate"},"probabilities":{"0":0.45,"1":0,"2":0.55}}}`, + want: "Immediate 55% 1.1 / 2\n", + }, + { + name: "questions in order", + stdin: "I was charged twice", + args: []string{ + "--ask", "Is a refund being requested?", + "--ask", "Which team?", "--choice", "billing", "--choice", "technical", + "--ask", "How urgent?", "--level", "routine", "--level", "soon", "--level", "immediate", + }, + questions: `{"Is a refund being requested?":{"type":"noul","instructions":"Is a refund being requested?"},"Which team?":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}},"How urgent?":{"type":"score","instructions":"How urgent?","criteria":["routine","soon","immediate"]}}`, + answers: `{"How urgent?":{"type":"score","score":1.2,"legend":{"0":"routine","1":"soon","2":"immediate"},"probabilities":{"0":0.1,"1":0.6,"2":0.3}},"Which team?":{"type":"choice","choice":"billing","probabilities":{"billing":0.91,"technical":0.09}},"Is a refund being requested?":{"type":"noul","noul":0.97}}`, + want: "Is a refund being requested? yes 97%\nWhich team? billing 91%\nHow urgent? soon 60% 1.2 / 2\n", + }, + } { + t.Run(tt.name, func(t *testing.T) { + var got struct { + Model string `json:"model"` + Runner string `json:"runner"` + State json.RawMessage `json:"state"` + Questions json.RawMessage `json:"questions"` + } + out, err := runDecide(t, tt.stdin, func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Error(err) + } + _, _ = io.WriteString(w, `{"model":"nimble","answers":`+tt.answers+`}`) + }, append([]string{"nimble", "--runner", "llamacpp"}, tt.args...)...) + if err != nil { + t.Fatal(err) + } + if tt.state == "" { + tt.state = `"I was charged twice"` + } + if got.Model != "nimble" || got.Runner != "llamacpp" || string(got.State) != tt.state || string(got.Questions) != tt.questions { + t.Fatalf("unexpected request: %+v", got) + } + if out != tt.want { + t.Fatalf("got output %q, want %q", out, tt.want) + } + }) + } + + const response = `{"model":"nimble","answers":{"Is this urgent?":{"type":"noul","noul":0.9}},"usage":{"input_tokens":12,"output_tokens":1}}` + out, err := runDecide(t, "", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, response) + }, "nimble", "hello", "--ask", "Is this urgent?", "--format", "json") + if err != nil || out != response+"\n" { + t.Fatalf("json output = %q, %v", out, err) + } +} + +func TestRunDecisionModelErrors(t *testing.T) { + const tooLong = "text and schema must not exceed 64 KiB" + _, err := runDecide(t, "", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusRequestEntityTooLarge) + _, _ = io.WriteString(w, `{"error":"`+tooLong+`"}`) + }, "nimble", "hello", "--ask", "Is this urgent?") + var status api.StatusError + if !errors.As(err, &status) || status.StatusCode != http.StatusRequestEntityTooLarge || status.ErrorMessage != tooLong { + t.Fatalf("lost server error: %v", err) + } + + _, err = runDecide(t, "", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, `{"model":"nimble","answers":{}}`) + }, "nimble", "hello", "--ask", "Is this urgent?") + if err == nil || !strings.Contains(err.Error(), `no answer to "Is this urgent?"`) { + t.Fatalf("missing answer: %v", err) + } + + unexpected := func(w http.ResponseWriter, r *http.Request) { t.Error("asked without text and a question") } + for _, tt := range []struct { + args []string + err string + }{ + {[]string{"nimble"}, "answers questions about text"}, + {[]string{"nimble", "hello"}, "answers questions about text"}, + {[]string{"nimble", "--ask", "Is this urgent?"}, "answers questions about text"}, + {[]string{"nimble", "hello", "--choice", "billing", "--ask", "Which team?"}, "must come after the --ask it answers"}, + {[]string{"nimble", "hello", "--ask", "How urgent?", "--choice", "billing", "--level", "low"}, "--choice or --level, not both"}, + {[]string{"gemma4", "Win a free cruise", "--ask", "Is this spam?"}, "gemma4 is not a decision model"}, + } { + if _, err := runDecide(t, "", unexpected, tt.args...); err == nil || !strings.Contains(err.Error(), tt.err) { + t.Errorf("args %q: got %v", tt.args, err) + } + } +} diff --git a/cmd/interactive.go b/cmd/interactive.go index ff7fffbb2..b6892fd25 100644 --- a/cmd/interactive.go +++ b/cmd/interactive.go @@ -118,7 +118,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { scanner, err := readline.New(readline.Prompt{ Prompt: ">>> ", AltPrompt: "... ", - Placeholder: "Send a message (/? for help)", AltPlaceholder: "Press Enter to send", }) if err != nil { @@ -135,8 +134,20 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { var sb strings.Builder var multiline MultilineState var thinkExplicitlySet bool = opts.Think != nil + // Decision models retain the first message and answer questions about it. + var decisionState *api.Message for { + switch { + case !opts.Decision: + scanner.Prompt.Placeholder = "Send a message (/? for help)" + case opts.Questions.Len() > 0: + scanner.Prompt.Placeholder = "Enter text to evaluate (/? for help)" + case decisionState != nil: + scanner.Prompt.Placeholder = "Ask yes/no questions about the text, one per line (/? for help)" + default: + scanner.Prompt.Placeholder = "Enter text to evaluate, then ask yes/no questions (/? for help)" + } line, err := scanner.Readline() switch { case errors.Is(err, io.EOF): @@ -234,6 +245,12 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { return err } applyShowResponseToRunOptions(&opts, info) + opts.Decision = slices.Contains(info.Capabilities, model.CapabilityDecision) + decisionState = nil + if opts.Decision { + // Decision models can't generate; the first answer loads the model. + continue + } opts.Think, err = inferThinkingOption(&info.Capabilities, &opts, thinkExplicitlySet) if err != nil { return err @@ -277,6 +294,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { fmt.Printf("Created new model '%s'\n", args[1]) continue case strings.HasPrefix(line, "/clear"): + decisionState = nil opts.Messages = []api.Message{} if opts.System != "" { newMessage := api.Message{Role: "system", Content: opts.System} @@ -513,18 +531,46 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { } if sb.Len() > 0 && multiline == MultilineNone { - newMessage := api.Message{Role: "user", Content: sb.String()} - - if opts.MultiModal { - msg, images, err := extractFileData(sb.String()) - if err != nil { - return err + input := sb.String() + sb.Reset() + if opts.Decision && decisionState != nil { + // Each line is a question, so pasting several asks them together. + var questions []string + for _, line := range strings.Split(input, "\n") { + if line = strings.TrimSpace(line); line != "" { + questions = append(questions, line) + } } - - newMessage.Content = msg - newMessage.Images = images + if len(questions) > 0 { + if err := decide(cmd, opts, *decisionState, yesNoQuestions(questions...)); err != nil { + fmt.Printf("error: %v\n", err) + } + } + continue } + newMessage := api.Message{Role: "user", Content: input} + + if opts.MultiModal { + if newMessage.Content, newMessage.Images, err = extractFileData(input); err != nil { + return err + } + } + + if opts.Decision { + newMessage.Content = strings.TrimSpace(newMessage.Content) + switch { + case newMessage.Content == "" && len(newMessage.Images) == 0: + case opts.Questions.Len() > 0: + // With --ask, every message is a state to answer the questions about. + if err := decide(cmd, opts, newMessage, opts.Questions); err != nil { + fmt.Printf("error: %v\n", err) + } + default: + decisionState = &newMessage + } + continue + } opts.Messages = append(opts.Messages, newMessage) assistant, err := chat(cmd, opts) @@ -532,7 +578,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { if strings.Contains(err.Error(), "does not support thinking") || strings.Contains(err.Error(), "invalid think value") { fmt.Printf("error: %v\n", err) - sb.Reset() continue } return err @@ -540,8 +585,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { if assistant != nil { opts.Messages = append(opts.Messages, *assistant) } - - sb.Reset() } } } diff --git a/decision/types.go b/decision/types.go index 758bddb5e..b2a63ebdc 100644 --- a/decision/types.go +++ b/decision/types.go @@ -1,30 +1,18 @@ package decision import ( - "encoding/json" - "github.com/ollama/ollama/api" "github.com/ollama/ollama/internal/orderedmap" ) // Questions preserves request order, which determines the order of scored fields. // Its zero value is ready to use with Set. -type Questions = orderedmap.Map[string, Question] +type Questions = api.SystemOneQuestions -type Request struct { - Model string `json:"model"` - State json.RawMessage `json:"state"` - Images []api.ImageData `json:"images,omitempty"` - Videos []json.RawMessage `json:"videos,omitempty"` - Questions *Questions `json:"questions"` - KeepAlive *api.Duration `json:"keep_alive,omitempty"` -} - -type Question struct { - Type string `json:"type"` - Instructions json.RawMessage `json:"instructions"` - Criteria json.RawMessage `json:"criteria"` -} +type ( + Request = api.SystemOneRequest + Question = api.SystemOneQuestion +) // Choice maps a single-token answer code to a value in the question's schema. type Choice struct { @@ -53,10 +41,7 @@ type Response struct { PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"` } -type Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} +type Usage = api.SystemOneUsage type NoulAnswer struct { Type string `json:"type"` diff --git a/docs/capabilities/decision.mdx b/docs/capabilities/decision.mdx index 8e23702f7..46ea98f47 100644 --- a/docs/capabilities/decision.mdx +++ b/docs/capabilities/decision.mdx @@ -23,6 +23,55 @@ Nimble, Tev1, [Clef](https://ollama.com/library/clef), and [Clef Flash](https:// Local requests do not need an API key. +## Ask from the terminal + +`ollama run` answers questions about a state: text or JSON, piped in or given after the model name. Ask each question with `--ask`. It is answered yes or no, unless you follow it with labels to choose from (`--choice`) or levels to score it on (`--level`). + +```shell examples.sh +# Ask a yes/no question +ollama run nimble --ask "Is a refund being requested?" < ticket.txt + +# Choose a label +ollama run nimble --ask "Which team should handle this?" \ + --choice billing="Payments and refunds" \ + --choice technical="Software problems" < ticket.txt + +# Score on a scale, from the lowest level to the highest +ollama run nimble --ask "How urgent is this?" \ + --level "Routine" \ + --level "Soon" \ + --level "Immediate" < ticket.txt + +# Ask several questions at once +ollama run nimble --ask "Is a refund being requested?" \ + --ask "Which team should handle this?" \ + --choice billing="Payments and refunds" \ + --choice technical="Software problems" \ + --ask "How urgent is this?" \ + --level "Routine" \ + --level "Soon" \ + --level "Immediate" < ticket.txt + +# Use a JSON file as the state, and refer to its fields in backticks +ollama run nimble --ask 'Is a refund being requested in `body`?' < ticket.json + +# Judge an image with Clef +ollama run clef ./receipt.png --ask "Is this a restaurant receipt?" + +# Print the full response +ollama run nimble --ask "Is a refund being requested?" --format json < ticket.txt +``` + +Each answer prints on its own line. A yes/no answer shows the likelier answer and its probability, a label shows its probability, and a score shows the most likely level, its probability, and the score on the scale: + +```text +Is a refund being requested? yes 97% +Which team should handle this? billing 91% +How urgent is this? Soon 64% 1.2 / 2 +``` + +Without a state, `ollama run nimble` opens an interactive prompt. Your first message is the state, and after that each line is a yes/no question about it, so you can paste several at once; `/clear` starts over. With `--ask`, each message is a new state to answer those questions about. + ## Choose a label Put your input in `state` and describe the possible answers in `criteria`: diff --git a/docs/cli.mdx b/docs/cli.mdx index 8ed110eed..ae4215544 100644 --- a/docs/cli.mdx +++ b/docs/cli.mdx @@ -79,6 +79,23 @@ Output is a JSON array: echo "Hello world" | ollama run nomic-embed-text ``` +### Ask a decision model + +``` +ollama run nimble --ask "Is a refund being requested?" < ticket.txt + +ollama run nimble --ask "Which team should handle this?" \ + --choice billing="Payments and refunds" \ + --choice technical="Software problems" < ticket.txt + +ollama run nimble --ask "How urgent is this?" \ + --level "Routine" \ + --level "Soon" \ + --level "Immediate" < ticket.txt +``` + +The state is piped in or given after the model name, as text or JSON. Each `--ask` question is answered yes or no, unless it's followed by labels to choose from (`--choice`) or levels to score it on (`--level`). See [Decision](/capabilities/decision#ask-from-the-terminal). + ### Download a model ``` diff --git a/server/routes.go b/server/routes.go index e14e036ab..200af3e03 100644 --- a/server/routes.go +++ b/server/routes.go @@ -990,7 +990,13 @@ func (s *Server) SystemOneHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "System One requires a local decision model"}) return } - m, err := s.getExistingModelForRunner(ref.Name.String(), "") + if runner, err := normalizeRunner(req.Runner); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } else { + req.Runner = runner + } + m, err := s.getExistingModelForRunner(ref.Name.String(), req.Runner) if err != nil { if errors.Is(err, os.ErrNotExist) { c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)}) diff --git a/server/systemone_test.go b/server/systemone_test.go index 50e0c4d89..ce14b7ac6 100644 --- a/server/systemone_test.go +++ b/server/systemone_test.go @@ -106,6 +106,59 @@ func TestDecisionModelRejectsCompletion(t *testing.T) { } } +func TestSystemOneHandlerRunnerSelection(t *testing.T) { + gin.SetMode(gin.TestMode) + t.Setenv("OLLAMA_MODELS", t.TempDir()) + for _, variant := range []struct{ runner, format string }{ + {manifest.RunnerLlamaCPP, manifest.FormatGGUF}, + {manifest.RunnerMLX, manifest.FormatSafetensors}, + } { + writeShowManifestVariant(t, "decision-"+variant.runner, variant.runner, variant.format, model.ConfigV2{ + ModelFormat: variant.format, Renderer: "clef", Capabilities: []string{"decision"}, + }, map[string]any{"general.architecture": "clef", "clef.decision.type": "clef"}) + } + writeManifestListFixture(t, "decision-list", + manifestListFixtureChild{name: "decision-llamacpp", runner: manifest.RunnerLlamaCPP, format: manifest.FormatGGUF}, + manifestListFixtureChild{name: "decision-mlx", runner: manifest.RunnerMLX, format: manifest.FormatSafetensors}, + ) + defaultModel, err := GetModel("decision-list") + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct{ runner, want string }{ + {"", defaultModel.Runner}, + {"llamacpp", manifest.RunnerLlamaCPP}, + {"mlx", manifest.RunnerMLX}, + {" llama.cpp ", manifest.RunnerLlamaCPP}, + } { + t.Run(tt.runner, func(t *testing.T) { + runner := &systemOneTestRunner{} + s := newServerWithMockRunner(t, &runner.mockRunner) + selectedRunner := make(chan string, 1) + s.sched.loadFn = func(req *LlmRequest, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool { + selectedRunner <- req.model.Runner + req.successCh <- &runnerRef{llama: runner} + return false + } + w := createRequest(t, s.SystemOneHandler, map[string]any{ + "model": "decision-list", "runner": tt.runner, "state": "refund please", + "questions": map[string]any{"refund": map[string]any{"type": "noul", "instructions": "Refund requested?"}}, + }) + if w.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", w.Code, w.Body) + } + select { + case got := <-selectedRunner: + if got != tt.want { + t.Fatalf("scheduled runner=%q, want %q", got, tt.want) + } + default: + t.Fatal("no runner was scheduled") + } + }) + } +} + func TestSystemOneHandler(t *testing.T) { gin.SetMode(gin.TestMode) t.Setenv("OLLAMA_MODELS", t.TempDir()) @@ -190,6 +243,7 @@ func TestSystemOneHandler(t *testing.T) { } tests := []testCase{ {"success", `{"model":"decision-test","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false}, + {"unknown runner", `{"model":"decision-test","runner":"invalid","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false}, {"Clef image above text body limit", `{"model":"renamed-clef","state":"x","images":["` + base64.StdEncoding.EncodeToString(make([]byte, 70<<10)) + `"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false}, {"over image body limit", `{"model":"renamed-clef","images":["` + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false}, {"images require a supported encoding", `{"model":"decision-test","state":"x","images":["aW1hZ2U="],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},