diff --git a/decision/clef.go b/decision/clef.go new file mode 100644 index 000000000..31d0b419a --- /dev/null +++ b/decision/clef.go @@ -0,0 +1,246 @@ +package decision + +import ( + "bytes" + "encoding/json" + "fmt" + "maps" + "math" + "slices" + "strconv" + "strings" + + "github.com/ollama/ollama/llm" +) + +// encodeClef preserves the reference encoder's JSON and segment boundaries, +// retaining its choice order for the shared answer decoder. +func encodeClef(req Request, c *Compiled) error { + state, err := clefContent(req.State) + if err != nil { + return fmt.Errorf("state: %w", err) + } + input := llm.ScoreRequest{Images: req.Images, ImagePosition: 1} + add := func(text string) [2]int { + n := len(input.Segments) + input.Segments = append(input.Segments, text) + return [2]int{n, n + 1} + } + add("<|im_start|>system\nRead the complete state and schema. Decide every field jointly. Each answer must be exactly one of that field's allowed options.<|im_end|>\n<|im_start|>user\nSTATE:\n") + add(state) + add("\n\nSCHEMA FIELDS:\n") + for name, q := range req.Questions.All() { + f, err := clefField(name, q) + if err != nil { + return fmt.Errorf("question %q: %w", name, err) + } + add(fmt.Sprintf("\nFIELD %d\nID: %s\nTYPE: %s\nINSTRUCTION: ", len(c.fields)+1, name, q.Type)) + field := llm.ScoreField{Question: add(f.Description)} + switch q.Type { + case "choice": + field.Type = 1 + case "score": + field.Type = 2 + } + add("\nALLOWED OPTIONS:\n") + for j, option := range f.Choices { + add(fmt.Sprintf("OPTION %d: ", j+1)) + semantics := map[string]any{"option_id": fmt.Sprint(option.Value)} + if option.Description != nil { + semantics["description"] = option.Description + } + raw, err := clefJSON(semantics) + if err != nil { + return err + } + field.Options = append(field.Options, add(raw)) + add("\n") + } + add("END FIELD\n") + input.Fields = append(input.Fields, field) + c.fields = append(c.fields, f) + } + add("\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJOINT SCHEMA DECISIONS:") + c.Request = input + return nil +} + +func clefField(name string, q Question) (compiledField, error) { + f := compiledField{Field: Field{Name: name}, typ: q.Type} + if strings.TrimSpace(name) == "" { + return f, fmt.Errorf("field name must not be empty") + } + f.Description = name + if raw := bytes.TrimSpace(q.Instructions); len(raw) > 0 && string(raw) != "null" && string(raw) != `""` { + var err error + f.Description, err = clefContent(raw) + if err != nil { + return f, fmt.Errorf("instructions: %w", err) + } + } + var criteria any + if len(q.Criteria) > 0 { + var err error + criteria, err = clefValue(q.Criteria) + if err != nil { + return f, fmt.Errorf("criteria: %w", err) + } + } + add := func(value, description any) { + f.Choices = append(f.Choices, Choice{Value: value, Description: description}) + } + switch q.Type { + case "noul": + descriptions := map[string]any{"true": "The proposition is true or the answer is yes.", "false": "The proposition is false or the answer is no."} + if criteria != nil { + values, ok := criteria.(map[string]any) + if !ok { + return f, fmt.Errorf("noul criteria must be an object") + } + for key, value := range values { + if key != "true" && key != "false" { + return f, fmt.Errorf("unknown noul criterion %q", key) + } + descriptions[key] = value + } + } + add(true, descriptions["true"]) + add(false, descriptions["false"]) + case "choice": + values, ok := criteria.(map[string]any) + if !ok { + return f, fmt.Errorf("choice criteria must be an object") + } + for _, key := range slices.Sorted(maps.Keys(values)) { + if strings.TrimSpace(key) == "" { + return f, fmt.Errorf("choice keys must not be empty") + } + add(key, values[key]) + } + case "score": + values, ok := criteria.([]any) + if !ok { + return f, fmt.Errorf("score criteria must be an array") + } + for i, value := range values { + add(strconv.Itoa(i), value) + } + default: + return f, fmt.Errorf("type must be choice, noul, or score") + } + if len(f.Choices) < 2 || len(f.Choices) > 26 { + return f, fmt.Errorf("criteria must contain 2–26 candidates") + } + return f, nil +} + +func clefValue(raw json.RawMessage) (any, error) { + if !json.Valid(raw) { + return nil, fmt.Errorf("must be a JSON value") + } + var value any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + err := dec.Decode(&value) + return value, err +} + +func clefContent(raw json.RawMessage) (string, error) { + value, err := clefValue(raw) + if err != nil { + return "", err + } + if s, ok := value.(string); ok { + return s, nil + } + return clefJSON(value) +} + +// clefJSON matches Python's json.dumps(ensure_ascii=False, sort_keys=True, +// separators=(",", ":")), including its integer/float distinction. +// encoding/json sorts map keys, but its number and Unicode escaping rules +// differ from the reference and would change the model's input tokens. +func clefJSON(value any) (string, error) { + switch v := value.(type) { + case nil: + return "null", nil + case bool: + return strconv.FormatBool(v), nil + case string: + var b strings.Builder + b.WriteByte('"') + for _, r := range v { + switch r { + case '"', '\\': + b.WriteByte('\\') + b.WriteRune(r) + case '\b': + b.WriteString(`\b`) + case '\f': + b.WriteString(`\f`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + default: + if r < 0x20 { + fmt.Fprintf(&b, `\u%04x`, r) + } else { + b.WriteRune(r) + } + } + } + b.WriteByte('"') + return b.String(), nil + case json.Number: + if !strings.ContainsAny(string(v), ".eE") { + if v == "-0" { + return "0", nil + } + return string(v), nil + } + n, err := v.Float64() + if math.IsInf(n, 0) { + if n < 0 { + return "-Infinity", nil + } + return "Infinity", nil + } + if err != nil { + return "", err + } + if a := math.Abs(n); a >= 1e16 || (a != 0 && a < 1e-4) { + return strconv.FormatFloat(n, 'e', -1, 64), nil + } + s := strconv.FormatFloat(n, 'f', -1, 64) + if !strings.ContainsRune(s, '.') { + s += ".0" + } + return s, nil + case []any: + parts := make([]string, len(v)) + for i, child := range v { + text, err := clefJSON(child) + if err != nil { + return "", err + } + parts[i] = text + } + return "[" + strings.Join(parts, ",") + "]", nil + case map[string]any: + parts := make([]string, 0, len(v)) + for _, key := range slices.Sorted(maps.Keys(v)) { + name, _ := clefJSON(key) + text, err := clefJSON(v[key]) + if err != nil { + return "", err + } + parts = append(parts, name+":"+text) + } + return "{" + strings.Join(parts, ",") + "}", nil + default: + return "", fmt.Errorf("unsupported JSON value %T", value) + } +} diff --git a/decision/clef_test.go b/decision/clef_test.go new file mode 100644 index 000000000..891cf323e --- /dev/null +++ b/decision/clef_test.go @@ -0,0 +1,184 @@ +package decision + +import ( + "encoding/json" + "math" + "strings" + "testing" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/llm" +) + +func TestCompileClef(t *testing.T) { + var req Request + if err := json.Unmarshal([]byte(`{"model":"renamed-model","state":{"z":2,"a":"café & "},"questions":{"team":{"type":"choice","instructions":"Route it","criteria":{"z":null,"a":"First"}},"yes":{"type":"noul","instructions":{"z":2,"a":1}},"rating":{"type":"score","instructions":"Rate it","criteria":["Low","High"]}}}`), &req); err != nil { + t.Fatal(err) + } + c, err := CompileWithEncoder(req, "clef") + if err != nil { + t.Fatal(err) + } + if err := c.Render(func([]api.Message) (string, error) { t.Fatal("Clef must not use a chat template"); return "", nil }); err != nil { + t.Fatal(err) + } + input := c.Request + if len(input.Fields) != 3 || len(c.Request.Rows) != 0 { + t.Fatal("expected one joint request") + } + if input.Segments[1] != `{"a":"café & ","z":2}` { + t.Fatalf("state = %s", input.Segments[1]) + } + option := func(f, i int) string { + return strings.Join(input.Segments[input.Fields[f].Options[i][0]:input.Fields[f].Options[i][1]], "") + } + if option(0, 0) != `{"description":"First","option_id":"a"}` || option(0, 1) != `{"option_id":"z"}` { + t.Fatalf("choice ordering / null semantics changed: %s, %s", option(0, 0), option(0, 1)) + } + if option(1, 0) != `{"description":"The proposition is true or the answer is yes.","option_id":"true"}` { + t.Fatal(option(1, 0)) + } + if input.Fields[0].Type != 1 || input.Fields[1].Type != 0 || input.Fields[2].Type != 2 { + t.Fatal("incorrect type ids") + } + prompt := strings.Join(input.Segments, "") + if !strings.HasSuffix(prompt, "\n\n\n\nJOINT SCHEMA DECISIONS:") || !strings.Contains(prompt, "INSTRUCTION: {\"a\":1,\"z\":2}\nALLOWED OPTIONS:\n") { + t.Fatal(prompt) + } + response, err := c.Answer(req.Model, llm.ScoreResponse{Logits: [][]float32{{2, 0}, {2, 0}, {0, 2}}, InputTokens: 99}) + if err != nil { + t.Fatal(err) + } + a, _ := response.Answers.Get("team") + choice := a.(ChoiceAnswer) + n, _ := response.Answers.Get("yes") + yes := n.(NoulAnswer) + if choice.Choice != "a" || math.Abs(choice.Confidence-0.4729346589968384) > 1e-8 || yes.Noul < 0.88 || response.Usage.OutputTokens != 0 { + t.Fatalf("incorrect joint answer: %+v %+v", choice, yes) + } +} + +func TestCompileClefImages(t *testing.T) { + req := testRequest(t) + req.Images = []api.ImageData{[]byte("first image"), []byte("second image")} + c, err := CompileWithEncoder(req, "clef") + if err != nil { + t.Fatal(err) + } + if c.Request.ImagePosition != 1 || len(c.Request.Images) != 2 || string(c.Request.Images[1]) != "second image" { + t.Fatalf("images were not preserved before state: %+v", c.Request) + } + if _, err := Compile(req); err == nil { + t.Fatal("candidate scoring silently ignored images") + } +} + +func TestClefCanonicalJSON(t *testing.T) { + for _, tt := range []struct{ input, want string }{ + {`{"v":1e6}`, `{"v":1000000.0}`}, + {`{"v":-0}`, `{"v":0}`}, + {`{"v":1e-5}`, `{"v":1e-05}`}, + {`[1.0,-0.0,1e15,1e16,1e-4,1e-999,1e999,-1e999]`, `[1.0,-0.0,1000000000000000.0,1e+16,0.0001,0.0,Infinity,-Infinity]`}, + {`{"z":9007199254740993,"a":[true,false,null]}`, `{"a":[true,false,null],"z":9007199254740993}`}, + {`{"v":"\u2028\u2029"}`, "{\"v\":\"

\"}"}, + {`{"v":"\\u2028\\u2029"}`, `{"v":"\\u2028\\u2029"}`}, + {`{"v":"\\\u2028"}`, "{\"v\":\"\\\\
\"}"}, + {`{"v":"\u0000\b\f\n\r\t\"\\\u007f<>&café"}`, "{\"v\":\"\\u0000\\b\\f\\n\\r\\t\\\"\\\\\x7f<>&café\"}"}, + {`null`, `null`}, + {`true`, `true`}, + {`"plain text"`, `plain text`}, + } { + t.Run(tt.input, func(t *testing.T) { + got, err := clefContent(json.RawMessage(tt.input)) + if err != nil || got != tt.want { + t.Fatalf("got %q, %v; want %q", got, err, tt.want) + } + }) + } + for _, input := range []string{"", `{`, `null true`} { + if _, err := clefContent(json.RawMessage(input)); err == nil { + t.Fatalf("accepted invalid JSON %q", input) + } + } +} + +func TestClefSchema(t *testing.T) { + var req Request + if err := json.Unmarshal([]byte(`{"model":"clef","state":null,"questions":{"team":{"type":"choice","criteria":{"z":null,"a":{"text":"First","weight":1e6}}},"yes":{"type":"noul","instructions":null,"criteria":{"true":null}},"rating":{"type":"score","instructions":"","criteria":[null,{"text":"High"}]}}}`), &req); err != nil { + t.Fatal(err) + } + c, err := CompileWithEncoder(req, "clef") + if err != nil { + t.Fatal(err) + } + if c.Request.Segments[1] != "null" { + t.Fatal("lost null state") + } + for i, name := range []string{"team", "yes", "rating"} { + if got := c.Request.Segments[c.Request.Fields[i].Question[0]]; got != name { + t.Fatalf("fallback instructions = %q, want %q", got, name) + } + } + for _, tt := range []struct { + field, option int + want string + }{ + {0, 0, `{"description":{"text":"First","weight":1000000.0},"option_id":"a"}`}, + {0, 1, `{"option_id":"z"}`}, + {1, 0, `{"option_id":"true"}`}, + {2, 0, `{"option_id":"0"}`}, + {2, 1, `{"description":{"text":"High"},"option_id":"1"}`}, + } { + span := c.Request.Fields[tt.field].Options[tt.option] + if got := c.Request.Segments[span[0]]; got != tt.want { + t.Fatalf("option = %q, want %q", got, tt.want) + } + } + answer, err := c.Answer(req.Model, llm.ScoreResponse{Logits: [][]float32{{2, 0}, {2, 0}, {0, 2}}}) + if err != nil { + t.Fatal(err) + } + rating, _ := answer.Answers.Get("rating") + legend, err := json.Marshal(rating.(ScoreAnswer).Legend) + if err != nil || string(legend) != `{"0":null,"1":{"text":"High"}}` { + t.Fatalf("legend = %s, %v", legend, err) + } + if _, err := Compile(req); err == nil { + t.Fatal("broadened the candidate-scoring schema") + } +} + +func TestClefValidation(t *testing.T) { + for _, raw := range []string{ + `{}`, `{"model":"clef","state":null,"questions":{}}`, + `{"model":"clef","questions":{"q":{"type":"noul"}}}`, + `{"model":"clef","state":null,"questions":{"":{"type":"noul"}}}`, + `{"model":"clef","state":null,"questions":{"q":{"type":"other"}}}`, + `{"model":"clef","state":null,"questions":{"q":{"type":"choice","criteria":{"a":null}}}}`, + `{"model":"clef","state":null,"questions":{"q":{"type":"score","criteria":{}}}}`, + `{"model":"clef","state":null,"questions":{"q":{"type":"noul","criteria":[true,false]}}}`, + `{"model":"clef","state":null,"questions":{"q":{"type":"noul","criteria":{"yes":null}}}}`, + } { + var req Request + if err := json.Unmarshal([]byte(raw), &req); err != nil { + t.Fatal(err) + } + if _, err := CompileWithEncoder(req, "clef"); err == nil { + t.Fatalf("accepted invalid request %s", raw) + } + } +} + +func TestDecisionRejectsVideo(t *testing.T) { + for _, encoding := range []string{"", "clef"} { + req := testRequest(t) + req.Videos = []json.RawMessage{json.RawMessage(`"video.mp4"`)} + if _, err := CompileWithEncoder(req, encoding); err == nil || !strings.Contains(err.Error(), "video") { + t.Fatalf("encoding %q silently accepted video: %v", encoding, err) + } + req.Videos = nil + if _, err := CompileWithEncoder(req, encoding); err != nil { + t.Fatal(err) + } + } +} diff --git a/decision/systemone.go b/decision/systemone.go index 8f07da7e2..0fb4279ba 100644 --- a/decision/systemone.go +++ b/decision/systemone.go @@ -27,13 +27,37 @@ type Compiled struct { messages [][]api.Message } +// Compile validates the request and prepares candidate scoring for Nimble and Tev. func Compile(req Request) (*Compiled, error) { + return CompileWithEncoder(req, "") +} + +// CompileWithEncoder validates the request and encodes it using the named input format. +// An empty encoding uses the candidate-scoring format shared by Nimble and Tev. +func CompileWithEncoder(req Request, encoding string) (*Compiled, error) { if strings.TrimSpace(req.Model) == "" { return nil, fmt.Errorf("model is required") } if req.Questions.Len() < 1 || req.Questions.Len() > 64 { return nil, fmt.Errorf("questions must contain 1–64 fields") } + if len(req.Videos) > 0 { + return nil, fmt.Errorf("video inputs are not supported") + } + switch encoding { + case "": + if len(req.Images) > 0 { + return nil, fmt.Errorf("this decision model does not support images") + } + case "clef": + c := &Compiled{} + if err := encodeClef(req, c); err != nil { + return nil, err + } + return c, nil + default: + return nil, fmt.Errorf("unsupported decision encoding %q", encoding) + } context, err := content(req.State) if err != nil { return nil, fmt.Errorf("state: %w", err) @@ -206,7 +230,11 @@ func (c *Compiled) Answer(model string, result llm.ScoreResponse) (Response, err } } if f.typ == "noul" { - response.Answers.Set(f.Name, NoulAnswer{Type: f.typ, Noul: p[1]}) + for j, choice := range f.Choices { + if choice.Value == true { + response.Answers.Set(f.Name, NoulAnswer{Type: f.typ, Noul: p[j]}) + } + } continue } probabilities := &Probabilities{} diff --git a/decision/systemone_test.go b/decision/systemone_test.go index 3e6a16b0e..663a091c7 100644 --- a/decision/systemone_test.go +++ b/decision/systemone_test.go @@ -90,6 +90,12 @@ func TestCompile(t *testing.T) { } } +func TestCompileUnsupportedEncoding(t *testing.T) { + if _, err := CompileWithEncoder(testRequest(t), "unknown"); err == nil { + t.Fatal("accepted unsupported decision encoding") + } +} + func TestStructuredStateFrames(t *testing.T) { req := testRequest(t) req.State = json.RawMessage(`{"frames":["first","second"],"position":7}`) @@ -207,8 +213,10 @@ func TestInvalidRequests(t *testing.T) { q, _ := req.Questions.Get("urgency") q.Criteria = json.RawMessage(`["x"` + strings.Repeat(`,"x"`, 26) + `]`) req.Questions.Set("urgency", q) - if _, err := Compile(req); err == nil { - t.Fatal("accepted 27 candidates") + for _, encoding := range []string{"", "clef"} { + if _, err := CompileWithEncoder(req, encoding); err == nil { + t.Fatalf("encoding %q accepted 27 candidates", encoding) + } } } diff --git a/decision/types.go b/decision/types.go index a3dfb6481..9b09fbfdd 100644 --- a/decision/types.go +++ b/decision/types.go @@ -12,10 +12,12 @@ import ( type Questions = orderedmap.Map[string, Question] type Request struct { - Model string `json:"model"` - State json.RawMessage `json:"state"` - Questions *Questions `json:"questions"` - KeepAlive *api.Duration `json:"keep_alive,omitempty"` + Model string `json:"model"` + State json.RawMessage `json:"state"` + Images []api.ImageData `json:"images,omitempty"` + Videos []json.RawMessage `json:"videos,omitempty"` + Questions *Questions `json:"questions"` + KeepAlive *api.Duration `json:"keep_alive,omitempty"` } type Question struct { @@ -28,7 +30,7 @@ type Question struct { type Choice struct { Code string `json:"code"` Value any `json:"value"` - Description string `json:"description"` + Description any `json:"description"` } // Field is a question's schema as presented to the model. @@ -41,7 +43,7 @@ type Field struct { type ( Answers = orderedmap.Map[string, any] Probabilities = orderedmap.Map[string, float64] - Legend = orderedmap.Map[string, string] + Legend = orderedmap.Map[string, any] ) type Response struct { diff --git a/docs/api/introduction.mdx b/docs/api/introduction.mdx index 0b42a6b30..2d7a73d4b 100644 --- a/docs/api/introduction.mdx +++ b/docs/api/introduction.mdx @@ -22,7 +22,7 @@ The Anthropic client appends `/v1/messages` to its base URL. Cloud requests need an API key. Local requests do not. To use cloud models through your local server, [sign in to Ollama](/api/authentication#signing-in). -Creating or deleting models requires a local Ollama server. For supported API features, see [OpenAI compatibility](/api/openai-compatibility) and [Anthropic compatibility](/api/anthropic-compatibility). +Creating or deleting models requires a local Ollama server. [System One](/capabilities/decision) also requires a local server and is not yet available on Ollama Cloud. For supported API features, see [OpenAI compatibility](/api/openai-compatibility) and [Anthropic compatibility](/api/anthropic-compatibility). ## Ollama API example diff --git a/docs/capabilities/decision.mdx b/docs/capabilities/decision.mdx new file mode 100644 index 000000000..5ba05daf2 --- /dev/null +++ b/docs/capabilities/decision.mdx @@ -0,0 +1,113 @@ +--- +title: Decision +description: Classify text, estimate yes/no probabilities, and score ordered criteria. +--- + + +Currently available locally in Ollama. + + +Use System One to classify text, answer yes/no questions, or score text against a rubric. + +## Setup + +Start [Ollama](/quickstart) v0.35.0 or later, then download [Nimble](https://ollama.com/library/nimble): + +```shell setup.sh +ollama pull nimble +``` + +Local requests do not need an API key. + +## Choose a label + +Put your input in `state` and describe the possible answers in `criteria`: + +```shell choice.sh +curl http://localhost:11434/v1/systemone \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "nimble", + "state": "Our checkout has returned 500 errors since 9am.", + "questions": { + "label": { + "type": "choice", + "instructions": "Which label fits this ticket?", + "criteria": { + "billing": "Payments and refunds", + "bug": "Software errors", + "account": "Login and account access" + } + } + } + }' +``` + +Example response: + +```json +{ + "model": "nimble", + "answers": { + "label": { + "type": "choice", + "choice": "bug", + "probabilities": {"billing": 0.0125, "bug": 0.9781, "account": 0.0093}, + "confidence": 0.8906 + } + }, + "usage": {"input_tokens": 174, "output_tokens": 1} +} +``` + +## Question types + +| Type | Criteria | Result | +| --- | --- | --- | +| `choice` | 2–26 named options with descriptions | The most likely option and each option's probability | +| `noul` | Descriptions for `"false"` and `"true"` | The probability of `true`, from 0 to 1 | +| `score` | 2–26 descriptions, ordered from lowest to highest | The probability-weighted average of the zero-based levels | + +For the urgency criteria below: + +```text +score = 0 × P(Routine) + 1 × P(Soon) + 2 × P(Immediate) +``` + +For `choice` and `score`, `confidence` measures how strongly the model favors one answer over the others, from 0 to 1. A higher value does not guarantee the answer is correct. + +## Ask multiple questions + +Combine question types in one request. Here, both questions use the same ticket: + +```shell multiple.sh +curl http://localhost:11434/v1/systemone \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "nimble", + "state": {"ticket": "I was charged twice. Please refund the extra payment."}, + "questions": { + "refund": { + "type": "noul", + "instructions": "Is the customer requesting a refund?", + "criteria": { + "false": "No refund is requested", + "true": "The customer requests a refund" + } + }, + "urgency": { + "type": "score", + "instructions": "How urgently does this ticket need a response?", + "criteria": [ + "Routine: no time pressure", + "Soon: a customer is inconvenienced", + "Immediate: a critical service is unavailable" + ] + } + } + }' +``` + +This returns a refund probability of `0.9989` and an urgency score of `0.8308` on the 0–2 scale (rounded). Results can vary with the model and configuration. + +See the [API reference](/api/systemone) for request fields, limits, errors, and token usage. diff --git a/docs/docs.json b/docs/docs.json index 1f7524517..44a655bcd 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -128,6 +128,7 @@ "/capabilities/streaming", "/capabilities/thinking", "/capabilities/structured-outputs", + "/capabilities/decision", "/capabilities/vision", "/capabilities/embeddings", "/capabilities/tool-calling", @@ -249,6 +250,7 @@ "POST /api/generate", "POST /api/chat", "POST /api/embed", + "POST /v1/systemone", "GET /api/tags", "GET /api/ps", "POST /api/show", diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 874a196a0..60ea409ef 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -828,6 +828,195 @@ components: items: $ref: "#/components/schemas/TokenLogprob" description: Most likely tokens and their log probabilities at this position + SystemOneContent: + description: A nonempty string, or an object or array serialized as JSON text. Not interpreted as chat messages or multimodal input. + oneOf: + - type: string + pattern: '\S' + - type: object + additionalProperties: true + - type: array + items: {} + SystemOneChoiceQuestion: + title: Choice + type: object + required: [type, instructions, criteria] + properties: + type: + type: string + enum: [choice] + instructions: + $ref: "#/components/schemas/SystemOneContent" + criteria: + type: object + description: Option keys mapped to descriptions. A null description uses the key itself. Keys must not be blank; ties follow the model's option order. + minProperties: 2 + maxProperties: 26 + propertyNames: + pattern: '\S' + additionalProperties: + type: [string, "null"] + SystemOneNoulQuestion: + title: Noul + type: object + required: [type, instructions] + properties: + type: + type: string + enum: [noul] + instructions: + $ref: "#/components/schemas/SystemOneContent" + criteria: + type: object + description: Optional descriptions for the two outcomes. Omitted entries use No and Yes. + properties: + "false": + type: string + default: "No" + "true": + type: string + default: "Yes" + additionalProperties: false + SystemOneScoreQuestion: + title: Score + type: object + required: [type, instructions, criteria] + properties: + type: + type: string + enum: [score] + instructions: + $ref: "#/components/schemas/SystemOneContent" + criteria: + type: array + description: Descriptions ordered from the lowest score (index 0) to the highest. Defines a scale from 0 to the number of criteria minus 1. + minItems: 2 + maxItems: 26 + items: + type: string + SystemOneRequest: + type: object + required: [model, state, questions] + properties: + model: + type: string + pattern: '\S' + description: Local model trained for System One, such as nimble. Requires compatible GGUF weights and a scoring-capable runner; cloud and MLX/Safetensors models are not supported. + state: + $ref: "#/components/schemas/SystemOneContent" + images: + type: array + description: Base64-encoded images shared by all questions, in request order. Requires Clef or Clef Flash with vision weights. URLs and data URLs are not supported. + items: + type: string + format: byte + questions: + type: object + description: Named questions about the shared state. Answers are not passed to later questions. + minProperties: 1 + maxProperties: 64 + propertyNames: + pattern: '\S' + additionalProperties: + oneOf: + - $ref: "#/components/schemas/SystemOneChoiceQuestion" + - $ref: "#/components/schemas/SystemOneNoulQuestion" + - $ref: "#/components/schemas/SystemOneScoreQuestion" + keep_alive: + oneOf: + - type: string + - type: number + description: How long to keep the model loaded after the request, as a duration string (such as 5m) or seconds. Zero unloads after the request; a negative value keeps it loaded. Defaults to the server's keep-alive setting (5m unless configured otherwise). + SystemOneProbabilities: + type: object + description: Probabilities normalized over the supplied candidates, summing to 1 subject to floating-point precision. + additionalProperties: + type: number + minimum: 0 + maximum: 1 + SystemOneConfidence: + type: number + minimum: 0 + maximum: 1 + description: Distribution concentration, calculated as 1 - H(p) / ln(N), where H(p) is entropy and N is the candidate count. Zero means uniform probabilities; values near 1 mean one candidate dominates. Not calibrated correctness. + SystemOneChoiceAnswer: + title: Choice + type: object + required: [type, choice, probabilities, confidence] + properties: + type: + type: string + enum: [choice] + choice: + type: string + description: Option key with the highest probability. Ties follow the model's option order. + probabilities: + $ref: "#/components/schemas/SystemOneProbabilities" + confidence: + $ref: "#/components/schemas/SystemOneConfidence" + SystemOneNoulAnswer: + title: Noul + type: object + required: [type, noul] + properties: + type: + type: string + enum: [noul] + noul: + type: number + minimum: 0 + maximum: 1 + description: Probability of true among the false and true candidates. This is a number, not a Boolean. + SystemOneScoreAnswer: + title: Score + type: object + required: [type, score, legend, probabilities, confidence] + properties: + type: + type: string + enum: [score] + score: + type: number + minimum: 0 + maximum: 25 + description: Probability-weighted average of the zero-based criterion indices, from 0 to the number of criteria minus 1. Not rounded to a level or normalized to 0–1. + legend: + type: object + description: Zero-based indices as string keys mapped to the criterion descriptions. + additionalProperties: + type: string + probabilities: + $ref: "#/components/schemas/SystemOneProbabilities" + description: Probabilities keyed by zero-based criterion indices as strings. + confidence: + $ref: "#/components/schemas/SystemOneConfidence" + SystemOneResponse: + type: object + required: [model, answers, usage] + properties: + model: + type: string + description: Model name from the request. + answers: + type: object + description: Answers keyed by the question names in the request. + additionalProperties: + oneOf: + - $ref: "#/components/schemas/SystemOneChoiceAnswer" + - $ref: "#/components/schemas/SystemOneNoulAnswer" + - $ref: "#/components/schemas/SystemOneScoreAnswer" + usage: + type: object + required: [input_tokens, output_tokens] + properties: + input_tokens: + type: integer + minimum: 0 + description: Total evaluated input tokens, including image positions. Shared context is counted again when the model scores questions separately. + output_tokens: + type: integer + minimum: 0 + description: Tokens generated internally for scoring, including prefix preparation and retries. May exceed the question count; not the length of the JSON response. ErrorResponse: type: object properties: @@ -835,6 +1024,109 @@ components: type: string description: Error message describing what went wrong paths: + /v1/systemone: + post: + summary: Answer typed questions + description: | + Answer choice, yes/no, and scoring questions with a local System One model. Requires Ollama v0.35.0 or later. See the [decision guide](/capabilities/decision) for examples. + operationId: systemone + x-mint: + href: /api/systemone + metadata: + title: System One + sidebarTitle: System One + content: | + + Currently available locally in Ollama. + + + - Returns one JSON response; streaming, video, tools, and generation controls are not supported. + - Requests without images must fit within 64 KiB; requests with images must fit within 32 MiB, including base64 and JSON. The complete input must fit the loaded context window. Nimble and Tev also require two token positions for scoring. Input is never truncated. + x-codeSamples: + - lang: bash + label: Choose a label + source: | + curl http://localhost:11434/v1/systemone \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "nimble", + "state": "Our checkout has returned 500 errors since 9am.", + "questions": { + "label": { + "type": "choice", + "instructions": "Which label fits this ticket?", + "criteria": { + "billing": "Payments and refunds", + "bug": "Software errors", + "account": "Login and account access" + } + } + } + }' + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/SystemOneRequest" + example: + model: nimble + state: Our checkout has returned 500 errors since 9am. + questions: + label: + type: choice + instructions: Which label fits this ticket? + criteria: + billing: Payments and refunds + bug: Software errors + account: Login and account access + responses: + "200": + description: Answers and token usage for all questions. Example probabilities and confidence are rounded to four decimal places; results and usage can vary with the model and server configuration. + content: + application/json: + schema: + $ref: "#/components/schemas/SystemOneResponse" + example: + model: nimble + answers: + label: + type: choice + choice: bug + probabilities: + billing: 0.0125 + bug: 0.9781 + account: 0.0093 + confidence: 0.8906 + usage: + input_tokens: 174 + output_tokens: 1 + "400": + description: Invalid request, unsupported model or runner, or a rendered prompt that exceeds the loaded context. Cloud models are rejected. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + "404": + description: Local model not found. Download the model before making a request. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + "413": + description: Request body exceeds 64 KiB without images or 32 MiB with images. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + error: request body must not exceed 64 KiB without images + "500": + description: Model loading, rendering, or scoring failed. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" /api/generate: post: summary: Generate a response diff --git a/llama/clef/clef.cpp b/llama/clef/clef.cpp new file mode 100644 index 000000000..f5ad2e6fe --- /dev/null +++ b/llama/clef/clef.cpp @@ -0,0 +1,375 @@ +// Clef joint schema head, ported from Cloudflare's Apache-2.0 reference. +// https://huggingface.co/Cloudflare/clef/blob/main/joint_schema_model.py +#include "clef.h" +#include "ggml-backend.h" +#include "ggml.h" +#include "gguf.h" +#include +#include +#include +#include +#include +#include + +namespace { +struct matrix { + int rows = 0, cols = 0; + std::vector data; + matrix() = default; + matrix(int r, int c) : rows(r), cols(c), data(size_t(r) * c) {} + float &at(int r, int c) { return data[size_t(r) * cols + c]; } + float at(int r, int c) const { return data[size_t(r) * cols + c]; } +}; +void require(bool condition, const char *message) { + if (!condition) + throw std::runtime_error(message); +} +// Use the backend registry so builds with dynamically selected CPU variants +// work without linking to a particular ggml-cpu library. +struct cpu_backend { + ggml_backend_t handle = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + cpu_backend() { + require(handle != nullptr, "Clef: CPU backend unavailable"); + auto reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(handle)); + auto set_threads = reinterpret_cast( + ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads")); + if (set_threads) + set_threads(handle, 8); + } + ~cpu_backend() { ggml_backend_free(handle); } +}; +struct graph { + ggml_context *ctx; + explicit graph(size_t bytes) { + ctx = ggml_init({bytes + (2 << 20), nullptr, true}); + require(ctx != nullptr, "Clef: cannot allocate graph"); + } + ~graph() { ggml_free(ctx); } + ggml_tensor *input(const matrix &m) { + auto t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, m.cols, m.rows); + t->data = const_cast(m.data.data()); + return t; + } + matrix run(ggml_tensor *output, int rows, int cols) { + auto gf = ggml_new_graph(ctx); + ggml_build_forward_expand(gf, output); + static thread_local cpu_backend cpu; + require(ggml_backend_graph_compute(cpu.handle, gf) == GGML_STATUS_SUCCESS, + "Clef: graph evaluation failed"); + matrix result(rows, cols); + memcpy(result.data.data(), output->data, result.data.size() * sizeof(float)); + return result; + } +}; +matrix linear(const matrix &x, const matrix &w) { + require(x.cols == w.cols, "Clef: linear shape mismatch"); + graph g(size_t(x.rows) * w.rows * sizeof(float)); + auto tx = g.input(x), tw = g.input(w); + ggml_set_no_alloc(g.ctx, false); + return g.run(ggml_mul_mat(g.ctx, tw, tx), x.rows, w.rows); +} +matrix add(matrix a, const matrix &b) { + require(a.cols == b.cols && (b.rows == 1 || a.rows == b.rows), "Clef: addition shape mismatch"); + for (int r = 0; r < a.rows; ++r) + for (int c = 0; c < a.cols; ++c) + a.at(r, c) += b.at(b.rows == 1 ? 0 : r, c); + return a; +} +matrix slice(const matrix &m, int start, int end) { + require(start >= 0 && start < end && end <= m.rows, "Clef: invalid span"); + matrix out(end - start, m.cols); + std::copy(m.data.begin() + size_t(start) * m.cols, m.data.begin() + size_t(end) * m.cols, + out.data.begin()); + return out; +} +matrix mean(const matrix &m, int start, int end) { + require(start >= 0 && start < end && end <= m.rows, "Clef: invalid span"); + matrix out(1, m.cols); + for (int r = start; r < end; ++r) + for (int c = 0; c < m.cols; ++c) + out.at(0, c) += m.at(r, c) / (end - start); + return out; +} +void append(matrix &a, const matrix &b) { + require(a.rows == 0 || a.cols == b.cols, "Clef: append shape mismatch"); + a.cols = b.cols; + a.rows += b.rows; + a.data.insert(a.data.end(), b.data.begin(), b.data.end()); +} +float cosine(const matrix &a, int ar, const matrix &b, int br, double eps = 1e-8) { + require(a.cols == b.cols, "Clef: cosine shape mismatch"); + double dot = 0, aa = 0, bb = 0; + for (int c = 0; c < a.cols; ++c) { + dot += double(a.at(ar, c)) * b.at(br, c); + aa += double(a.at(ar, c)) * a.at(ar, c); + bb += double(b.at(br, c)) * b.at(br, c); + } + return dot / (std::max(std::sqrt(aa), eps) * std::max(std::sqrt(bb), eps)); +} +} // namespace + +struct clef_head::impl { + std::ifstream file; + gguf_context *meta = nullptr; + ggml_context *tensors = nullptr; + std::map weights; + int hidden, width, heads, routing_layers, layers; + int output_index; + ggml_tensor *output; + + explicit impl(const std::string &path) : file(path, std::ios::binary) { + try { + meta = gguf_init_from_file(path.c_str(), {true, &tensors}); + require(meta && tensors && file.good(), "Clef: cannot open model"); + const int arch_key = gguf_find_key(meta, "general.architecture"); + require(arch_key >= 0 && gguf_get_kv_type(meta, arch_key) == GGUF_TYPE_STRING, + "Clef: missing architecture"); + const std::string prefix = std::string(gguf_get_val_str(meta, arch_key)) + ".decision."; + auto integer = [&](const std::string &key) { + int i = gguf_find_key(meta, (prefix + key).c_str()); + require(i >= 0 && gguf_get_kv_type(meta, i) == GGUF_TYPE_UINT32, + "Clef: missing head configuration"); + return int(gguf_get_val_u32(meta, i)); + }; + hidden = integer("hidden_size"); + width = integer("width"); + heads = integer("heads"); + routing_layers = integer("routing_layers"); + layers = integer("layers"); + require(hidden > 0 && hidden <= 16384 && width > 0 && width <= 4096 && heads > 0 && + width % heads == 0 && routing_layers >= 0 && routing_layers <= 32 && layers >= 0 && + layers <= 32, + "Clef: invalid head configuration"); + for (int i = 0; i < gguf_get_n_tensors(meta); ++i) { + std::string name = gguf_get_tensor_name(meta, i); + if (name.rfind("clef.", 0) != 0) + continue; + auto t = ggml_get_tensor(tensors, name.c_str()); + require(t->ne[2] == 1 && t->ne[3] == 1 && t->type == GGML_TYPE_F32, + "Clef: head tensors must be float32 matrices"); + matrix m(t->ne[1], t->ne[0]); + read(i, 0, m.data.data(), m.data.size() * sizeof(float)); + weights.emplace(name.substr(5), std::move(m)); + } + output_index = gguf_find_tensor(meta, "output.weight"); + require(output_index >= 0, "Clef: output embeddings are missing"); + output = ggml_get_tensor(tensors, "output.weight"); + require(output->ne[0] == hidden, "Clef: output embedding shape mismatch"); + } catch (...) { + if (tensors) + ggml_free(tensors); + if (meta) + gguf_free(meta); + throw; + } + } + ~impl() { + ggml_free(tensors); + gguf_free(meta); + } + void read(int index, size_t offset, void *data, size_t bytes) { + file.seekg(gguf_get_data_offset(meta) + gguf_get_tensor_offset(meta, index) + offset); + file.read(static_cast(data), bytes); + require(file.good(), "Clef: truncated tensor data"); + } + const matrix &w(const std::string &key) { return weights.at(key); } + matrix norm(const matrix &x, const std::string &key) { + const auto &weight = w(key + ".weight"), &bias = w(key + ".bias"); + require(weight.data.size() == size_t(x.cols) && bias.data.size() == size_t(x.cols), + "Clef: layer norm shape mismatch"); + matrix out = x; + for (int r = 0; r < x.rows; ++r) { + double m = 0, v = 0; + for (int c = 0; c < x.cols; ++c) + m += x.at(r, c) / x.cols; + for (int c = 0; c < x.cols; ++c) + v += (x.at(r, c) - m) * (x.at(r, c) - m) / x.cols; + for (int c = 0; c < x.cols; ++c) + out.at(r, c) = (x.at(r, c) - m) / std::sqrt(v + 1e-5) * weight.data[c] + bias.data[c]; + } + return out; + } + matrix project(const matrix &x, const std::string &key, bool bias = false) { + matrix out = linear(x, w(key + ".weight")); + return bias ? add(std::move(out), w(key + ".bias")) : out; + } + matrix feedforward(matrix x, const std::string &first, const std::string &last) { + x = project(x, first, true); + for (auto &v : x.data) + v = 0.5f * v * (1.0f + std::erf(v / std::sqrt(2.0f))); + return project(x, last, true); + } + matrix attention(const matrix &query, const matrix &memory, const std::string &key) { + const auto &weight = w(key + ".in_proj_weight"), &bias = w(key + ".in_proj_bias"); + require(weight.rows == 3 * width && weight.cols == width && bias.data.size() == size_t(3 * width), + "Clef: attention shape mismatch"); + auto projection = [&](const matrix &x, int n) { + auto y = linear(x, slice(weight, n * width, (n + 1) * width)); + for (int r = 0; r < y.rows; ++r) + for (int c = 0; c < width; ++c) + y.at(r, c) += bias.data[n * width + c]; + return y; + }; + auto q = projection(query, 0), k = projection(memory, 1), v = projection(memory, 2); + const int d = width / heads, nq = q.rows, nk = k.rows; + graph g((size_t(nq) * nk * heads * 3 + size_t(nq + nk) * width * 8) * sizeof(float)); + auto tq = g.input(q), tk = g.input(k), tv = g.input(v); + ggml_set_no_alloc(g.ctx, false); + auto split = [&](ggml_tensor *t, int n) { + return ggml_cont(g.ctx, ggml_permute(g.ctx, ggml_reshape_3d(g.ctx, t, d, heads, n), 0, 2, 1, 3)); + }; + tq = split(tq, nq); + tk = split(tk, nk); + tv = split(tv, nk); + auto scores = + ggml_soft_max(g.ctx, ggml_scale(g.ctx, ggml_mul_mat(g.ctx, tk, tq), 1.0f / std::sqrt(float(d)))); + tv = ggml_cont(g.ctx, ggml_transpose(g.ctx, tv)); + auto out = ggml_mul_mat(g.ctx, tv, scores); + out = ggml_cont(g.ctx, ggml_permute(g.ctx, out, 0, 2, 1, 3)); + auto result = g.run(ggml_reshape_2d(g.ctx, out, width, nq), nq, width); + return project(result, key + ".out_proj", true); + } + matrix lexical(const std::vector &tokens, int start, int end) { + require(start >= 0 && start < end && end <= int(tokens.size()), "Clef: invalid lexical span"); + matrix result(1, hidden); + const auto traits = ggml_get_type_traits(output->type); + require(traits->to_float != nullptr || output->type == GGML_TYPE_F32, + "Clef: unsupported output embedding type"); + size_t bytes = ggml_row_size(output->type, hidden); + std::vector raw(bytes); + std::vector row(hidden); + for (int i = start; i < end; ++i) { + require(tokens[i] >= 0 && tokens[i] < output->ne[1], "Clef: invalid token"); + read(output_index, size_t(tokens[i]) * bytes, raw.data(), bytes); + if (output->type == GGML_TYPE_F32) + memcpy(row.data(), raw.data(), bytes); + else + traits->to_float(raw.data(), row.data(), hidden); + for (int c = 0; c < hidden; ++c) + result.at(0, c) += row[c] / (end - start); + } + return result; + } + std::vector> score(const std::vector> &states, + const std::vector &tokens, const common_json &fields) { + require(states.size() == tokens.size() && !states.empty(), "Clef: missing backbone hidden states"); + require(fields.is_array() && !fields.empty() && fields.size() <= 64, "Clef: expected 1-64 fields"); + matrix x; + for (const auto &row : states) { + require(row.size() == size_t(hidden), "Clef: hidden size mismatch"); + matrix m(1, hidden); + m.data = row; + append(x, m); + } + x = norm(x, "hidden_norm"); + auto memory = project(x, "memory_projection"), global = slice(x, x.rows - 1, x.rows); + matrix questions, lexical_options, queries; + std::vector counts, types; + auto span = [](const common_json &value) { + require(value.is_array() && value.size() == 2 && value[0].is_number_integer() && + value[1].is_number_integer(), + "Clef: malformed span"); + return std::pair{value[0].get(), value[1].get()}; + }; + for (const auto &field : fields) { + int type = field.at("type").get(); + require(type >= 0 && type < 3, "Clef: invalid question type"); + types.push_back(type); + auto [qs, qe] = span(field.at("question")); + auto question = mean(x, qs, qe); + append(questions, question); + const auto &options = field.at("options"); + require(options.is_array() && options.size() >= 2 && options.size() <= 26, + "Clef: expected 2-26 options"); + counts.push_back(options.size()); + matrix contexts, lex; + for (const auto &option : options) { + auto [s, e] = span(option); + append(contexts, mean(x, s, e)); + append(lex, lexical(tokens, s, e)); + } + append(lexical_options, lex); + append(queries, add(add(project(contexts, "option_context_projection"), + project(lex, "option_lexical_projection")), + project(question, "option_question_projection"))); + } + for (int i = 0; i < routing_layers; ++i) { + auto key = "evidence_layers." + std::to_string(i); + queries = add(queries, attention(norm(queries, key + ".query_norm"), + norm(memory, key + ".memory_norm"), key + ".attention")); + queries = add(queries, feedforward(norm(queries, key + ".feedforward_norm"), + key + ".feedforward.0", key + ".feedforward.3")); + } + auto base = project(questions, "question_projection"); + matrix summaries; + int offset = 0; + for (int f = 0; f < base.rows; ++f) { + std::vector scores(counts[f]); + for (int j = 0; j < counts[f]; ++j) + for (int c = 0; c < width; ++c) + scores[j] += queries.at(offset + j, c) * base.at(f, c) / std::sqrt(float(width)); + float peak = *std::max_element(scores.begin(), scores.end()), sum = 0; + for (auto &v : scores) { + v = std::exp(v - peak); + sum += v; + } + matrix summary(1, width); + for (int j = 0; j < counts[f]; ++j) + for (int c = 0; c < width; ++c) + summary.at(0, c) += scores[j] / sum * queries.at(offset + j, c); + append(summaries, summary); + offset += counts[f]; + } + auto fs = + add(add(base, norm(summaries, "option_summary_norm")), project(global, "global_projection")); + const auto &te = w("type_embedding.weight"); + require(te.rows == 3 && te.cols == width, "Clef: type embedding shape mismatch"); + for (int f = 0; f < fs.rows; ++f) + for (int c = 0; c < width; ++c) + fs.at(f, c) += te.at(types[f], c); + for (int i = 0; i < layers; ++i) { + auto key = "layers." + std::to_string(i); + auto normalized = norm(fs, key + ".norm1"); + fs = add(fs, attention(normalized, normalized, key + ".self_attn")); + fs = add(fs, attention(norm(fs, key + ".norm2"), memory, key + ".multihead_attn")); + fs = add(fs, feedforward(norm(fs, key + ".norm3"), key + ".linear1", key + ".linear2")); + } + fs = norm(fs, "field_norm"); + auto options = norm(queries, "option_norm"); + float prior_scale = std::exp(std::min(w("prior_logit_scale").data.at(0), std::log(100.0f))); + float joint_scale = std::exp(std::min(w("joint_logit_scale").data.at(0), std::log(100.0f))); + float gate = 1.0f / (1.0f + std::exp(-w("residual_gate").data.at(0))); + std::vector> result; + offset = 0; + for (int f = 0; f < fs.rows; ++f) { + matrix features(counts[f], 4 * width); + for (int j = 0; j < counts[f]; ++j) + for (int c = 0; c < width; ++c) { + float a = fs.at(f, c), b = options.at(offset + j, c); + features.at(j, c) = a; + features.at(j, width + c) = b; + features.at(j, 2 * width + c) = a * b; + features.at(j, 3 * width + c) = std::abs(a - b); + } + auto residual = feedforward(features, "residual_scorer.0", "residual_scorer.3"); + auto anchor = add(slice(questions, f, f + 1), global); + std::vector logits; + for (int j = 0; j < counts[f]; ++j) { + float v = prior_scale * cosine(lexical_options, offset + j, anchor, 0, 1e-12) + + gate * (joint_scale * cosine(fs, f, options, offset + j) + residual.at(j, 0)); + require(std::isfinite(v), "Clef: non-finite logit"); + logits.push_back(v); + } + result.push_back(std::move(logits)); + offset += counts[f]; + } + return result; + } +}; +clef_head::clef_head(const std::string &path) : p(new impl(path)) {} +clef_head::~clef_head() = default; +std::vector> clef_head::score(const std::vector> &hidden, + const std::vector &tokens, + const common_json &fields) { + return p->score(hidden, tokens, fields); +} diff --git a/llama/clef/clef.h b/llama/clef/clef.h new file mode 100644 index 000000000..ea2607d32 --- /dev/null +++ b/llama/clef/clef.h @@ -0,0 +1,20 @@ +#pragma once +#include "json.h" +#include +#include +#include +#include + +// The backbone is evaluated by llama-server. This head consumes every final +// hidden state and the token spans produced by Cloudflare's record format. +class clef_head { + public: + explicit clef_head(const std::string &path); + ~clef_head(); + std::vector> score(const std::vector> &hidden, + const std::vector &tokens, const common_json &fields); + + private: + struct impl; + std::unique_ptr p; +}; diff --git a/llama/clef/convert.py b/llama/clef/convert.py new file mode 100644 index 000000000..ac18c32b2 --- /dev/null +++ b/llama/clef/convert.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Convert Cloudflare Clef with its trained joint head, using pinned llama.cpp. + +Usage: python convert.py --llama-cpp build/llama-src MODEL --outtype q8_0 --outfile MODEL.gguf +For the vision projector, add --mmproj --outtype f16 and use a separate output file. +All remaining arguments are passed to convert_hf_to_gguf.py. +""" +import argparse +import json +from pathlib import Path +import sys + +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument('--llama-cpp', type=Path, required=True) +args, remaining = parser.parse_known_args() +sys.path[:0] = [str(args.llama_cpp.resolve()), str((args.llama_cpp / 'gguf-py').resolve())] + +import numpy as np +from safetensors import safe_open +from conversion.qwen import Qwen3_5TextModel +import convert_hf_to_gguf + +original_tensors = Qwen3_5TextModel.prepare_tensors +original_metadata = Qwen3_5TextModel.set_gguf_parameters + + +def prepare_tensors(self): + original_tensors(self) + with safe_open(self.dir_model / 'joint_head.safetensors', framework='pt', device='cpu') as head: + for name in head.keys(): + # Keep the small head in float32, including its scalar gates. + data = head.get_tensor(name).float().numpy() + self.gguf_writer.add_tensor('clef.' + name, np.atleast_1d(data)) + + +def set_gguf_parameters(self): + original_metadata(self) + config = json.loads((self.dir_model / 'joint_head_config.json').read_text()) + self.gguf_writer.add_string(f'{self.gguf_writer.arch}.decision.type', 'clef') + for name, value in config.items(): + self.gguf_writer.add_uint32(f'{self.gguf_writer.arch}.decision.{name}', value) + + +Qwen3_5TextModel.prepare_tensors = prepare_tensors +Qwen3_5TextModel.set_gguf_parameters = set_gguf_parameters +sys.argv = [sys.argv[0], '--no-mtp', *remaining] +convert_hf_to_gguf.main() diff --git a/llama/compat/002-clef.patch b/llama/compat/002-clef.patch new file mode 100644 index 000000000..6753b26bf --- /dev/null +++ b/llama/compat/002-clef.patch @@ -0,0 +1,299 @@ +--- a/common/common.cpp ++++ b/common/common.cpp +@@ -1240,6 +1240,18 @@ + std::vector samplers_seq_config; + }; + ++common_decision_type common_get_decision_type(const llama_model * model) { ++ char value[128]; ++ if (llama_model_meta_val_str(model, "general.architecture", value, sizeof(value)) < 0) { ++ return COMMON_DECISION_TYPE_NONE; ++ } ++ const std::string key = std::string(value) + ".decision.type"; ++ if (llama_model_meta_val_str(model, key.c_str(), value, sizeof(value)) < 0) { ++ return COMMON_DECISION_TYPE_NONE; ++ } ++ return std::strcmp(value, "clef") == 0 ? COMMON_DECISION_TYPE_CLEF : COMMON_DECISION_TYPE_UNKNOWN; ++} ++ + common_init_result::common_init_result(common_params & params, bool model_only) : + pimpl(new impl{}) { + auto mparams = common_model_params_to_llama(params); +@@ -1291,6 +1303,18 @@ + } + + const llama_vocab * vocab = llama_model_get_vocab(model); ++ ++ if (common_get_decision_type(model) == COMMON_DECISION_TYPE_CLEF) { ++ params.embedding = cparams.embeddings = true; ++ cparams.ctx_type = LLAMA_CONTEXT_TYPE_DECISION; ++ params.pooling_type = cparams.pooling_type = LLAMA_POOLING_TYPE_NONE; ++ // The joint head consumes every hidden state; retain the whole input. ++ const auto n_ctx = cparams.n_ctx ? cparams.n_ctx : llama_model_n_ctx_train(model); ++ params.n_batch = cparams.n_batch = n_ctx; ++ params.n_ubatch = cparams.n_ubatch = n_ctx; ++ params.n_outputs_max = cparams.n_outputs_max = n_ctx; ++ params.n_outputs_max_per_seq = cparams.n_outputs_max_per_seq = 1; ++ } + + // load and optionally apply lora adapters + for (auto & la : params.lora_adapters) { +--- a/common/common.h ++++ b/common/common.h +@@ -940,6 +940,14 @@ + + struct common_sampler; + ++enum common_decision_type { ++ COMMON_DECISION_TYPE_NONE, ++ COMMON_DECISION_TYPE_CLEF, ++ COMMON_DECISION_TYPE_UNKNOWN, ++}; ++ ++common_decision_type common_get_decision_type(const struct llama_model * model); ++ + // note: defines the model, context, samplers, ets. lifetimes + struct common_init_result { + common_init_result(common_params & params, bool model_only = false); +--- a/tools/server/server-task.h ++++ b/tools/server/server-task.h +@@ -152,6 +152,7 @@ + // used by SERVER_TASK_TYPE_INFERENCE + task_params params; + server_tokens tokens; ++ common_json score_fields; // optional decision question and option spans + + // only used by CLI, this allow tokenizing CLI inputs on server side + // we need this because mtmd_context and vocab are not accessible outside of server_context +@@ -452,6 +453,7 @@ + }; + + struct server_task_result_embd : server_task_result { ++ common_json score_logits; + std::vector> embedding; + + int32_t n_tokens; +--- a/tools/server/server-task.cpp ++++ b/tools/server/server-task.cpp +@@ -1464,6 +1464,9 @@ + // server_task_result_embd + // + json server_task_result_embd::to_json() { ++ if (!score_logits.is_null()) { ++ return json {{"logits", score_logits}, {"tokens_evaluated", n_tokens}}; ++ } + return res_type == TASK_RESPONSE_TYPE_OAI_EMBD + ? to_json_oaicompat() + : to_json_non_oaicompat(); +--- a/tools/server/server-context.cpp ++++ b/tools/server/server-context.cpp +@@ -1,4 +1,5 @@ + #include "server-context.h" ++#include "clef.h" + #include "server-chat.h" + #include "server-common.h" + #include "server-http.h" +@@ -247,6 +248,7 @@ + // multimodal + mtmd_context * mctx = nullptr; + mtmd::batch_ptr mbatch = nullptr; ++ std::vector> score_embeddings; + + // speculative decoding + common_speculative * spec; +@@ -370,6 +372,7 @@ + SLT_DBG(*this, "%s", "\n"); + + spec_is_replay = false; ++ score_embeddings.clear(); + + last_nl_pos = 0; + generated_text = ""; +@@ -428,6 +431,18 @@ + + SLT_TRC(*this, "init sampler, took %0.2f ms, tokens: text = %d, total = %d\n", + (ggml_time_us() - t_start) / 1000.0, n_text, (int) prompt.tokens.size()); ++ } ++ ++ void collect_score_embeddings(const llama_batch & batch) { ++ if (!task || task->score_fields.is_null()) return; ++ // Embedding mode returns every token, including image tokens with logits disabled. ++ const int width = llama_model_n_embd_out(llama_get_model(ctx_tgt)); ++ for (int i = 0; i < batch.n_tokens; ++i) { ++ if (batch.seq_id[i][0] != id) continue; ++ const float * row = llama_get_embeddings_ith(ctx_tgt, i); ++ if (!row) throw std::runtime_error("decision model did not return hidden states"); ++ score_embeddings.emplace_back(row, row + width); ++ } + } + + bool need_embd() const { +@@ -748,9 +763,7 @@ + // returns 0 on success + // caller need to update prompt.tokens after a successful call to keep track of the processing progress + // note: this is not a member of server_slot because we want to run it inside yield_to_queue +-// slot is passed as const to avoid accidental modification of the slot state +-// some pointers are allowed to be used, they are not used by to_json() +-static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch, size_t idx, size_t & n_tokens_out) { ++static int process_mtmd_chunk(server_slot & slot, mtmd::batch_ptr & mbatch, size_t idx, size_t & n_tokens_out) { + GGML_ASSERT(slot.mctx); + const auto & mctx = slot.mctx; + const auto & input_tokens = slot.task->tokens; +@@ -761,12 +774,11 @@ + if (mbatch) { + float * embd = mtmd_batch_get_output_embd(mbatch.get(), chunk.get()); + if (embd) { +- void * cb_data = slot.spec; ++ void * cb_data = &slot; + static auto cb = [](llama_batch batch, void * user_data) { +- common_speculative * spec = static_cast(user_data); +- if (!common_speculative_process(spec, batch)) { +- return 1; +- } ++ auto & slot = *static_cast(user_data); ++ if (!common_speculative_process(slot.spec, batch)) return 1; ++ slot.collect_score_embeddings(batch); + return 0; + }; + +@@ -849,6 +861,7 @@ + // - when not in sleeping state + // - and, with thread-safe APIs (e.g., tokenizer calls) + llama_model * model_tgt = nullptr; ++ std::unique_ptr clef; + + mtmd_context * mctx = nullptr; + // note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct +@@ -2166,6 +2179,20 @@ + res->index = slot.task->index; + res->n_tokens = slot.task->n_tokens(); + res->res_type = slot.task->params.res_type; ++ ++ if (!slot.task->score_fields.is_null()) { ++ try { ++ if (!clef) clef.reset(new clef_head(params_base.model.path)); ++ llama_tokens tokens; ++ for (size_t i = 0; i < slot.task->tokens.size(); ++i) tokens.push_back(slot.task->tokens[i]); ++ res->score_logits = clef->score(slot.score_embeddings, tokens, slot.task->score_fields); ++ } catch (const std::exception & e) { ++ send_error(slot, e.what(), ERROR_TYPE_SERVER); ++ return; ++ } ++ queue_results.send(std::move(res)); ++ return; ++ } + + const int n_embd_out = llama_model_n_embd_out(model_tgt); + +@@ -3814,6 +3841,7 @@ + }; + + iterate(slots, [&](server_slot & slot) { ++ slot.collect_score_embeddings(batch_view); + // optionally send prompt processing progress + if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_DONE_PROMPT) { + if (slot.task->params.stream && slot.task->params.return_progress) { +@@ -4274,6 +4302,10 @@ + GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL); + + auto res = create_response(); ++ if (common_get_decision_type(ctx_server.model_tgt) == COMMON_DECISION_TYPE_CLEF) { ++ res->error(format_error_response("This model requires a decision request", ERROR_TYPE_INVALID_REQUEST)); ++ return res; ++ } + auto completion_id = gen_chatcmplid(); + auto & rd = res->rd; + auto & params = this->params; +@@ -5466,6 +5498,43 @@ + // OAI-compat + task.params.res_type = res_type; + task.params.embd_normalize = embd_normalize; ++ if (body.contains("score_fields")) { ++ task.score_fields = body.at("score_fields"); ++ task.params.cache_prompt = false; // the head needs every hidden state ++ if (body.contains("images") && !body.at("images").empty()) { ++ if (!ctx_server.mctx) throw std::invalid_argument("model does not support images"); ++ const auto & text = task.tokens.get_tokens(); ++ const int position = body.at("image_position").get(); ++ if (position < 0 || size_t(position) > text.size()) throw std::invalid_argument("invalid image position"); ++ std::string markers; ++ for (size_t j = 0; j < body.at("images").size(); ++j) markers += mtmd_get_marker(ctx_server.mctx); ++ auto media_inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, ++ json{{"prompt_string", markers + "\n"}, {"multimodal_data", body.at("images")}}, true, true, ctx_server.init_opt); ++ auto & media = media_inputs[0]; ++ if (text.size() + media.size() > size_t(meta->slot_n_ctx)) throw std::invalid_argument("decision prompt exceeds the model context (input is never truncated)"); ++ server_tokens input(llama_tokens(text.begin(), text.begin() + position), true); ++ for (size_t j = 0; j < media.size();) { ++ if (media[j] == LLAMA_TOKEN_NULL) { ++ const auto & chunk = media.find_chunk(j); ++ input.push_back(chunk.get()); ++ j += mtmd_input_chunk_get_n_tokens(chunk.get()); ++ } else { ++ input.push_back(media[j++]); ++ } ++ } ++ input.insert(llama_tokens(text.begin() + position, text.end())); ++ for (auto & field : task.score_fields) { ++ auto shift = [&](json & span) { ++ const int start = span.at(0).get(), end = span.at(1).get(); ++ if (start < position && end > position) throw std::invalid_argument("image splits a scoring span"); ++ if (start >= position) span = json::array({start + media.size(), end + media.size()}); ++ }; ++ shift(field.at("question")); ++ for (auto & option : field.at("options")) shift(option); ++ } ++ task.tokens = std::move(input); ++ } ++ } + + tasks.push_back(std::move(task)); + } +--- a/include/llama.h ++++ b/include/llama.h +@@ -224,6 +224,7 @@ + enum llama_context_type { + LLAMA_CONTEXT_TYPE_DEFAULT = 0, + LLAMA_CONTEXT_TYPE_MTP = 1, ++ LLAMA_CONTEXT_TYPE_DECISION = 2, // hidden states without vocabulary logits + }; + + // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) +--- a/src/llama-context.cpp ++++ b/src/llama-context.cpp +@@ -27,6 +27,7 @@ + + static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) { + switch (ctx_type) { ++ case LLAMA_CONTEXT_TYPE_DECISION: + case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT; + case LLAMA_CONTEXT_TYPE_MTP : return LLM_GRAPH_TYPE_DECODER_MTP; + } +@@ -2112,7 +2113,7 @@ + const auto n_embd = hparams.n_embd; + const auto n_embd_out = hparams.n_embd_out(); + +- bool has_logits = true; ++ bool has_logits = cparams.ctx_type != LLAMA_CONTEXT_TYPE_DECISION; + bool has_embd = cparams.embeddings; + bool has_embd_nextn = cparams.embeddings_nextn; + +--- a/src/models/qwen35.cpp ++++ b/src/models/qwen35.cpp +@@ -215,11 +215,12 @@ + cb(cur, "result_norm", -1); + res->t_embd = cur; + +- // LM head +- cur = build_lora_mm(model.output, cur, model.output_s); +- +- cb(cur, "result_output", -1); +- res->t_logits = cur; ++ // Decision heads consume the hidden states, not vocabulary logits. ++ if (cparams.ctx_type != LLAMA_CONTEXT_TYPE_DECISION) { ++ cur = build_lora_mm(model.output, cur, model.output_s); ++ cb(cur, "result_output", -1); ++ res->t_logits = cur; ++ } + + ggml_build_forward_expand(gf, cur); + } diff --git a/llama/compat/llama-ollama-compat.cpp b/llama/compat/llama-ollama-compat.cpp index 3e1cbddd4..848b84fa0 100644 --- a/llama/compat/llama-ollama-compat.cpp +++ b/llama/compat/llama-ollama-compat.cpp @@ -3242,6 +3242,12 @@ bool translate_metadata(const llama_model_loader * ml, std::lock_guard lk(g_loader_path_mutex); g_loader_paths[ml] = fname ? fname : ""; } + // Clef's joint head is consumed by the server after backbone evaluation. + const int decision_key = gguf_find_key(meta, (arch_name + ".decision.type").c_str()); + if (decision_key >= 0 && gguf_get_kv_type(meta, decision_key) == GGUF_TYPE_STRING && + std::strcmp(gguf_get_val_str(meta, decision_key), "clef") == 0) { + add_skip_prefix(ml, "clef."); + } // embeddinggemma must run before gemma3: it switches arch_name to // "gemma-embedding", which is what later checks (and the loader's KV // prefix) need to see. diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txt index 9153c9bf0..aabc1a28d 100644 --- a/llama/server/CMakeLists.txt +++ b/llama/server/CMakeLists.txt @@ -186,6 +186,8 @@ FetchContent_Declare( ${_ollama_compat_patch_cmd} ) FetchContent_MakeAvailable(llama_cpp) +target_sources(server-context PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../clef/clef.cpp) +target_include_directories(server-context PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../clef) if(NOT COMMAND license_add_file) message(FATAL_ERROR "llama.cpp license registry (cmake/license.cmake) not found; check the llama.cpp update") diff --git a/llm/llama_server_score.go b/llm/llama_server_score.go index b5d64a292..fc3474d06 100644 --- a/llm/llama_server_score.go +++ b/llm/llama_server_score.go @@ -9,6 +9,7 @@ import ( "math" "net/http" "slices" + "strings" "github.com/ollama/ollama/api" ) @@ -35,7 +36,7 @@ func (s *llamaServerRunner) Score(ctx context.Context, input ScoreRequest) (Scor badRequest := func(format string, args ...any) (ScoreResponse, error) { return ScoreResponse{}, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: fmt.Sprintf(format, args...)} } - if len(input.Rows) < 1 || len(input.Rows) > 64 { + if len(input.Segments) == 0 && len(input.Fields) == 0 && (len(input.Rows) < 1 || len(input.Rows) > 64) { return badRequest("scoring requires 1–64 prompts") } if input.MaxTokens < 1 || input.MaxTokens > s.ContextLength() { @@ -52,6 +53,10 @@ func (s *llamaServerRunner) Score(ctx context.Context, input ScoreRequest) (Scor return result, fmt.Errorf("unexpected server status: %s", status) } + if len(input.Segments) > 0 || len(input.Fields) > 0 { + return s.scoreFields(ctx, input) + } + rows := make([]scoreRowTokens, len(input.Rows)) parseSpecial, ordinary := true, false for i, row := range input.Rows { @@ -159,8 +164,8 @@ func (s *llamaServerRunner) primeSharedPrefix(ctx context.Context, rows []scoreR return 0, nil } tokens := first[:shared+scoreCheckpointOffset] - output, err := s.scoreCompletion(ctx, scoreCompletionRequest{Prompt: tokens, CachePrompt: true, NPredict: 0}) - if err != nil { + var output scoreCompletionResponse + if err := s.scoreRequest(ctx, "/completion", scoreCompletionRequest{Prompt: tokens, CachePrompt: true, NPredict: 0}, &output); err != nil { return 0, err } if output.Truncated || output.TokensEvaluated != len(tokens) { @@ -212,8 +217,8 @@ func (s *llamaServerRunner) scoreRow(ctx context.Context, row scoreRowTokens) ([ for _, id := range row.candidates { req.LogitBias = append(req.LogitBias, [2]float64{float64(id), bias}) } - output, err := s.scoreCompletion(ctx, req) - if err != nil { + var output scoreCompletionResponse + if err := s.scoreRequest(ctx, "/completion", req, &output); err != nil { return nil, 0, err } if output.Truncated || output.TokensEvaluated != len(row.tokens) || output.TokensPredicted != 1 || len(output.Probabilities) != 1 || len(output.Probabilities[0].TopProbs) == 0 { @@ -245,37 +250,121 @@ func (s *llamaServerRunner) scoreRow(ctx context.Context, row scoreRowTokens) ([ return nil, 0, fmt.Errorf("scoring candidates were outranked by other tokens") } -func (s *llamaServerRunner) scoreCompletion(ctx context.Context, input scoreCompletionRequest) (scoreCompletionResponse, error) { - var output scoreCompletionResponse +func (s *llamaServerRunner) scoreRequest(ctx context.Context, path string, input, output any) error { data, err := json.Marshal(input) if err != nil { - return output, err + return err } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/completion", s.port), bytes.NewReader(data)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d%s", s.port, path), bytes.NewReader(data)) if err != nil { - return output, err + return err } req.Header.Set("Content-Type", "application/json") res, err := s.httpClient().Do(req) if err != nil { if ctx.Err() != nil { - return output, ctx.Err() + return ctx.Err() } if msg := s.lastErrMsg(); msg != "" { - return output, fmt.Errorf("scoring failed: %s: %w", msg, err) + return fmt.Errorf("scoring failed: %s: %w", msg, err) } - return output, err + return err } defer res.Body.Close() body, err := io.ReadAll(res.Body) if err != nil { - return output, err + return err } if res.StatusCode != http.StatusOK { - return output, api.StatusError{StatusCode: res.StatusCode, ErrorMessage: s.statusErrorMessage(body)} + return api.StatusError{StatusCode: res.StatusCode, ErrorMessage: s.statusErrorMessage(body)} } - if err := json.Unmarshal(body, &output); err != nil { - return output, fmt.Errorf("invalid scoring response: %w", err) + if err := json.Unmarshal(body, output); err != nil { + return fmt.Errorf("invalid scoring response: %w", err) } - return output, nil + return nil +} + +func (s *llamaServerRunner) scoreFields(ctx context.Context, input ScoreRequest) (ScoreResponse, error) { + bad := func(message string) (ScoreResponse, error) { + return ScoreResponse{}, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: message} + } + if len(input.Fields) < 1 || len(input.Fields) > 64 || len(input.Segments) == 0 { + return bad("invalid decision scoring request") + } + var err error + // Do not concatenate before tokenizing: BPE may merge across the reference + // encoder's segment boundaries, changing both embeddings and span offsets. + var tokens []int + offsets := []int{0} + cache := map[string][]int{} + special := true + for _, segment := range input.Segments { + ids, ok := cache[segment] + if !ok { + ids, err = s.tokenize(ctx, segment, false, &special) + if err != nil { + return ScoreResponse{}, err + } + cache[segment] = ids + } + tokens = append(tokens, ids...) + offsets = append(offsets, len(tokens)) + if len(tokens) > input.MaxTokens { + return bad("decision prompt exceeds the model context (input is never truncated)") + } + } + if len(tokens) == 0 { + return bad("decision prompt must not be empty") + } + fields := make([]ScoreField, len(input.Fields)) + for i, f := range input.Fields { + if f.Type < 0 || f.Type > 2 || len(f.Options) < 2 || len(f.Options) > 26 { + return bad("invalid decision field") + } + // Map the question and its options from segment indexes to token offsets. + spans := append([][2]int{f.Question}, f.Options...) + for j, span := range spans { + start, end := span[0], span[1] + if start < 0 || start >= end || end >= len(offsets) || offsets[start] == offsets[end] { + return bad("invalid decision token span") + } + spans[j] = [2]int{offsets[start], offsets[end]} + } + fields[i] = ScoreField{Type: f.Type, Question: spans[0], Options: spans[1:]} + } + var images []api.ImageData + for _, image := range input.Images { + data, err := llamaServerMediaBytes(image) + if err != nil { + return bad(err.Error()) + } + if !strings.HasPrefix(http.DetectContentType(data), "image/") { + return bad("invalid decision image") + } + images = append(images, data) + } + imagePosition := 0 + if len(images) > 0 { + if input.ImagePosition < 0 || input.ImagePosition >= len(offsets) { + return bad("invalid image position") + } + imagePosition = offsets[input.ImagePosition] + } + request := struct { + Images []api.ImageData `json:"images,omitempty"` + ImagePosition int `json:"image_position"` + Input []int `json:"input"` + Fields []ScoreField `json:"score_fields"` + }{images, imagePosition, tokens, fields} + var result []struct { + Logits [][]float32 `json:"logits"` + Tokens int `json:"tokens_evaluated"` + } + if err := s.scoreRequest(ctx, "/embedding", request, &result); err != nil { + return ScoreResponse{}, err + } + if len(result) != 1 || result[0].Tokens > input.MaxTokens || (len(images) == 0 && result[0].Tokens != len(tokens)) || (len(images) > 0 && result[0].Tokens <= len(tokens)) || len(result[0].Logits) != len(fields) { + return ScoreResponse{}, fmt.Errorf("decision runner did not score the complete request") + } + return ScoreResponse{Logits: result[0].Logits, InputTokens: result[0].Tokens}, nil } diff --git a/llm/llama_server_score_test.go b/llm/llama_server_score_test.go index 931eca103..9995ee289 100644 --- a/llm/llama_server_score_test.go +++ b/llm/llama_server_score_test.go @@ -419,3 +419,127 @@ func TestLlamaServerScoreCancellation(t *testing.T) { }) } } + +func TestLlamaServerScoreFields(t *testing.T) { + for _, failure := range []string{"", "truncated", "missing head", "malformed JSON", "upstream error", "too long", "invalid span", "max exceeds context", "empty fields", "canceled", "image", "image over context", "invalid image", "invalid image position"} { + t.Run(failure, func(t *testing.T) { + calls := 0 + mux := http.NewServeMux() + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"ok"}`) }) + mux.HandleFunc("/tokenize", func(w http.ResponseWriter, r *http.Request) { + var input struct { + Content string `json:"content"` + AddSpecial bool `json:"add_special"` + ParseSpecial bool `json:"parse_special"` + } + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + t.Fatal(err) + } + if input.AddSpecial || !input.ParseSpecial { + t.Error("incorrect tokenizer settings") + } + ids := map[string][]int{"prefix": {1, 2}, "question": {3}, "one": {4, 5}, "two": {6}, "suffix": {7}}[input.Content] + if ids == nil { + t.Error("segments must be tokenized separately") + } + json.NewEncoder(w).Encode(map[string]any{"tokens": ids}) + }) + mux.HandleFunc("/embedding", func(w http.ResponseWriter, r *http.Request) { + calls++ + var input struct { + Input []int `json:"input"` + Fields []ScoreField `json:"score_fields"` + Images []api.ImageData `json:"images"` + ImagePosition int `json:"image_position"` + } + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(input.Input, []int{1, 2, 3, 4, 5, 6, 7}) || !reflect.DeepEqual(input.Fields, []ScoreField{{Type: 0, Question: [2]int{2, 3}, Options: [][2]int{{3, 5}, {5, 6}}}}) { + t.Errorf("wrong token spans: %+v", input) + } + switch failure { + case "image": + if len(input.Images) != 1 || input.ImagePosition != 2 { + t.Errorf("missing image or wrong image position: %+v", input) + } + fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":11}]`) + case "image over context": + fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":17}]`) + case "upstream error": + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error":{"message":"out of memory"}}`) + case "malformed JSON": + fmt.Fprint(w, `{`) + case "truncated": + fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":6}]`) + case "missing head": + fmt.Fprint(w, `[{"embedding":[[2,0]]}]`) + default: + fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":7}]`) + } + }) + srv := httptest.NewServer(mux) + defer srv.Close() + runner := &llamaServerRunner{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client(), cmd: fakeRunningCmd(), options: api.Options{Runner: api.Runner{NumCtx: 8}}, sem: semaphore.NewWeighted(1)} + input := ScoreRequest{MaxTokens: 8, Segments: []string{"prefix", "question", "one", "two", "suffix"}, Fields: []ScoreField{{Type: 0, Question: [2]int{1, 2}, Options: [][2]int{{2, 3}, {3, 4}}}}} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + invalid := true + switch failure { + case "image", "image over context", "invalid image", "invalid image position": + input.Images = []api.ImageData{{137, 80, 78, 71, 13, 10, 26, 10}} + input.ImagePosition = 1 + if failure == "invalid image" { + input.Images[0] = []byte("not an image") + } + if failure == "invalid image position" { + input.ImagePosition = 99 + } + if failure == "image" || failure == "image over context" { + invalid = false + input.MaxTokens = 16 + runner.options.NumCtx = 16 + } + case "too long": + input.MaxTokens = 6 + case "invalid span": + input.Fields[0].Question = [2]int{-1, 2} + case "max exceeds context": + input.MaxTokens = 9 + case "empty fields": + input.Fields = nil + case "canceled": + cancel() + default: + invalid = false + } + result, err := runner.Score(ctx, input) + if failure == "" || failure == "image" { + wantTokens := 7 + if failure == "image" { + wantTokens = 11 + } + if err != nil || result.InputTokens != wantTokens || result.OutputTokens != 0 { + t.Fatalf("result=%+v err=%v", result, err) + } + } else if err == nil { + t.Fatal("expected error") + } + var status api.StatusError + if failure == "upstream error" && (!errors.As(err, &status) || status.StatusCode != 500 || !IsOutOfMemory(err)) { + t.Fatalf("lost upstream status/OOM diagnostic: %v", err) + } + if failure == "canceled" && !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + if invalid && calls != 0 { + t.Fatal("invalid request reached the model") + } + if !runner.sem.TryAcquire(1) { + t.Fatal("scoring did not release the runner") + } + runner.sem.Release(1) + }) + } +} diff --git a/llm/score.go b/llm/score.go index e7d10ee76..a1bedb332 100644 --- a/llm/score.go +++ b/llm/score.go @@ -1,16 +1,25 @@ package llm -import "context" +import ( + "context" -// Scorer is an optional runner capability for next-token candidate scoring. -// Prompts are already rendered; no chat template or generation options apply. + "github.com/ollama/ollama/api" +) + +// Scorer is an optional runner capability for bounded decision scoring. +// Inputs are already encoded; no chat template or generation options apply. type Scorer interface { Score(context.Context, ScoreRequest) (ScoreResponse, error) } +// ScoreRequest uses either prompt Rows or Segments and Fields for a decision head. type ScoreRequest struct { - Rows []ScoreRow `json:"rows"` - MaxTokens int `json:"max_tokens"` + Rows []ScoreRow `json:"rows,omitempty"` + Segments []string `json:"segments,omitempty"` + Fields []ScoreField `json:"fields,omitempty"` + MaxTokens int `json:"max_tokens"` + Images []api.ImageData `json:"images,omitempty"` + ImagePosition int `json:"image_position,omitempty"` // Segment boundary at which images are inserted. } type ScoreRow struct { @@ -25,3 +34,12 @@ type ScoreResponse struct { InputTokens int `json:"input_tokens"` // Sum of complete prompt lengths, including shared tokens. OutputTokens int `json:"output_tokens,omitempty"` // Tokens generated internally to obtain candidate scores. } + +// ScoreField identifies a question and its allowed options for a decision head. +// Spans index ScoreRequest.Segments; the runner maps them to token offsets. +// Type is 0 for noul, 1 for choice, and 2 for score. +type ScoreField struct { + Type int `json:"type"` + Question [2]int `json:"question"` + Options [][2]int `json:"options"` +} diff --git a/server/capabilities_test.go b/server/capabilities_test.go index 2d934334f..8eb25c635 100644 --- a/server/capabilities_test.go +++ b/server/capabilities_test.go @@ -67,11 +67,34 @@ func TestCreateCapabilities(t *testing.T) { if !slices.Contains(shown.Capabilities, model.CapabilityDecision) || !strings.Contains(shown.Modelfile, "CAPABILITY decision\n") { t.Fatalf("show %s lost capability: %+v", name, shown) } + if want := []model.Capability{model.CapabilityDecision}; !slices.Equal(shown.Capabilities, want) || shown.Thinking != nil { + t.Fatalf("show %s capabilities = %v thinking = %v, want only %v", name, shown.Capabilities, shown.Thinking, want) + } + m, err := GetModel(name) + if err != nil { + t.Fatal(err) + } + if err := m.CheckCapabilities(model.CapabilityCompletion, model.CapabilityDecision); err != nil { + t.Fatalf("%s serving capabilities: %v", name, err) + } modelfile = shown.Modelfile if name == "roundtrip" { modelfile = "FROM roundtrip\n" } } + listed, err := listModels(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, m := range listed { + want := []model.Capability{model.CapabilityDecision} + if m.Name == "base:latest" { + want = []model.Capability{model.CapabilityCompletion, model.CapabilityVision} + } + if !slices.Equal(m.Capabilities, want) { + t.Fatalf("list %s capabilities = %v, want %v", m.Name, m.Capabilities, want) + } + } }) } } diff --git a/server/images.go b/server/images.go index 60c69347d..25c4a8538 100644 --- a/server/images.go +++ b/server/images.go @@ -164,6 +164,15 @@ func (m *Model) Capabilities() []model.Capability { return capabilities } +// publicCapabilities hides a decision model's other capabilities from show and +// list so clients don't offer it for general chat. Serving still uses Capabilities. +func publicCapabilities(capabilities []model.Capability) []model.Capability { + if slices.Contains(capabilities, model.CapabilityDecision) { + return []model.Capability{model.CapabilityDecision} + } + return capabilities +} + func (m *Model) capabilitiesForTemplate(source templateCapabilitySource) []model.Capability { capabilities := []model.Capability{} var modelArch string @@ -199,10 +208,13 @@ func (m *Model) ggufCapabilities(capabilities []model.Capability, source templat case templateCapabilityChat: capabilities = chatTemplateCapabilities(capabilities, m.metadata.String("tokenizer.chat_template")) } - if m.metadata.Valid("pooling_type") { + switch { + case m.metadata.String("decision.type") != "": + capabilities = appendCapability(capabilities, model.CapabilityDecision) + case m.metadata.Valid("pooling_type"): capabilities = appendCapability(capabilities, model.CapabilityEmbedding) - } else { - // If no embedding is specified, we assume the model supports completion. + default: + // Otherwise, assume the model supports completion. capabilities = appendCapability(capabilities, model.CapabilityCompletion) } if m.metadata.Valid("vision.block_count") { @@ -472,6 +484,12 @@ func (m *Model) modelFamilyCapabilities(capabilities []model.Capability) []model } func (m *Model) filterUnsupportedCapabilities(capabilities []model.Capability, modelArch string) []model.Capability { + if m.metadata.String("decision.type") != "" { + capabilities = slices.DeleteFunc(capabilities, func(c model.Capability) bool { + return c == model.CapabilityCompletion || c == model.CapabilityInsert || + c == model.CapabilityTools || c == model.CapabilityThinking + }) + } if suppressAudioCapability(m, modelArch) { capabilities = slices.DeleteFunc(capabilities, func(c model.Capability) bool { return c == model.CapabilityAudio diff --git a/server/images_test.go b/server/images_test.go index 3abcec24d..3c2ae88de 100644 --- a/server/images_test.go +++ b/server/images_test.go @@ -748,6 +748,12 @@ func TestPullModelManifestListDownloadsSelectedChildOnly(t *testing.T) { } func TestModelCapabilities(t *testing.T) { + decisionModelPath, _ := createBinFile(t, gguftest.KV{ + "general.architecture": "qwen35", + "qwen35.decision.type": "clef", + "tokenizer.chat_template": `{% if tools %}{{ tools }}{% endif %}{{ messages }}`, + }, []*gguftest.Tensor{}) + // Create completion model (llama architecture without vision) completionModelPath, _ := createBinFile(t, gguftest.KV{ "general.architecture": "llama", @@ -812,6 +818,22 @@ func TestModelCapabilities(t *testing.T) { model Model expectedCaps []model.Capability }{ + { + name: "Clef exposes decision instead of completion", + model: Model{ + ModelPath: decisionModelPath, + }, + expectedCaps: []model.Capability{model.CapabilityDecision}, + }, + { + name: "Clef filters inherited generation capabilities", + model: Model{ + ModelPath: decisionModelPath, + ProjectorPaths: []string{visionModelPath}, + Config: model.ConfigV2{Capabilities: []string{"decision", "vision", "completion", "insert", "tools", "thinking"}}, + }, + expectedCaps: []model.Capability{model.CapabilityDecision, model.CapabilityVision}, + }, { name: "model with image generation capability via config", model: Model{ diff --git a/server/model_list.go b/server/model_list.go index 83003bdfd..9097204b0 100644 --- a/server/model_list.go +++ b/server/model_list.go @@ -200,7 +200,7 @@ func describeModelFromManifest(name model.Name, mf *manifest.Manifest, runner st return summary, nil } summary.Details.ParentModel = m.ParentModel - summary.Capabilities = m.Capabilities() + summary.Capabilities = publicCapabilities(m.Capabilities()) if m.ModelPath != "" && m.isGGUF() { if summary.Details.ContextLength == 0 { diff --git a/server/routes.go b/server/routes.go index adb6b89f9..e42236847 100644 --- a/server/routes.go +++ b/server/routes.go @@ -894,29 +894,36 @@ func (s *Server) GenerateHandler(c *gin.Context) { func (s *Server) SystemOneHandler(c *gin.Context) { // TODO(parthsareen): Check token limits before copying state and schema into // each question's prompt. This byte cap limits memory use until then. - c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64<<10) + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 32<<20) + body, err := io.ReadAll(c.Request.Body) var req decision.Request - if err := c.ShouldBindJSON(&req); err != nil { + if err == nil { + err = json.Unmarshal(body, &req) + } + if err != nil { var sizeErr *http.MaxBytesError if errors.As(err, &sizeErr) { - c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "request body must not exceed 64 KiB"}) + c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "request body must not exceed 32 MiB"}) return } c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - compiled, err := decision.Compile(req) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + if len(req.Images) == 0 && len(body) > 64<<10 { + c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "request body must not exceed 64 KiB without images"}) return } ref, err := parseAndValidateModelRef(req.Model) if err != nil { + if errors.Is(err, errModelRequired) { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } writeModelRefParseError(c, err, http.StatusNotFound, fmt.Sprintf("model '%s' not found", req.Model)) return } if ref.Source == modelSourceCloud { - c.JSON(http.StatusBadRequest, gin.H{"error": "System One requires a local Nimble or Tev model"}) + c.JSON(http.StatusBadRequest, gin.H{"error": "System One requires a local decision model"}) return } name, err := getExistingName(ref.Name) @@ -933,14 +940,23 @@ func (s *Server) SystemOneHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q is not supported by System One; use a local GGUF model", req.Model)}) return } - r, _, _, err := s.scheduleRunner(c.Request.Context(), name.String(), "", []model.Capability{model.CapabilityCompletion, model.CapabilityDecision}, nil, req.KeepAlive, nil) + compiled, err := decision.CompileWithEncoder(req, m.metadata.String("decision.type")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + caps := []model.Capability{model.CapabilityDecision} + if len(req.Images) > 0 { + caps = append(caps, model.CapabilityVision) + } + r, _, _, err := s.scheduleRunner(c.Request.Context(), name.String(), "", caps, nil, req.KeepAlive, nil) if err != nil { handleScheduleError(c, req.Model, err) return } scorer, ok := r.(llm.Scorer) if !ok { - c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q does not support System One scoring; use a local Nimble or Tev model with a scoring-capable runner", req.Model)}) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q does not support System One scoring; use a local decision model with a scoring-capable runner", req.Model)}) return } if err := compiled.Render(func(messages []api.Message) (string, error) { @@ -1702,14 +1718,16 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) { Template: m.Template.String(), Details: modelDetails, Messages: msgs, - Capabilities: m.Capabilities(), - Thinking: m.Thinking(), + Capabilities: publicCapabilities(m.Capabilities()), ModifiedAt: mf.FileInfo().ModTime(), Requires: m.Config.Requires, // Several integrations crash on a nil/omitempty+empty ModelInfo, so by // default we return an empty map. ModelInfo: make(map[string]any), } + if !slices.Contains(resp.Capabilities, model.CapabilityDecision) { + resp.Thinking = m.Thinking() + } if summaries, err := manifestSummariesForShow(name, mf.SelectedDigest()); err != nil { return nil, err diff --git a/server/routes_create_test.go b/server/routes_create_test.go index 60d54f527..2aea11b4a 100644 --- a/server/routes_create_test.go +++ b/server/routes_create_test.go @@ -3444,3 +3444,28 @@ func TestCreateManifestListWarnsOnDrift(t *testing.T) { } }) } + +func TestCreateClefDecisionHead(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + _, digest := createBinFile(t, gguftest.KV{ + "general.architecture": "qwen35", + "qwen35.decision.type": "clef", + }, nil) + s := &Server{} + for _, req := range []api.CreateRequest{ + {Model: "custom-decision", Files: map[string]string{"model.gguf": digest}, Stream: &stream}, + {Model: "copied-decision", From: "custom-decision", Stream: &stream}, + } { + w := createRequest(t, s.CreateHandler, req) + if w.Code != http.StatusOK { + t.Fatalf("create: %d %s", w.Code, w.Body) + } + m, err := GetModel(req.Model) + if err != nil { + t.Fatal(err) + } + if m.metadata.String("decision.type") != "clef" || !slices.Contains(m.Capabilities(), model.CapabilityDecision) { + t.Fatalf("encoding/capability not preserved: %+v", m.Config) + } + } +} diff --git a/server/systemone_test.go b/server/systemone_test.go index e406a2a37..3d4b041fc 100644 --- a/server/systemone_test.go +++ b/server/systemone_test.go @@ -2,6 +2,7 @@ package server import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -31,6 +32,9 @@ type systemOneTestRunner struct { func (r *systemOneTestRunner) Score(ctx context.Context, input llm.ScoreRequest) (llm.ScoreResponse, error) { r.calls++ r.request = input + if len(input.Fields) > 0 { + return llm.ScoreResponse{Logits: [][]float32{{2, 0}}, InputTokens: 123}, r.err + } return llm.ScoreResponse{Logits: [][]float32{{0, 2}}, InputTokens: 123, OutputTokens: 2}, r.err } @@ -44,6 +48,7 @@ func TestSystemOneHandler(t *testing.T) { contextLength int undeclared bool }{ + {"renamed-clef", "qwen35", "", "Ignored for the joint head", "", 1024, false}, {"decision-test", "qwen35", "qwen3.5", "Model-specific scoring instructions.", "", 1024, false}, {"gguf-decision", "qwen35", "", "Native model scoring instructions.", "", 4096, false}, {"go-template", "qwen35", "", "Model-specific scoring instructions.", "custom:{{ range .Messages }}{{ .Role }}:{{ .Content }}\n{{ end }}answer:", 1024, false}, @@ -55,8 +60,14 @@ func TestSystemOneHandler(t *testing.T) { if modelConfig.template == "" { kv["tokenizer.chat_template"] = "{{ messages }}" } + if modelConfig.name == "renamed-clef" { + kv[modelConfig.architecture+".decision.type"] = "clef" + } _, digest := createBinFile(t, kv, nil) caps := []string{"completion", "decision"} + if modelConfig.name == "renamed-clef" { + caps = []string{"decision", "vision"} + } if modelConfig.undeclared { caps = []string{"completion"} } @@ -102,6 +113,13 @@ func TestSystemOneHandler(t *testing.T) { expire bool }{ {"success", `{"model":"decision-test","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false}, + {"Clef image above text body limit", `{"model":"renamed-clef","state":"x","images":["` + base64.StdEncoding.EncodeToString(make([]byte, 70<<10)) + `"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false}, + {"over image body limit", `{"model":"renamed-clef","images":["` + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false}, + {"images require a supported encoding", `{"model":"decision-test","state":"x","images":["aW1hZ2U="],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false}, + {"Clef videos unsupported", `{"model":"renamed-clef","state":"x","videos":["video.mp4"],"questions":{"refund":{"type":"noul"}}}`, nil, 400, 0, false}, + {"candidate videos unsupported", `{"model":"decision-test","state":"x","videos":["video.mp4"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false}, + {"Clef null state and default instructions", `{"model":"renamed-clef","state":null,"questions":{"refund":{"type":"noul"}}}`, nil, 200, 1, false}, + {"Clef joint head", `{"model":"renamed-clef","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false}, {"GGUF success without renderer", `{"model":"gguf-decision","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false}, {"Modelfile template", `{"model":"go-template","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false}, {"no system prompt", `{"model":"no-system","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false}, @@ -112,6 +130,9 @@ func TestSystemOneHandler(t *testing.T) { {"runner failure", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, errors.New("runner failed"), 500, 1, false}, {"runtime OOM", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, errors.New("out of memory"), 500, 1, true}, {"invalid schema", `{"model":"decision-test","state":"x","questions":{}}`, nil, 400, 0, false}, + {"invalid Clef schema", `{"model":"renamed-clef","state":"x","questions":{}}`, nil, 400, 0, false}, + {"omitted model", `{"state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false}, + {"blank model", `{"model":" ","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false}, {"missing model", `{"model":"missing","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 404, 0, false}, {"cloud", `{"model":"decision:cloud","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false}, {"safetensors unsupported", `{"model":"safetensors-decision","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false}, @@ -167,6 +188,9 @@ func TestSystemOneHandler(t *testing.T) { t.Fatalf("runner keep-alive=%v, want %v", duration, want) } } + if tt.name == "Clef image above text body limit" && (len(runner.request.Images) != 1 || len(runner.request.Images[0]) != 70<<10) { + t.Fatal("images were not passed to scoring") + } if tt.status == 200 { var response struct { Answers map[string]struct { @@ -177,9 +201,19 @@ func TestSystemOneHandler(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { t.Fatal(err) } - if response.Answers["refund"].Noul < 0.88 || response.Usage.InputTokens != 123 || response.Usage.OutputTokens != 2 { + outputTokens := 2 + if len(runner.request.Fields) > 0 { + outputTokens = 0 + } + if response.Answers["refund"].Noul < 0.88 || response.Usage.InputTokens != 123 || response.Usage.OutputTokens != outputTokens { t.Fatalf("incorrect scoring response: %s", w.Body) } + if len(runner.request.Fields) > 0 { + if len(runner.request.Rows) != 0 || len(runner.request.Fields) != 1 || runner.request.MaxTokens != 2048 { + t.Fatal("invalid joint scoring request") + } + return + } prompt := runner.request.Rows[0].Prompt wantContext := 1024 if ref.model.HasGoTemplate {