mlx: inkling follow up (#19806)

This commit is contained in:
Daniel Hiltgen 2026-11-20 19:03:47 -08:00 • committed by GitHub
parent cecac8bf1b
commit 23b94a2daf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 78 additions and 6 deletions

View file

@ -39,6 +39,7 @@ type Client struct {
modelName string
contextLength atomic.Int64
softContextLength int // recommended limit to avoid poor performance
numCtxAuto bool
memory atomic.Uint64
embeddingDimensions atomic.Value // []int
done chan struct{}
@ -54,7 +55,7 @@ var ErrRuntimeUnavailable = errors.New("MLX runtime is not available")
// NewClient prepares a new MLX runner client for LLM models.
// The subprocess is not started until Load() is called.
func NewClient(modelName string, softContextLength int) (*Client, error) {
func NewClient(modelName string, softContextLength int, numCtxAuto bool) (*Client, error) {
if err := CheckRuntime(); err != nil {
return nil, err
}
@ -62,6 +63,7 @@ func NewClient(modelName string, softContextLength int) (*Client, error) {
c := &Client{
modelName: modelName,
softContextLength: softContextLength,
numCtxAuto: numCtxAuto,
done: make(chan struct{}),
client: http.DefaultClient,
}
@ -318,6 +320,15 @@ func (c *Client) reportedContextLength(modelContextLength int) int {
return modelContextLength
}
func (c *Client) parallelContextHint() int {
// The VRAM-tier default is a soft limit. Only an explicitly configured
// context should reduce the per-slot memory projection.
if c.numCtxAuto {
return 0
}
return c.softContextLength
}
// Detokenize implements llm.LlamaServer.
func (c *Client) Detokenize(ctx context.Context, tokens []int) (string, error) {
ids32 := make([]int32, len(tokens))
@ -461,7 +472,7 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D
}
// Spawn subprocess: ollama runner --model <name> --port <port>
cmd := exec.Command(exe, "runner", "--model", c.modelName, "--port", strconv.Itoa(port), "--ctx-size", strconv.Itoa(c.softContextLength))
cmd := exec.Command(exe, "runner", "--model", c.modelName, "--port", strconv.Itoa(port), "--ctx-size", strconv.Itoa(c.parallelContextHint()))
cmd.Env = os.Environ()
loadedLibraryPath, err := mlx.LoadedLibraryPath()
if err != nil {

View file

@ -6,6 +6,28 @@ import (
"github.com/ollama/ollama/mlx/mlxtest"
)
func TestParallelContextUsesExplicitNumCtx(t *testing.T) {
const trained = 1 << 20
for _, tc := range []struct {
name string
auto bool
want int
}{
{"VRAM tier", true, 256 << 10},
{"explicit", false, 4096},
} {
t.Run(tc.name, func(t *testing.T) {
c := Client{softContextLength: 4096, numCtxAuto: tc.auto}
if got := parallelContextLength(trained, c.parallelContextHint()); got != tc.want {
t.Errorf("projected context = %d, want %d", got, tc.want)
}
if got := c.reportedContextLength(trained); got != 4096 {
t.Errorf("reported context = %d, want 4096", got)
}
})
}
}
func TestParallelContextLength(t *testing.T) {
for _, tc := range []struct {
name string

View file

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"math"
"strings"
@ -149,8 +150,17 @@ func (p *inkling2Parser) Add(s string, done bool) (string, string, []api.ToolCal
// Only search for a terminator after the tool call object.
tokens = []string{inklingEnd, inklingEndSample}
searchFrom = findJSONEnd(p.buffer) + 1
if searchFrom == 0 && !done {
break
if searchFrom == 0 {
if !done {
break
}
// A token limit can end the stream inside a valid tool call.
var raw json.RawMessage
err := json.NewDecoder(strings.NewReader(p.buffer)).Decode(&raw)
if err == io.EOF || (err == io.ErrUnexpectedEOF && strings.HasPrefix(strings.TrimSpace(p.buffer), "{")) {
p.buffer = ""
break
}
}
}
marker, index := "", len(p.buffer)

View file

@ -98,6 +98,36 @@ func TestInkling2ParserChunkBoundaries(t *testing.T) {
}
}
func TestInkling2TruncatedToolCall(t *testing.T) {
const complete = inklingModel + "save" + inklingTool + `{"name":"save","args":{}}` + inklingEnd
const header = inklingModel + "lookup" + inklingTool
const payload = `{"name":"lookup","args":{"text":"before <|end_message|> after","id":[17]}}`
for end := range len(payload) + 1 {
p := inkling2Parser{}
_, _, calls, err := p.Add(complete+header+payload[:end], false)
if err != nil || len(calls) != 1 || calls[0].Function.Name != "save" {
t.Fatalf("offset %d before done: calls %v, error %v", end, calls, err)
}
content, thinking, calls, err := p.Add("", true)
if err != nil || content != "" || thinking != "" {
t.Fatalf("offset %d on done: content %q, thinking %q, error %v", end, content, thinking, err)
}
if end == len(payload) {
if len(calls) != 1 || calls[0].Function.Name != "lookup" {
t.Fatalf("complete call: %v", calls)
}
} else if len(calls) != 0 {
t.Fatalf("offset %d emitted a truncated call: %v", end, calls)
}
}
for _, body := range []string{`{"name":!}`, `{"name":!`, `{"name":` + inklingEnd, `{}`, `tru`, `["`} {
p := inkling2Parser{}
if _, _, _, err := p.Add(header+body, true); err == nil {
t.Errorf("expected error for malformed call %q", body)
}
}
}
func TestInkling2ParserJinjaToolCalls(t *testing.T) {
// The Jinja template leaves literal control spellings in JSON strings and
// accepts an empty string as a function name.

View file

@ -628,8 +628,7 @@ func (s *Scheduler) load(req *LlmRequest, systemInfo ml.SystemInfo, gpus []ml.De
}
}
} else {
modelName := req.model.ShortName
llama, err = mlxrunner.NewClient(modelName, req.opts.NumCtx)
llama, err = mlxrunner.NewClient(req.model.ShortName, req.opts.NumCtx, req.numCtxAuto)
}
if err != nil {
slog.Info("failed to create server", "model", req.model.ShortName, "error", err)