diff --git a/mlxrunner/client.go b/mlxrunner/client.go index 405bd870f..50b9d12e0 100644 --- a/mlxrunner/client.go +++ b/mlxrunner/client.go @@ -39,6 +39,7 @@ type Client struct { modelName string contextLength atomic.Int64 softContextLength int // recommended limit to avoid poor performance + numCtxAuto bool memory atomic.Uint64 embeddingDimensions atomic.Value // []int done chan struct{} @@ -54,7 +55,7 @@ var ErrRuntimeUnavailable = errors.New("MLX runtime is not available") // NewClient prepares a new MLX runner client for LLM models. // The subprocess is not started until Load() is called. -func NewClient(modelName string, softContextLength int) (*Client, error) { +func NewClient(modelName string, softContextLength int, numCtxAuto bool) (*Client, error) { if err := CheckRuntime(); err != nil { return nil, err } @@ -62,6 +63,7 @@ func NewClient(modelName string, softContextLength int) (*Client, error) { c := &Client{ modelName: modelName, softContextLength: softContextLength, + numCtxAuto: numCtxAuto, done: make(chan struct{}), client: http.DefaultClient, } @@ -318,6 +320,15 @@ func (c *Client) reportedContextLength(modelContextLength int) int { return modelContextLength } +func (c *Client) parallelContextHint() int { + // The VRAM-tier default is a soft limit. Only an explicitly configured + // context should reduce the per-slot memory projection. + if c.numCtxAuto { + return 0 + } + return c.softContextLength +} + // Detokenize implements llm.LlamaServer. func (c *Client) Detokenize(ctx context.Context, tokens []int) (string, error) { ids32 := make([]int32, len(tokens)) @@ -461,7 +472,7 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D } // Spawn subprocess: ollama runner --model --port - cmd := exec.Command(exe, "runner", "--model", c.modelName, "--port", strconv.Itoa(port), "--ctx-size", strconv.Itoa(c.softContextLength)) + cmd := exec.Command(exe, "runner", "--model", c.modelName, "--port", strconv.Itoa(port), "--ctx-size", strconv.Itoa(c.parallelContextHint())) cmd.Env = os.Environ() loadedLibraryPath, err := mlx.LoadedLibraryPath() if err != nil { diff --git a/mlxrunner/parallel_test.go b/mlxrunner/parallel_test.go index c21c6d8d3..de5908950 100644 --- a/mlxrunner/parallel_test.go +++ b/mlxrunner/parallel_test.go @@ -6,6 +6,28 @@ import ( "github.com/ollama/ollama/mlx/mlxtest" ) +func TestParallelContextUsesExplicitNumCtx(t *testing.T) { + const trained = 1 << 20 + for _, tc := range []struct { + name string + auto bool + want int + }{ + {"VRAM tier", true, 256 << 10}, + {"explicit", false, 4096}, + } { + t.Run(tc.name, func(t *testing.T) { + c := Client{softContextLength: 4096, numCtxAuto: tc.auto} + if got := parallelContextLength(trained, c.parallelContextHint()); got != tc.want { + t.Errorf("projected context = %d, want %d", got, tc.want) + } + if got := c.reportedContextLength(trained); got != 4096 { + t.Errorf("reported context = %d, want 4096", got) + } + }) + } +} + func TestParallelContextLength(t *testing.T) { for _, tc := range []struct { name string diff --git a/model/parsers/inkling2.go b/model/parsers/inkling2.go index 689881ee9..b1a017714 100644 --- a/model/parsers/inkling2.go +++ b/model/parsers/inkling2.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "math" "strings" @@ -149,8 +150,17 @@ func (p *inkling2Parser) Add(s string, done bool) (string, string, []api.ToolCal // Only search for a terminator after the tool call object. tokens = []string{inklingEnd, inklingEndSample} searchFrom = findJSONEnd(p.buffer) + 1 - if searchFrom == 0 && !done { - break + if searchFrom == 0 { + if !done { + break + } + // A token limit can end the stream inside a valid tool call. + var raw json.RawMessage + err := json.NewDecoder(strings.NewReader(p.buffer)).Decode(&raw) + if err == io.EOF || (err == io.ErrUnexpectedEOF && strings.HasPrefix(strings.TrimSpace(p.buffer), "{")) { + p.buffer = "" + break + } } } marker, index := "", len(p.buffer) diff --git a/model/parsers/inkling2_test.go b/model/parsers/inkling2_test.go index 802599556..5ef33da52 100644 --- a/model/parsers/inkling2_test.go +++ b/model/parsers/inkling2_test.go @@ -98,6 +98,36 @@ func TestInkling2ParserChunkBoundaries(t *testing.T) { } } +func TestInkling2TruncatedToolCall(t *testing.T) { + const complete = inklingModel + "save" + inklingTool + `{"name":"save","args":{}}` + inklingEnd + const header = inklingModel + "lookup" + inklingTool + const payload = `{"name":"lookup","args":{"text":"before <|end_message|> after","id":[17]}}` + for end := range len(payload) + 1 { + p := inkling2Parser{} + _, _, calls, err := p.Add(complete+header+payload[:end], false) + if err != nil || len(calls) != 1 || calls[0].Function.Name != "save" { + t.Fatalf("offset %d before done: calls %v, error %v", end, calls, err) + } + content, thinking, calls, err := p.Add("", true) + if err != nil || content != "" || thinking != "" { + t.Fatalf("offset %d on done: content %q, thinking %q, error %v", end, content, thinking, err) + } + if end == len(payload) { + if len(calls) != 1 || calls[0].Function.Name != "lookup" { + t.Fatalf("complete call: %v", calls) + } + } else if len(calls) != 0 { + t.Fatalf("offset %d emitted a truncated call: %v", end, calls) + } + } + for _, body := range []string{`{"name":!}`, `{"name":!`, `{"name":` + inklingEnd, `{}`, `tru`, `["`} { + p := inkling2Parser{} + if _, _, _, err := p.Add(header+body, true); err == nil { + t.Errorf("expected error for malformed call %q", body) + } + } +} + func TestInkling2ParserJinjaToolCalls(t *testing.T) { // The Jinja template leaves literal control spellings in JSON strings and // accepts an empty string as a function name. diff --git a/server/sched.go b/server/sched.go index d8f5a10e7..c60d43e5e 100644 --- a/server/sched.go +++ b/server/sched.go @@ -628,8 +628,7 @@ func (s *Scheduler) load(req *LlmRequest, systemInfo ml.SystemInfo, gpus []ml.De } } } else { - modelName := req.model.ShortName - llama, err = mlxrunner.NewClient(modelName, req.opts.NumCtx) + llama, err = mlxrunner.NewClient(req.model.ShortName, req.opts.NumCtx, req.numCtxAuto) } if err != nil { slog.Info("failed to create server", "model", req.model.ShortName, "error", err)