mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
mlx: inkling follow up (#19806)
This commit is contained in:
parent
cecac8bf1b
commit
23b94a2daf
5 changed files with 78 additions and 6 deletions
|
|
@ -39,6 +39,7 @@ type Client struct {
|
|||
modelName string
|
||||
contextLength atomic.Int64
|
||||
softContextLength int // recommended limit to avoid poor performance
|
||||
numCtxAuto bool
|
||||
memory atomic.Uint64
|
||||
embeddingDimensions atomic.Value // []int
|
||||
done chan struct{}
|
||||
|
|
@ -54,7 +55,7 @@ var ErrRuntimeUnavailable = errors.New("MLX runtime is not available")
|
|||
|
||||
// NewClient prepares a new MLX runner client for LLM models.
|
||||
// The subprocess is not started until Load() is called.
|
||||
func NewClient(modelName string, softContextLength int) (*Client, error) {
|
||||
func NewClient(modelName string, softContextLength int, numCtxAuto bool) (*Client, error) {
|
||||
if err := CheckRuntime(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -62,6 +63,7 @@ func NewClient(modelName string, softContextLength int) (*Client, error) {
|
|||
c := &Client{
|
||||
modelName: modelName,
|
||||
softContextLength: softContextLength,
|
||||
numCtxAuto: numCtxAuto,
|
||||
done: make(chan struct{}),
|
||||
client: http.DefaultClient,
|
||||
}
|
||||
|
|
@ -318,6 +320,15 @@ func (c *Client) reportedContextLength(modelContextLength int) int {
|
|||
return modelContextLength
|
||||
}
|
||||
|
||||
func (c *Client) parallelContextHint() int {
|
||||
// The VRAM-tier default is a soft limit. Only an explicitly configured
|
||||
// context should reduce the per-slot memory projection.
|
||||
if c.numCtxAuto {
|
||||
return 0
|
||||
}
|
||||
return c.softContextLength
|
||||
}
|
||||
|
||||
// Detokenize implements llm.LlamaServer.
|
||||
func (c *Client) Detokenize(ctx context.Context, tokens []int) (string, error) {
|
||||
ids32 := make([]int32, len(tokens))
|
||||
|
|
@ -461,7 +472,7 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D
|
|||
}
|
||||
|
||||
// Spawn subprocess: ollama runner --model <name> --port <port>
|
||||
cmd := exec.Command(exe, "runner", "--model", c.modelName, "--port", strconv.Itoa(port), "--ctx-size", strconv.Itoa(c.softContextLength))
|
||||
cmd := exec.Command(exe, "runner", "--model", c.modelName, "--port", strconv.Itoa(port), "--ctx-size", strconv.Itoa(c.parallelContextHint()))
|
||||
cmd.Env = os.Environ()
|
||||
loadedLibraryPath, err := mlx.LoadedLibraryPath()
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,28 @@ import (
|
|||
"github.com/ollama/ollama/mlx/mlxtest"
|
||||
)
|
||||
|
||||
func TestParallelContextUsesExplicitNumCtx(t *testing.T) {
|
||||
const trained = 1 << 20
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
auto bool
|
||||
want int
|
||||
}{
|
||||
{"VRAM tier", true, 256 << 10},
|
||||
{"explicit", false, 4096},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := Client{softContextLength: 4096, numCtxAuto: tc.auto}
|
||||
if got := parallelContextLength(trained, c.parallelContextHint()); got != tc.want {
|
||||
t.Errorf("projected context = %d, want %d", got, tc.want)
|
||||
}
|
||||
if got := c.reportedContextLength(trained); got != 4096 {
|
||||
t.Errorf("reported context = %d, want 4096", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParallelContextLength(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
|
|
@ -149,8 +150,17 @@ func (p *inkling2Parser) Add(s string, done bool) (string, string, []api.ToolCal
|
|||
// Only search for a terminator after the tool call object.
|
||||
tokens = []string{inklingEnd, inklingEndSample}
|
||||
searchFrom = findJSONEnd(p.buffer) + 1
|
||||
if searchFrom == 0 && !done {
|
||||
break
|
||||
if searchFrom == 0 {
|
||||
if !done {
|
||||
break
|
||||
}
|
||||
// A token limit can end the stream inside a valid tool call.
|
||||
var raw json.RawMessage
|
||||
err := json.NewDecoder(strings.NewReader(p.buffer)).Decode(&raw)
|
||||
if err == io.EOF || (err == io.ErrUnexpectedEOF && strings.HasPrefix(strings.TrimSpace(p.buffer), "{")) {
|
||||
p.buffer = ""
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
marker, index := "", len(p.buffer)
|
||||
|
|
|
|||
|
|
@ -98,6 +98,36 @@ func TestInkling2ParserChunkBoundaries(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestInkling2TruncatedToolCall(t *testing.T) {
|
||||
const complete = inklingModel + "save" + inklingTool + `{"name":"save","args":{}}` + inklingEnd
|
||||
const header = inklingModel + "lookup" + inklingTool
|
||||
const payload = `{"name":"lookup","args":{"text":"before <|end_message|> after","id":[17]}}`
|
||||
for end := range len(payload) + 1 {
|
||||
p := inkling2Parser{}
|
||||
_, _, calls, err := p.Add(complete+header+payload[:end], false)
|
||||
if err != nil || len(calls) != 1 || calls[0].Function.Name != "save" {
|
||||
t.Fatalf("offset %d before done: calls %v, error %v", end, calls, err)
|
||||
}
|
||||
content, thinking, calls, err := p.Add("", true)
|
||||
if err != nil || content != "" || thinking != "" {
|
||||
t.Fatalf("offset %d on done: content %q, thinking %q, error %v", end, content, thinking, err)
|
||||
}
|
||||
if end == len(payload) {
|
||||
if len(calls) != 1 || calls[0].Function.Name != "lookup" {
|
||||
t.Fatalf("complete call: %v", calls)
|
||||
}
|
||||
} else if len(calls) != 0 {
|
||||
t.Fatalf("offset %d emitted a truncated call: %v", end, calls)
|
||||
}
|
||||
}
|
||||
for _, body := range []string{`{"name":!}`, `{"name":!`, `{"name":` + inklingEnd, `{}`, `tru`, `["`} {
|
||||
p := inkling2Parser{}
|
||||
if _, _, _, err := p.Add(header+body, true); err == nil {
|
||||
t.Errorf("expected error for malformed call %q", body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInkling2ParserJinjaToolCalls(t *testing.T) {
|
||||
// The Jinja template leaves literal control spellings in JSON strings and
|
||||
// accepts an empty string as a function name.
|
||||
|
|
|
|||
|
|
@ -628,8 +628,7 @@ func (s *Scheduler) load(req *LlmRequest, systemInfo ml.SystemInfo, gpus []ml.De
|
|||
}
|
||||
}
|
||||
} else {
|
||||
modelName := req.model.ShortName
|
||||
llama, err = mlxrunner.NewClient(modelName, req.opts.NumCtx)
|
||||
llama, err = mlxrunner.NewClient(req.model.ShortName, req.opts.NumCtx, req.numCtxAuto)
|
||||
}
|
||||
if err != nil {
|
||||
slog.Info("failed to create server", "model", req.model.ShortName, "error", err)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue