ollama/mlxrunner/runner.go
Daniel Hiltgen cecac8bf1b
mlxrunner: add Inkling 2 Mini (#19799)
* mlxrunner: add Inkling 2 Mini

* mlx: fuse Inkling 2 Mini residual mixing on Metal

Fuse RMS normalization and query scoring in one kernel. Apply the
softmax weights in a second kernel to avoid storing per-query products.

* create: expose Inkling 2 Mini context length in model metadata

Copy model_max_length to max_position_embeddings during import so
ollama show reports the checkpoint's context length. Reject configs
that specify conflicting limits.

* mlxrunner: reduce Metal dispatches in Inkling 2 Mini decode

Reuse the prefill mask shader to compute relative position bias in one
dispatch for single-query attention. On M3 Ultra with Metal 3, compute
the loop controller's gain and forget projections in one matrix multiply.
2026-11-20 18:38:44 -08:00

448 lines
13 KiB
Go

package mlxrunner
import (
"context"
"errors"
"log/slog"
"maps"
"math"
"net"
"net/http"
"slices"
"strings"
"golang.org/x/sync/errgroup"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/llm"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlx/mlxthread"
"github.com/ollama/ollama/mlxrunner/batch"
"github.com/ollama/ollama/mlxrunner/cache"
"github.com/ollama/ollama/mlxrunner/model"
_ "github.com/ollama/ollama/mlxrunner/model/architectures"
"github.com/ollama/ollama/mlxrunner/sample"
"github.com/ollama/ollama/mlxrunner/tokenizer"
)
// request is a short-lived struct that carries a completion request through
// a channel from the HTTP handler to the runner goroutine. The ctx field
// must travel with the request so that cancellation propagates across the
// channel boundary.
type request struct {
CompletionRequest
Responses chan CompletionResponse
Ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner.
Tokens []int32
MediaItems []mediaItem
Layout any // opaque PrepareMedia layout state, stamped on every batch
SamplerOpts sample.Options
Grammar *grammarCompilation
}
func (r request) context() context.Context { return r.Ctx }
func (r request) reject(err error) {
r.Grammar.close()
if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
// An unadmitted request has no queued responses, so the production
// buffer accepts this error without blocking scheduler teardown.
select {
case r.Responses <- CompletionResponse{Error: completionStatusError(err)}:
default:
}
}
close(r.Responses)
}
func completionStatusError(err error) *api.StatusError {
var statusErr api.StatusError
if !errors.As(err, &statusErr) {
statusErr = api.StatusError{
StatusCode: http.StatusInternalServerError,
ErrorMessage: err.Error(),
}
}
return &statusErr
}
// An evaluation computes embeddings or decision scores from the supplied input without
// entering the token decode loop. Its callback runs on the MLX worker.
type evaluationRequest struct {
ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner.
tokens []int32
items []mediaItem
open func(*batchSlot, []*prefixCache) (*evaluationState, error)
done chan error
}
func (r *evaluationRequest) reject(err error) { r.done <- err }
func (r *evaluationRequest) context() context.Context { return r.ctx }
type speechRequest struct {
ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner.
input llm.SpeechRequest
state model.SpeechState
queue *audioQueue
done chan error
}
func (r *speechRequest) reject(err error) { r.done <- err }
func (r *speechRequest) context() context.Context { return r.ctx }
// All request types share admission and the same slot limit. Once admitted,
// the scheduler advances each type using its own model operations.
type queuedRequest interface {
context() context.Context
// reject releases a request that has not acquired a slot.
reject(error)
}
type Runner struct {
parallel int
parallelContext int
fullContextBytes int64 // Estimated per-client storage at the projected context size.
Model model.Model
weights *mlx.Scope
Tokenizer *tokenizer.Tokenizer
Requests chan queuedRequest
Sampler *sample.Sampler
cache *prefixCache
contextLength int
mlxThread *mlxthread.Thread
// grammarEngine is the structured-output subsystem; nil when the grammar
// library or vocabulary failed to load.
grammarEngine *grammarEngine
// spec is the speculative-decoding subsystem. Nil when the model ships no
// draft head.
spec *speculation
// Last retention observation for new slots; accessed on the MLX worker.
snapshotLimitBytes int64
}
func (r *Runner) Load(modelName string) error {
weights, err := r.loadModel(modelName)
if err != nil {
return err
}
mlx.Eval(weights...)
mlx.ClearCache()
r.weights = mlx.NewScope()
r.weights.Attach(weights...)
configureWiredMemory()
r.configureParallelism()
return nil
}
func (r *Runner) loadModel(modelName string) (weights []*mlx.Array, err error) {
weights = mlx.ScopedArrays(func() []*mlx.Array {
root, e := model.Open(modelName)
if e != nil {
err = e
return nil
}
m, e := model.New(root)
if e != nil {
err = e
return nil
}
var skippedTensors []string
if s, ok := m.(model.SkippedTensorsModel); ok {
skippedTensors = s.SkippedTensorNames()
}
tensors, e := loadTensorsFromManifest(root, skippedTensors)
if e != nil {
err = e
return nil
}
// On Metal, materialize the loaded tensors with CPU reads before any
// weight graph exists, so the weight eval never commits a command buffer
// that waits on file data. CUDA loads read at dispatch and need no pre-pass.
if mlx.MetalIsAvailable() {
mlx.Eval(slices.Collect(maps.Values(tensors))...)
}
// Assign weights to model (model-specific logic). Target and draft weights
// must be loaded before the load scope ends so tensors from a combined
// manifest are not discarded before the draft model can retain them.
if err = m.LoadWeights(tensors); err != nil {
return nil
}
var draftModel model.DraftModel
draft, e := model.NewDraft(root, m)
if e != nil {
err = e
return nil
}
if draft != nil {
if err = draft.LoadWeights(tensors); err != nil {
return nil
}
draftModel = draft
} else if sd, ok := m.(model.SelfDraft); ok {
// Inline draft head: already loaded with the target; nil if none shipped.
draftModel = sd.SelfDraft()
}
w := mlx.Collect(m)
if draft != nil {
draftArrays := mlx.Collect(draft)
w = append(w, draftArrays...)
if root.Draft != nil {
slog.Info("Loaded draft model", "tensor_prefix", root.Draft.TensorPrefix, "config", root.Draft.Config, "arrays", len(draftArrays))
} else {
slog.Info("Loaded draft model", "arrays", len(draftArrays))
}
}
r.Model = m
mlx.EnableCompile()
// Speech-only models have no tokenizer; models that also chat need
// the text pipeline.
if _, ok := m.(model.SpeechModel); ok && m.Tokenizer() == nil {
return w
}
r.Tokenizer = m.Tokenizer()
r.contextLength = m.MaxContextLength()
// Decision and embedding models read hidden states directly. They do
// not need a text sampler or a synthetic generation forward for grammar.
if _, ok := m.(model.Scorer); ok {
return w
}
if _, ok := m.(interface{ EmbeddingDim() int }); ok {
return w
}
caches := m.NewCaches()
draftCaches := newDraftCaches(draftModel)
r.cache = newPrefixCache(slices.Concat(caches, draftCaches))
r.Sampler = sample.New(r.contextLength)
r.spec = newSpeculation(r, pipelineSlot, draftModel, caches, draftCaches)
var capacity int
if r.parallel <= 0 && mlx.MetalIsAvailable() {
capacity = parallelContextLength(r.contextLength, r.parallelContext)
}
width, fullContextBytes := modelShapes(m, draftModel, capacity)
r.fullContextBytes = fullContextBytes
r.grammarEngine = newGrammarEngine(width, r.Tokenizer)
return w
})
return weights, err
}
func (r *Runner) Close() {
r.cache.close()
r.cache = nil
if r.grammarEngine != nil {
r.grammarEngine.close()
r.grammarEngine = nil
}
r.weights.Close()
r.weights = nil
}
// newDraftCaches returns nil when the model ships no draft.
func newDraftCaches(draft model.DraftModel) []cache.Cache {
if draft == nil {
return nil
}
return draft.NewCaches()
}
// modelShapes inspects the existing one-token load graph without evaluating it.
// Cache dimensions come from the same forward used for inference, including the
// drafter, at the projected context size.
func modelShapes(m model.Model, draft model.DraftModel, tokens int) (width int, fullContextBytes int64) {
mlx.Scoped(func() {
caches := m.NewCaches()
var draftCaches []cache.Cache
if tokens > 0 {
draftCaches = newDraftCaches(draft)
}
defer func() {
for _, c := range slices.Concat(caches, draftCaches) {
if c != nil {
c.Free()
}
}
}()
b := &batch.Batch{
InputIDs: mlx.FromValues([]int32{0}, 1, 1),
SeqOffsets: []int32{0},
SeqQueryLens: []int32{1},
}
hidden, aux := m.Forward(b, cache.Layers(caches))
logits := m.Unembed(hidden)
width = logits.Dim(logits.NumDims() - 1)
if tokens <= 0 {
return
}
if len(draftCaches) > 0 {
if aux == nil {
return
}
b.Hidden = aux
draft.Forward(b, cache.Layers(caches), cache.Layers(draftCaches))
}
checkpoints := (tokens + prefillSnapshotInterval - 1) / prefillSnapshotInterval
for _, c := range caches {
bytes, ok := cache.EstimateMemory(c, tokens, checkpoints)
if !ok || bytes > math.MaxInt64-fullContextBytes {
fullContextBytes = 0
return
}
fullContextBytes += bytes
}
if fullContextBytes == 0 {
return
}
// Scoring keeps the target's hidden outputs and briefly holds both
// sides of a growing concatenation. It does not run the drafter.
hiddenBytes := int64(hidden.NumBytes())
if hiddenBytes > (math.MaxInt64-fullContextBytes)/int64(tokens)/2 {
fullContextBytes = 0
return
}
scoreBytes := fullContextBytes + 2*hiddenBytes*int64(tokens)
for _, c := range draftCaches {
bytes, ok := cache.EstimateMemory(c, tokens, checkpoints)
if !ok || bytes > math.MaxInt64-fullContextBytes {
fullContextBytes = 0
return
}
fullContextBytes += bytes
}
fullContextBytes = max(fullContextBytes, scoreBytes)
})
return width, fullContextBytes
}
func configureWiredMemory() {
if !mlx.GPUIsAvailable() {
return
}
active := mlx.ActiveMemory()
maxRecommended, err := mlx.MaxRecommendedWorkingSetSize()
if err != nil {
slog.Warn("Unable to query MLX recommended working set; using pageable memory", "error", err)
return
}
limit := min(active, maxRecommended)
previous, err := mlx.SetWiredLimit(limit)
if err != nil {
slog.Warn("Unable to configure MLX wired memory; using pageable memory",
"active", mlx.PrettyBytes(active),
"limit", mlx.PrettyBytes(limit),
"error", err)
return
}
if active > maxRecommended {
slog.Warn("MLX model exceeds the recommended working set; performance may be degraded",
"active", mlx.PrettyBytes(active),
"recommended", mlx.PrettyBytes(maxRecommended))
}
// Limiting residency to the loaded model's active allocations avoids
// reserving the remaining capacity for growing KV caches.
slog.Debug("Configured MLX wired memory",
"active", mlx.PrettyBytes(active),
"limit", mlx.PrettyBytes(limit),
"previous", mlx.PrettyBytes(previous))
}
// loadTensorsFromManifest loads tensor blobs into a flat map, excluding specified
// tensors, deduplicating by digest and remapping key suffixes.
//
// Uses a two-phase approach: first loads all raw tensors, then remaps
// .bias → _qbias with complete knowledge of which base names have .scale
// entries. This avoids a race condition where Go map iteration order could
// cause .bias to be processed before .scale within the same blob.
func loadTensorsFromManifest(root *model.Root, skippedTensors []string) (map[string]*mlx.Array, error) {
skip := make(map[string]bool, len(skippedTensors))
for _, name := range skippedTensors {
skip[name] = true
}
skipped, skippedBytes := 0, 0
// Phase 1: Load all tensors raw from all blobs
rawTensors := make(map[string]*mlx.Array)
seen := make(map[string]bool)
for _, layer := range root.Manifest.TensorLayers() {
if seen[layer.Digest] {
continue
}
seen[layer.Digest] = true
blobPath, err := manifest.BlobsPath(layer.Digest)
if err != nil {
return nil, err
}
for name, arr := range mlx.Load(blobPath) {
// Native safetensors loads are lazy. Filter excluded arrays before
// the Metal pre-pass materializes their payloads, including when a
// blob also contains tensors needed by the target or draft model.
if skip[name] {
skipped++
skippedBytes += arr.NumBytes()
continue
}
rawTensors[name] = arr
}
}
// Phase 2: Identify all base names that have .scale tensors and remap them
scaleBaseNames := make(map[string]bool)
allTensors := make(map[string]*mlx.Array, len(rawTensors))
for name, arr := range rawTensors {
if strings.HasSuffix(name, ".scale") {
baseName := strings.TrimSuffix(name, ".scale")
allTensors[baseName+"_scale"] = arr
scaleBaseNames[baseName] = true
}
}
// Phase 3: Process remaining tensors with complete scale knowledge
for name, arr := range rawTensors {
if strings.HasSuffix(name, ".scale") {
continue // already handled
}
if strings.HasSuffix(name, ".bias") && !strings.HasSuffix(name, ".weight_qbias") {
baseName := strings.TrimSuffix(name, ".bias")
if scaleBaseNames[baseName] {
allTensors[baseName+"_qbias"] = arr
} else {
allTensors[name] = arr
}
} else {
allTensors[name] = arr
}
}
slog.Info("Loaded tensors from manifest", "count", len(allTensors))
if skipped > 0 {
slog.Debug("Skipped tensors from manifest", "count", skipped, "bytes", skippedBytes)
}
return allTensors, nil
}
func (r *Runner) Run(host, port string, mux http.Handler) error {
g, ctx := errgroup.WithContext(context.Background())
g.Go(func() error { return r.runBatchRequests(ctx) })
g.Go(func() error {
slog.Info("Starting HTTP server", "host", host, "port", port)
return http.ListenAndServe(net.JoinHostPort(host, port), mux)
})
return g.Wait()
}