mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
* mlxrunner: add Inkling 2 Mini * mlx: fuse Inkling 2 Mini residual mixing on Metal Fuse RMS normalization and query scoring in one kernel. Apply the softmax weights in a second kernel to avoid storing per-query products. * create: expose Inkling 2 Mini context length in model metadata Copy model_max_length to max_position_embeddings during import so ollama show reports the checkpoint's context length. Reject configs that specify conflicting limits. * mlxrunner: reduce Metal dispatches in Inkling 2 Mini decode Reuse the prefill mask shader to compute relative position bias in one dispatch for single-query attention. On M3 Ultra with Metal 3, compute the loop controller's gain and forget projections in one matrix multiply.
448 lines
13 KiB
Go
448 lines
13 KiB
Go
package mlxrunner
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"maps"
|
|
"math"
|
|
"net"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
|
|
"golang.org/x/sync/errgroup"
|
|
|
|
"github.com/ollama/ollama/api"
|
|
"github.com/ollama/ollama/llm"
|
|
"github.com/ollama/ollama/manifest"
|
|
"github.com/ollama/ollama/mlx"
|
|
"github.com/ollama/ollama/mlx/mlxthread"
|
|
"github.com/ollama/ollama/mlxrunner/batch"
|
|
"github.com/ollama/ollama/mlxrunner/cache"
|
|
"github.com/ollama/ollama/mlxrunner/model"
|
|
_ "github.com/ollama/ollama/mlxrunner/model/architectures"
|
|
"github.com/ollama/ollama/mlxrunner/sample"
|
|
"github.com/ollama/ollama/mlxrunner/tokenizer"
|
|
)
|
|
|
|
// request is a short-lived struct that carries a completion request through
|
|
// a channel from the HTTP handler to the runner goroutine. The ctx field
|
|
// must travel with the request so that cancellation propagates across the
|
|
// channel boundary.
|
|
type request struct {
|
|
CompletionRequest
|
|
Responses chan CompletionResponse
|
|
|
|
Ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner.
|
|
Tokens []int32
|
|
MediaItems []mediaItem
|
|
Layout any // opaque PrepareMedia layout state, stamped on every batch
|
|
SamplerOpts sample.Options
|
|
Grammar *grammarCompilation
|
|
}
|
|
|
|
func (r request) context() context.Context { return r.Ctx }
|
|
|
|
func (r request) reject(err error) {
|
|
r.Grammar.close()
|
|
if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
|
|
// An unadmitted request has no queued responses, so the production
|
|
// buffer accepts this error without blocking scheduler teardown.
|
|
select {
|
|
case r.Responses <- CompletionResponse{Error: completionStatusError(err)}:
|
|
default:
|
|
}
|
|
}
|
|
close(r.Responses)
|
|
}
|
|
|
|
func completionStatusError(err error) *api.StatusError {
|
|
var statusErr api.StatusError
|
|
if !errors.As(err, &statusErr) {
|
|
statusErr = api.StatusError{
|
|
StatusCode: http.StatusInternalServerError,
|
|
ErrorMessage: err.Error(),
|
|
}
|
|
}
|
|
return &statusErr
|
|
}
|
|
|
|
// An evaluation computes embeddings or decision scores from the supplied input without
|
|
// entering the token decode loop. Its callback runs on the MLX worker.
|
|
type evaluationRequest struct {
|
|
ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner.
|
|
tokens []int32
|
|
items []mediaItem
|
|
open func(*batchSlot, []*prefixCache) (*evaluationState, error)
|
|
done chan error
|
|
}
|
|
|
|
func (r *evaluationRequest) reject(err error) { r.done <- err }
|
|
|
|
func (r *evaluationRequest) context() context.Context { return r.ctx }
|
|
|
|
type speechRequest struct {
|
|
ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner.
|
|
input llm.SpeechRequest
|
|
state model.SpeechState
|
|
queue *audioQueue
|
|
done chan error
|
|
}
|
|
|
|
func (r *speechRequest) reject(err error) { r.done <- err }
|
|
|
|
func (r *speechRequest) context() context.Context { return r.ctx }
|
|
|
|
// All request types share admission and the same slot limit. Once admitted,
|
|
// the scheduler advances each type using its own model operations.
|
|
type queuedRequest interface {
|
|
context() context.Context
|
|
// reject releases a request that has not acquired a slot.
|
|
reject(error)
|
|
}
|
|
|
|
type Runner struct {
|
|
parallel int
|
|
parallelContext int
|
|
fullContextBytes int64 // Estimated per-client storage at the projected context size.
|
|
Model model.Model
|
|
weights *mlx.Scope
|
|
Tokenizer *tokenizer.Tokenizer
|
|
Requests chan queuedRequest
|
|
Sampler *sample.Sampler
|
|
cache *prefixCache
|
|
contextLength int
|
|
mlxThread *mlxthread.Thread
|
|
// grammarEngine is the structured-output subsystem; nil when the grammar
|
|
// library or vocabulary failed to load.
|
|
grammarEngine *grammarEngine
|
|
// spec is the speculative-decoding subsystem. Nil when the model ships no
|
|
// draft head.
|
|
spec *speculation
|
|
|
|
// Last retention observation for new slots; accessed on the MLX worker.
|
|
snapshotLimitBytes int64
|
|
}
|
|
|
|
func (r *Runner) Load(modelName string) error {
|
|
weights, err := r.loadModel(modelName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
mlx.Eval(weights...)
|
|
mlx.ClearCache()
|
|
r.weights = mlx.NewScope()
|
|
r.weights.Attach(weights...)
|
|
configureWiredMemory()
|
|
r.configureParallelism()
|
|
return nil
|
|
}
|
|
|
|
func (r *Runner) loadModel(modelName string) (weights []*mlx.Array, err error) {
|
|
weights = mlx.ScopedArrays(func() []*mlx.Array {
|
|
root, e := model.Open(modelName)
|
|
if e != nil {
|
|
err = e
|
|
return nil
|
|
}
|
|
|
|
m, e := model.New(root)
|
|
if e != nil {
|
|
err = e
|
|
return nil
|
|
}
|
|
|
|
var skippedTensors []string
|
|
if s, ok := m.(model.SkippedTensorsModel); ok {
|
|
skippedTensors = s.SkippedTensorNames()
|
|
}
|
|
tensors, e := loadTensorsFromManifest(root, skippedTensors)
|
|
if e != nil {
|
|
err = e
|
|
return nil
|
|
}
|
|
|
|
// On Metal, materialize the loaded tensors with CPU reads before any
|
|
// weight graph exists, so the weight eval never commits a command buffer
|
|
// that waits on file data. CUDA loads read at dispatch and need no pre-pass.
|
|
if mlx.MetalIsAvailable() {
|
|
mlx.Eval(slices.Collect(maps.Values(tensors))...)
|
|
}
|
|
|
|
// Assign weights to model (model-specific logic). Target and draft weights
|
|
// must be loaded before the load scope ends so tensors from a combined
|
|
// manifest are not discarded before the draft model can retain them.
|
|
if err = m.LoadWeights(tensors); err != nil {
|
|
return nil
|
|
}
|
|
|
|
var draftModel model.DraftModel
|
|
draft, e := model.NewDraft(root, m)
|
|
if e != nil {
|
|
err = e
|
|
return nil
|
|
}
|
|
if draft != nil {
|
|
if err = draft.LoadWeights(tensors); err != nil {
|
|
return nil
|
|
}
|
|
draftModel = draft
|
|
} else if sd, ok := m.(model.SelfDraft); ok {
|
|
// Inline draft head: already loaded with the target; nil if none shipped.
|
|
draftModel = sd.SelfDraft()
|
|
}
|
|
|
|
w := mlx.Collect(m)
|
|
if draft != nil {
|
|
draftArrays := mlx.Collect(draft)
|
|
w = append(w, draftArrays...)
|
|
if root.Draft != nil {
|
|
slog.Info("Loaded draft model", "tensor_prefix", root.Draft.TensorPrefix, "config", root.Draft.Config, "arrays", len(draftArrays))
|
|
} else {
|
|
slog.Info("Loaded draft model", "arrays", len(draftArrays))
|
|
}
|
|
}
|
|
|
|
r.Model = m
|
|
mlx.EnableCompile()
|
|
// Speech-only models have no tokenizer; models that also chat need
|
|
// the text pipeline.
|
|
if _, ok := m.(model.SpeechModel); ok && m.Tokenizer() == nil {
|
|
return w
|
|
}
|
|
r.Tokenizer = m.Tokenizer()
|
|
r.contextLength = m.MaxContextLength()
|
|
// Decision and embedding models read hidden states directly. They do
|
|
// not need a text sampler or a synthetic generation forward for grammar.
|
|
if _, ok := m.(model.Scorer); ok {
|
|
return w
|
|
}
|
|
if _, ok := m.(interface{ EmbeddingDim() int }); ok {
|
|
return w
|
|
}
|
|
caches := m.NewCaches()
|
|
draftCaches := newDraftCaches(draftModel)
|
|
r.cache = newPrefixCache(slices.Concat(caches, draftCaches))
|
|
r.Sampler = sample.New(r.contextLength)
|
|
r.spec = newSpeculation(r, pipelineSlot, draftModel, caches, draftCaches)
|
|
var capacity int
|
|
if r.parallel <= 0 && mlx.MetalIsAvailable() {
|
|
capacity = parallelContextLength(r.contextLength, r.parallelContext)
|
|
}
|
|
width, fullContextBytes := modelShapes(m, draftModel, capacity)
|
|
r.fullContextBytes = fullContextBytes
|
|
r.grammarEngine = newGrammarEngine(width, r.Tokenizer)
|
|
|
|
return w
|
|
})
|
|
return weights, err
|
|
}
|
|
|
|
func (r *Runner) Close() {
|
|
r.cache.close()
|
|
r.cache = nil
|
|
if r.grammarEngine != nil {
|
|
r.grammarEngine.close()
|
|
r.grammarEngine = nil
|
|
}
|
|
r.weights.Close()
|
|
r.weights = nil
|
|
}
|
|
|
|
// newDraftCaches returns nil when the model ships no draft.
|
|
func newDraftCaches(draft model.DraftModel) []cache.Cache {
|
|
if draft == nil {
|
|
return nil
|
|
}
|
|
return draft.NewCaches()
|
|
}
|
|
|
|
// modelShapes inspects the existing one-token load graph without evaluating it.
|
|
// Cache dimensions come from the same forward used for inference, including the
|
|
// drafter, at the projected context size.
|
|
func modelShapes(m model.Model, draft model.DraftModel, tokens int) (width int, fullContextBytes int64) {
|
|
mlx.Scoped(func() {
|
|
caches := m.NewCaches()
|
|
var draftCaches []cache.Cache
|
|
if tokens > 0 {
|
|
draftCaches = newDraftCaches(draft)
|
|
}
|
|
defer func() {
|
|
for _, c := range slices.Concat(caches, draftCaches) {
|
|
if c != nil {
|
|
c.Free()
|
|
}
|
|
}
|
|
}()
|
|
b := &batch.Batch{
|
|
InputIDs: mlx.FromValues([]int32{0}, 1, 1),
|
|
SeqOffsets: []int32{0},
|
|
SeqQueryLens: []int32{1},
|
|
}
|
|
hidden, aux := m.Forward(b, cache.Layers(caches))
|
|
logits := m.Unembed(hidden)
|
|
width = logits.Dim(logits.NumDims() - 1)
|
|
if tokens <= 0 {
|
|
return
|
|
}
|
|
if len(draftCaches) > 0 {
|
|
if aux == nil {
|
|
return
|
|
}
|
|
b.Hidden = aux
|
|
draft.Forward(b, cache.Layers(caches), cache.Layers(draftCaches))
|
|
}
|
|
checkpoints := (tokens + prefillSnapshotInterval - 1) / prefillSnapshotInterval
|
|
for _, c := range caches {
|
|
bytes, ok := cache.EstimateMemory(c, tokens, checkpoints)
|
|
if !ok || bytes > math.MaxInt64-fullContextBytes {
|
|
fullContextBytes = 0
|
|
return
|
|
}
|
|
fullContextBytes += bytes
|
|
}
|
|
if fullContextBytes == 0 {
|
|
return
|
|
}
|
|
// Scoring keeps the target's hidden outputs and briefly holds both
|
|
// sides of a growing concatenation. It does not run the drafter.
|
|
hiddenBytes := int64(hidden.NumBytes())
|
|
if hiddenBytes > (math.MaxInt64-fullContextBytes)/int64(tokens)/2 {
|
|
fullContextBytes = 0
|
|
return
|
|
}
|
|
scoreBytes := fullContextBytes + 2*hiddenBytes*int64(tokens)
|
|
for _, c := range draftCaches {
|
|
bytes, ok := cache.EstimateMemory(c, tokens, checkpoints)
|
|
if !ok || bytes > math.MaxInt64-fullContextBytes {
|
|
fullContextBytes = 0
|
|
return
|
|
}
|
|
fullContextBytes += bytes
|
|
}
|
|
fullContextBytes = max(fullContextBytes, scoreBytes)
|
|
})
|
|
return width, fullContextBytes
|
|
}
|
|
|
|
func configureWiredMemory() {
|
|
if !mlx.GPUIsAvailable() {
|
|
return
|
|
}
|
|
|
|
active := mlx.ActiveMemory()
|
|
maxRecommended, err := mlx.MaxRecommendedWorkingSetSize()
|
|
if err != nil {
|
|
slog.Warn("Unable to query MLX recommended working set; using pageable memory", "error", err)
|
|
return
|
|
}
|
|
|
|
limit := min(active, maxRecommended)
|
|
previous, err := mlx.SetWiredLimit(limit)
|
|
if err != nil {
|
|
slog.Warn("Unable to configure MLX wired memory; using pageable memory",
|
|
"active", mlx.PrettyBytes(active),
|
|
"limit", mlx.PrettyBytes(limit),
|
|
"error", err)
|
|
return
|
|
}
|
|
|
|
if active > maxRecommended {
|
|
slog.Warn("MLX model exceeds the recommended working set; performance may be degraded",
|
|
"active", mlx.PrettyBytes(active),
|
|
"recommended", mlx.PrettyBytes(maxRecommended))
|
|
}
|
|
// Limiting residency to the loaded model's active allocations avoids
|
|
// reserving the remaining capacity for growing KV caches.
|
|
slog.Debug("Configured MLX wired memory",
|
|
"active", mlx.PrettyBytes(active),
|
|
"limit", mlx.PrettyBytes(limit),
|
|
"previous", mlx.PrettyBytes(previous))
|
|
}
|
|
|
|
// loadTensorsFromManifest loads tensor blobs into a flat map, excluding specified
|
|
// tensors, deduplicating by digest and remapping key suffixes.
|
|
//
|
|
// Uses a two-phase approach: first loads all raw tensors, then remaps
|
|
// .bias → _qbias with complete knowledge of which base names have .scale
|
|
// entries. This avoids a race condition where Go map iteration order could
|
|
// cause .bias to be processed before .scale within the same blob.
|
|
func loadTensorsFromManifest(root *model.Root, skippedTensors []string) (map[string]*mlx.Array, error) {
|
|
skip := make(map[string]bool, len(skippedTensors))
|
|
for _, name := range skippedTensors {
|
|
skip[name] = true
|
|
}
|
|
skipped, skippedBytes := 0, 0
|
|
// Phase 1: Load all tensors raw from all blobs
|
|
rawTensors := make(map[string]*mlx.Array)
|
|
seen := make(map[string]bool)
|
|
for _, layer := range root.Manifest.TensorLayers() {
|
|
if seen[layer.Digest] {
|
|
continue
|
|
}
|
|
seen[layer.Digest] = true
|
|
blobPath, err := manifest.BlobsPath(layer.Digest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for name, arr := range mlx.Load(blobPath) {
|
|
// Native safetensors loads are lazy. Filter excluded arrays before
|
|
// the Metal pre-pass materializes their payloads, including when a
|
|
// blob also contains tensors needed by the target or draft model.
|
|
if skip[name] {
|
|
skipped++
|
|
skippedBytes += arr.NumBytes()
|
|
continue
|
|
}
|
|
rawTensors[name] = arr
|
|
}
|
|
}
|
|
|
|
// Phase 2: Identify all base names that have .scale tensors and remap them
|
|
scaleBaseNames := make(map[string]bool)
|
|
allTensors := make(map[string]*mlx.Array, len(rawTensors))
|
|
for name, arr := range rawTensors {
|
|
if strings.HasSuffix(name, ".scale") {
|
|
baseName := strings.TrimSuffix(name, ".scale")
|
|
allTensors[baseName+"_scale"] = arr
|
|
scaleBaseNames[baseName] = true
|
|
}
|
|
}
|
|
|
|
// Phase 3: Process remaining tensors with complete scale knowledge
|
|
for name, arr := range rawTensors {
|
|
if strings.HasSuffix(name, ".scale") {
|
|
continue // already handled
|
|
}
|
|
if strings.HasSuffix(name, ".bias") && !strings.HasSuffix(name, ".weight_qbias") {
|
|
baseName := strings.TrimSuffix(name, ".bias")
|
|
if scaleBaseNames[baseName] {
|
|
allTensors[baseName+"_qbias"] = arr
|
|
} else {
|
|
allTensors[name] = arr
|
|
}
|
|
} else {
|
|
allTensors[name] = arr
|
|
}
|
|
}
|
|
|
|
slog.Info("Loaded tensors from manifest", "count", len(allTensors))
|
|
if skipped > 0 {
|
|
slog.Debug("Skipped tensors from manifest", "count", skipped, "bytes", skippedBytes)
|
|
}
|
|
return allTensors, nil
|
|
}
|
|
|
|
func (r *Runner) Run(host, port string, mux http.Handler) error {
|
|
g, ctx := errgroup.WithContext(context.Background())
|
|
|
|
g.Go(func() error { return r.runBatchRequests(ctx) })
|
|
|
|
g.Go(func() error {
|
|
slog.Info("Starting HTTP server", "host", host, "port", port)
|
|
return http.ListenAndServe(net.JoinHostPort(host, port), mux)
|
|
})
|
|
|
|
return g.Wait()
|
|
}
|