ollama/mlxrunner/parallel.go
Daniel Hiltgen cecac8bf1b
mlxrunner: add Inkling 2 Mini (#19799)
* mlxrunner: add Inkling 2 Mini

* mlx: fuse Inkling 2 Mini residual mixing on Metal

Fuse RMS normalization and query scoring in one kernel. Apply the
softmax weights in a second kernel to avoid storing per-query products.

* create: expose Inkling 2 Mini context length in model metadata

Copy model_max_length to max_position_embeddings during import so
ollama show reports the checkpoint's context length. Reject configs
that specify conflicting limits.

* mlxrunner: reduce Metal dispatches in Inkling 2 Mini decode

Reuse the prefill mask shader to compute relative position bias in one
dispatch for single-query attention. On M3 Ultra with Metal 3, compute
the loop controller's gain and forget projections in one matrix multiply.
2026-11-20 18:38:44 -08:00

64 lines
2 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package mlxrunner
import (
"log/slog"
"github.com/ollama/ollama/discover"
"github.com/ollama/ollama/mlx"
)
// Prefill and drafting also allocate buffers outside the cache. Reserve a shared
// 4 GiB: the measured 4k–64k workloads on M2/M5 Macs included about 3 GiB of
// additional temporary prefill storage on the older, smaller Mac.
const parallelWorkspace = 4 << 30
// num_ctx is a sizing hint. Requests can still use the full trained context.
func parallelContextLength(trained, requested int) int {
length := min(trained, 256*1024)
if requested > 0 {
length = min(length, requested)
}
return length
}
func defaultParallelism(fullContextBytes, headroom int64) int {
if fullContextBytes <= 0 || headroom <= parallelWorkspace {
return 1
}
for _, parallel := range []int{8, 4, 2} {
if fullContextBytes <= (headroom-parallelWorkspace)/int64(parallel) {
return parallel
}
}
return 1
}
func (r *Runner) configureParallelism() {
if r.parallel > 0 {
return
}
r.parallel = 1
// Automatic capacity is qualified on Metal. Other backends retain the
// existing serial default until their device budgets are qualified too.
if !mlx.MetalIsAvailable() || r.fullContextBytes <= 0 {
return
}
limit, err := mlx.MaxRecommendedWorkingSetSize()
if err != nil {
return
}
monitor, err := discover.NewMemoryMonitor()
if err != nil {
return
}
defer monitor.Close()
memory, err := monitor.Read()
if err != nil || memory.Pressure >= discover.PressureElevated || memory.FreeMemoryAboveTarget != nil && *memory.FreeMemoryAboveTarget < 0 {
return
}
headroom := min(int64(limit)-int64(mlx.ActiveMemory()), memory.AvailableRAM)
r.parallel = defaultParallelism(r.fullContextBytes, headroom)
slog.Info("MLX automatic parallelism", "parallel", r.parallel, "context_length", r.contextLength,
"projected_context_length", parallelContextLength(r.contextLength, r.parallelContext),
"full_context_bytes", r.fullContextBytes, "workspace", parallelWorkspace, "headroom", headroom)
}