ollama/mlxrunner/cache/memory.go
Daniel Hiltgen 0b243a141a
mlxrunner: choose default parallelism from available memory (#19785)
When OLLAMA_NUM_PARALLEL is unset, select 1, 2, 4 or 8 slots
using available memory and estimated per-client storage at the
model's context limit.

Preserve explicit settings and retain the serial default under
memory pressure, for unknown cache layouts and on other backends.
2026-11-20 07:13:42 -08:00

45 lines
1.3 KiB
Go

package cache
import "math"
// EstimateMemory budgets one request's live cache, copies and checkpoints using
// tensor shapes from a fresh cache's first forward. It does not evaluate arrays
// or allocate storage.
// Unknown or uninitialized caches cannot safely increase automatic parallelism.
func EstimateMemory(c Cache, tokens, checkpoints int) (int64, bool) {
if c == nil {
return 0, true // Some layers share another layer's cache.
}
if tokens <= 0 || checkpoints < 0 {
return 0, false
}
var bytes int64
for _, a := range c.State() {
if a != nil {
bytes += int64(a.NumBytes())
}
}
if bytes <= 0 || c.Offset() <= 0 {
return 0, false
}
var count int64
switch c := c.(type) {
case *KVCache:
bytes /= int64(c.Offset())
// Full-attention snapshots share the live prefix until it is copied.
count = 2 * ((int64(tokens) + int64(c.step) - 1) / int64(c.step) * int64(c.step))
case *RotatingKVCache:
bytes /= int64(min(c.Offset(), c.maxSize))
count = int64(min(tokens, c.maxSize)) * (2 + int64(checkpoints))
case *RecurrentCache:
// Bounded state is retained at each prompt checkpoint, plus live state
// and a copy when the next request resumes or rewinds it.
count = 2 + int64(checkpoints)
default:
return 0, false
}
if count <= 0 || bytes > math.MaxInt64/count {
return 0, false
}
return bytes * count, true
}