mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
When OLLAMA_NUM_PARALLEL is unset, select 1, 2, 4 or 8 slots using available memory and estimated per-client storage at the model's context limit. Preserve explicit settings and retain the serial default under memory pressure, for unknown cache layouts and on other backends.
45 lines
1.3 KiB
Go
45 lines
1.3 KiB
Go
package cache
|
|
|
|
import "math"
|
|
|
|
// EstimateMemory budgets one request's live cache, copies and checkpoints using
|
|
// tensor shapes from a fresh cache's first forward. It does not evaluate arrays
|
|
// or allocate storage.
|
|
// Unknown or uninitialized caches cannot safely increase automatic parallelism.
|
|
func EstimateMemory(c Cache, tokens, checkpoints int) (int64, bool) {
|
|
if c == nil {
|
|
return 0, true // Some layers share another layer's cache.
|
|
}
|
|
if tokens <= 0 || checkpoints < 0 {
|
|
return 0, false
|
|
}
|
|
var bytes int64
|
|
for _, a := range c.State() {
|
|
if a != nil {
|
|
bytes += int64(a.NumBytes())
|
|
}
|
|
}
|
|
if bytes <= 0 || c.Offset() <= 0 {
|
|
return 0, false
|
|
}
|
|
var count int64
|
|
switch c := c.(type) {
|
|
case *KVCache:
|
|
bytes /= int64(c.Offset())
|
|
// Full-attention snapshots share the live prefix until it is copied.
|
|
count = 2 * ((int64(tokens) + int64(c.step) - 1) / int64(c.step) * int64(c.step))
|
|
case *RotatingKVCache:
|
|
bytes /= int64(min(c.Offset(), c.maxSize))
|
|
count = int64(min(tokens, c.maxSize)) * (2 + int64(checkpoints))
|
|
case *RecurrentCache:
|
|
// Bounded state is retained at each prompt checkpoint, plus live state
|
|
// and a copy when the next request resumes or rewinds it.
|
|
count = 2 + int64(checkpoints)
|
|
default:
|
|
return 0, false
|
|
}
|
|
if count <= 0 || bytes > math.MaxInt64/count {
|
|
return 0, false
|
|
}
|
|
return bytes * count, true
|
|
}
|