ollama/mlxrunner/cache/cache.go
Daniel Hiltgen 73ee71ddcc
mlxrunner: adapt saved history to memory pressure (#19678)
* mlxrunner: adapt saved history to memory pressure

Retain Metal prefix history using observed memory headroom and execution
workspace rather than a fixed 8 GiB allowance. Release unused buffers and
cold history when system compression grows, preserving reusable paths and
active requests.

Cover pressure reclamation, snapshot ownership, memory observation, and
cacheless request isolation.

* address comments

* address comments
2026-11-19 15:41:42 -08:00

223 lines
8.3 KiB
Go

package cache
import (
"fmt"
"github.com/ollama/ollama/mlx"
)
// Layer is one cache slot used during a model forward. Attention and Recurrent
// provide its read/write operations. It may be a request's Cache or a temporary
// view of the corresponding caches from several requests. The view does not own
// those caches and cannot manage their snapshots or lifetime.
type Layer interface {
// State returns state roots that should be kept/evaluated.
State() []*mlx.Array
}
// Cache owns one cache slot's state and snapshots for a single request.
// A model usually needs one slot per layer, returned together by NewCaches.
type Cache interface {
Layer
Free()
Offset() int
// Snapshot retains state from fromOffset to the current offset without
// changing the active cache. Storage may be copied, shared immutably, or
// copied lazily before the active cache overwrites it.
Snapshot(fromOffset int) Snapshot
// PrepareSnapshots schedules the cache to capture a snapshot as its
// storage offset reaches each listed offset during subsequent writes.
// Offsets are storage offsets, must not already be passed (current
// offset <= offset), and must be sorted ascending and unique. Scheduled
// offsets persist across multiple writes until TakeSnapshots is called;
// captures accumulate. The previous schedule must be drained first.
//
// Unlike Snapshot, capture happens at the moment a write crosses the
// scheduled offset, so it can record states interior to a batched write
// without the caller breaking the write into pieces.
PrepareSnapshots(offsets []int)
// TakeSnapshots returns the snapshots captured since PrepareSnapshots,
// one per scheduled offset in the caller's order, and clears the
// schedule. An entry may be nil for an empty range or an offset that
// writes never reached, for example after cancellation.
TakeSnapshots() []Snapshot
// Restore brings the cache to target. If snapshot is nil, rewinds
// using the cache's own live state. Returns false if the requested state
// is unavailable or target is negative.
Restore(snapshot Snapshot, target int) bool
// Merge combines two sequential snapshots [a,b) and [b,c) into [a,c).
// Takes ownership of both inputs.
Merge(parent, child Snapshot) Snapshot
// Split divides a snapshot [a,c) at offset b into [a,b) and [b,c).
// Takes ownership of the input. Cache types that cannot split
// (e.g. recurrent) return (nil, snapshot).
Split(snapshot Snapshot, at int) (parent, child Snapshot)
}
// Layers exposes a request's cache slots through the forward-only interface.
// It converts the slice without copying any cache state or transferring ownership.
func Layers(caches []Cache) []Layer {
if caches == nil {
return nil
}
layers := make([]Layer, len(caches))
for i, c := range caches {
layers[i] = c
}
return layers
}
// Snapshot is paged-out cache state that can be restored later.
type Snapshot interface {
// Size returns the retained VRAM bytes charged to the snapshot. Immutable
// views may conservatively charge shared backing storage more than once. A lazy
// snapshot that still indexes a live cache buffer returns 0 — it owns
// no extra memory yet. Once materialized (the cache copies the range
// out before overwriting its slots), Size returns the owned bytes.
Size() int
// SetMaterializeHook installs a callback fired once when a lazy
// snapshot materializes (allocates its owned arrays). delta is the
// newly-allocated byte count. Pass nil to detach. Snapshots that are
// never lazy may treat this as a no-op.
SetMaterializeHook(func(delta int))
// Close releases the snapshot's ownership of retained arrays.
Close()
}
// pendingSnapshots holds the scheduled snapshot capture state shared by all
// cache kinds. The owning cache calls captureReached from its write path
// to retain state at each scheduled offset the write reaches.
type pendingSnapshots struct {
offsets []int // scheduled storage offsets, in caller order
captured []Snapshot // captured[i] corresponds to offsets[i]; nil until reached
base int // running capture cursor: the from-offset of the next capture
}
// prepare schedules the listed storage offsets. See Cache.PrepareSnapshots.
func (p *pendingSnapshots) prepare(currentOffset int, offsets []int) {
if p.captured != nil {
panic("PrepareSnapshots: previous schedule not drained; TakeSnapshots first or its captures leak")
}
for i, o := range offsets {
if o < currentOffset {
panic(fmt.Sprintf("PrepareSnapshots: offset %d already passed (current %d)", o, currentOffset))
}
// captureReached and scheduledIn walk offsets in storage order and rely
// on it being ascending and unique.
if i > 0 && o <= offsets[i-1] {
panic(fmt.Sprintf("PrepareSnapshots: offsets must be sorted and unique, got %v", offsets))
}
}
p.offsets = append([]int(nil), offsets...)
p.captured = make([]Snapshot, len(offsets))
p.base = currentOffset
}
// take returns the captured snapshots and clears the schedule. See
// Cache.TakeSnapshots. Captures fire in ascending offset order; offsets beyond
// the last completed write remain nil.
func (p *pendingSnapshots) take() []Snapshot {
out := p.captured
p.offsets, p.captured = nil, nil
return out
}
// captureReached captures a snapshot for every scheduled offset that equals
// reached and hasn't been captured yet, using snap to produce the rollback
// state. The capture cursor base holds the previous scheduled boundary: snap
// reads it (yielding a [base, reached) range for position-sliceable caches),
// then base advances to reached so the next capture starts there. base only
// advances when a capture actually fires — write boundaries between scheduled
// offsets must not move it, or a later capture would record a fromOffset past
// the previous scheduled offset and break the alignment between each capture's
// range and the trie edge the caller attaches it to.
func (p *pendingSnapshots) captureReached(reached int, snap func(offset int) Snapshot) {
captured := false
for i, o := range p.offsets {
if p.captured[i] == nil && o == reached {
p.captured[i] = snap(o)
captured = true
}
}
if captured {
p.base = reached
}
}
// scheduledIn returns the scheduled offsets in [start, end], ascending. A write
// spanning [start, end) drives captureReached at each of these so any capture
// due there fires; for position-sliceable caches, the resulting snapshots cover
// [previous-scheduled-offset, this-offset) via captureReached's running cursor.
// p.offsets is ascending and unique (prepare enforces it), so the range filter
// preserves both.
func (p *pendingSnapshots) scheduledIn(start, end int) []int {
var boundaries []int
for _, o := range p.offsets {
if o >= start && o <= end {
boundaries = append(boundaries, o)
}
}
return boundaries
}
// PendingCopyBytes bounds the additional storage a lazy snapshot may need before
// its live buffers can be reused. It does not evaluate or materialize the snapshot.
func PendingCopyBytes(s Snapshot) int {
if s == nil {
return 0
}
if lazy, ok := s.(interface{ pendingBytes() int }); ok {
return lazy.pendingBytes()
}
return 0
}
// CopyBytes measures the buffers a write may need to copy. Attention caches can
// keep capacity beyond their current offset, including after a rewind. State
// only exposes the live prefix and would undercount that copy.
func CopyBytes(c Cache) int {
if c == nil {
return 0
}
var state []*mlx.Array
switch c := c.(type) {
case *KVCache:
state = []*mlx.Array{c.keys, c.values}
case *RotatingKVCache:
state = []*mlx.Array{c.keys, c.values}
default:
state = c.State()
}
var size int
for _, a := range state {
if a != nil {
size += a.NumBytes()
}
}
return size
}
// CanMergeWithoutCopy reports whether Merge can compact these snapshots without
// allocating native storage. Unknown cache implementations remain conservative.
func CanMergeWithoutCopy(c Cache, parent, child Snapshot) bool {
switch c.(type) {
case *RecurrentCache, *RotatingKVCache:
return child != nil // Keep the last valid state if the child has no capture.
case *KVCache:
if parent == nil || child == nil {
return false // Do not merge across a missing part of sliceable history.
}
p, next := parent.(*kvSnapshot), child.(*kvSnapshot)
return p.keys == nil && next.keys == nil && p.cache == next.cache && p.toOffset == next.fromOffset
default:
return c == nil
}
}