mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
mlxrunner: reuse saved prefixes across request slots (#19677)
Reuse completed prefix checkpoints from another slot, including one still serving a request. Restore into recipient-owned state and copy full-attention history directly without materializing an intermediate donor snapshot. Cover donor lifetime, pending captures, cancellation, media and drafting boundaries, merged history, and cached evaluation.
This commit is contained in:
parent
312c3c8d0c
commit
b8db8d5b5c
11 changed files with 675 additions and 79 deletions
16
mlxrunner/cache/kvcache.go
vendored
16
mlxrunner/cache/kvcache.go
vendored
|
|
@ -246,13 +246,17 @@ func (c *KVCache) Restore(snapshot Snapshot, target int) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
// Own the data before feeding it: appendKV mutates the buffer a lazy snapshot
|
||||
// may still index into, so copy out first (no-op if already owned).
|
||||
snap.copyOut()
|
||||
|
||||
// Rewind to snapshot start, then feed snapshot.
|
||||
// Copy into this cache directly. A lazy peer snapshot can remain lazy:
|
||||
// the slice holds its source arrays until our write has evaluated.
|
||||
c.offset = snap.fromOffset
|
||||
mlx.Scoped(func() { c.appendKV(snap.keys, snap.values) })
|
||||
mlx.Scoped(func() {
|
||||
keys, values := snap.keys, snap.values
|
||||
if snap.cache != nil {
|
||||
keys = snap.cache.keys.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(snap.fromOffset, snap.toOffset), mlx.Slice())
|
||||
values = snap.cache.values.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(snap.fromOffset, snap.toOffset), mlx.Slice())
|
||||
}
|
||||
c.appendKV(keys, values)
|
||||
})
|
||||
|
||||
// Clamp to target if needed (target may be less than full snapshot).
|
||||
if target < c.offset {
|
||||
|
|
|
|||
97
mlxrunner/cache/lazy_test.go
vendored
97
mlxrunner/cache/lazy_test.go
vendored
|
|
@ -1,6 +1,7 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/ollama/ollama/mlx"
|
||||
|
|
@ -345,3 +346,99 @@ func TestKVRestoreLiveLazySnapshotIsOffsetMove(t *testing.T) {
|
|||
snap.Close()
|
||||
})
|
||||
}
|
||||
|
||||
func TestPeerRestoreKeepsDonorSnapshotLazy(t *testing.T) {
|
||||
for _, heads := range []int{1, 2} {
|
||||
for _, reuse := range []bool{false, true} {
|
||||
mlxtest.RunSubtest(t, fmt.Sprintf("heads%d/reuse%t", heads, reuse), func(t *mlxtest.T) {
|
||||
source, destination := NewKVCache(), NewKVCache()
|
||||
defer source.Free()
|
||||
defer destination.Free()
|
||||
k, v := distinctKV(0, 8, heads, 3)
|
||||
source.Update(nil, k, v)
|
||||
mlx.Eval(source.State()...)
|
||||
snap := source.lazySnapshot(0, 4)
|
||||
defer snap.Close()
|
||||
if reuse {
|
||||
k, v = distinctKV(100, 8, heads, 3)
|
||||
destination.Update(nil, k, v)
|
||||
mlx.Eval(destination.State()...)
|
||||
if !destination.Restore(nil, 0) {
|
||||
t.Fatal("destination rewind failed")
|
||||
}
|
||||
}
|
||||
if !destination.Restore(snap, 4) {
|
||||
t.Fatal("peer restore failed")
|
||||
}
|
||||
if snap.Size() != 0 {
|
||||
t.Fatalf("peer read allocated %d donor snapshot bytes", snap.Size())
|
||||
}
|
||||
// Overwrite and free the donor before evaluating the destination graph.
|
||||
if !source.Restore(nil, 0) {
|
||||
t.Fatal("donor rewind failed")
|
||||
}
|
||||
k, v = distinctKV(200, 8, heads, 3)
|
||||
source.Update(nil, k, v)
|
||||
mlx.Eval(source.State()...)
|
||||
source.Free()
|
||||
for field, a := range destination.State() {
|
||||
got := mlx.Contiguous(a, false).Floats()
|
||||
for i, value := range got {
|
||||
want := float32((i / 3) % 4)
|
||||
if field == 1 {
|
||||
want = -want
|
||||
}
|
||||
if value != want {
|
||||
t.Fatalf("state[%d][%d]=%v, want %v after donor overwrite", field, i, value, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerRestoreDoesNotRetainDonorAllocation(t *testing.T) {
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
const heads, dims, tokens = 2, 16, 4
|
||||
mlx.DefaultStream().Synchronize()
|
||||
baseline := settledActiveMemory()
|
||||
destination := NewKVCache()
|
||||
defer destination.Free()
|
||||
mlx.Scoped(func() {
|
||||
source := NewKVCache()
|
||||
defer source.Free()
|
||||
keys, values := distinctKV(0, 16384, heads, dims)
|
||||
source.Update(nil, keys, values)
|
||||
mlx.Eval(source.State()...)
|
||||
snapshot := source.lazySnapshot(0, tokens)
|
||||
defer snapshot.Close()
|
||||
if !destination.Restore(snapshot, tokens) {
|
||||
t.Fatal("peer restore failed")
|
||||
}
|
||||
mlx.Eval(destination.State()...)
|
||||
})
|
||||
mlx.DefaultStream().Synchronize()
|
||||
// Measure before reading the arrays: materializing a view for an assertion
|
||||
// must not accidentally detach the donor allocation and conceal retention.
|
||||
owned := 2 * destination.step * heads * dims * 4 // One K/V reservation, not the donor's capacity.
|
||||
if retained := settledActiveMemory() - baseline; retained > owned+64*1024 {
|
||||
t.Fatalf("recipient owns %d bytes but retains %d after donor destruction", owned, retained)
|
||||
}
|
||||
for field, state := range destination.State() {
|
||||
values := mlx.Contiguous(state, false).Floats()
|
||||
if len(values) != heads*tokens*dims {
|
||||
t.Fatalf("state[%d] contains %d values, want %d", field, len(values), heads*tokens*dims)
|
||||
}
|
||||
for i, value := range values {
|
||||
want := float32((i / dims) % tokens)
|
||||
if field == 1 {
|
||||
want = -want
|
||||
}
|
||||
if value != want {
|
||||
t.Fatalf("state[%d][%d]=%v, want %v", field, i, value, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -268,7 +268,7 @@ func (r *Runner) runEmbed(ctx context.Context, request embeddingRequest) (embedW
|
|||
return fail(http.StatusBadRequest, err)
|
||||
}
|
||||
var embedding []float32
|
||||
err = r.scheduleEvaluation(ctx, func(slot *batchSlot) (*evaluationState, error) {
|
||||
err = r.scheduleEvaluation(ctx, func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
|
||||
// Device queries and array operations stay on the MLX worker.
|
||||
if budget := EmbedCapacityBudget(); budget > 0 {
|
||||
if need := PredictedEmbedAllocationBytes(len(tokens), embedHiddenSize(r.Model)); need > budget {
|
||||
|
|
@ -282,7 +282,7 @@ func (r *Runner) runEmbed(ctx context.Context, request embeddingRequest) (embedW
|
|||
rows := hidden.Reshape(hidden.Dim(1), hidden.Dim(2))
|
||||
embedding = meanPoolAndNormalize(rows, len(tokens))
|
||||
return nil
|
||||
}), nil
|
||||
}, peers...), nil
|
||||
})
|
||||
if err != nil {
|
||||
return embedWireResponse{}, err
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ type activeEvaluation struct {
|
|||
state *evaluationState
|
||||
}
|
||||
|
||||
func (r *Runner) scheduleEvaluation(ctx context.Context, open func(*batchSlot) (*evaluationState, error)) error {
|
||||
func (r *Runner) scheduleEvaluation(ctx context.Context, open func(*batchSlot, []*prefixCache) (*evaluationState, error)) error {
|
||||
request := &evaluationRequest{ctx: ctx, open: open, done: make(chan error, 1)}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
|
@ -40,7 +40,13 @@ func (s *batchScheduler) admitEvaluation(request *evaluationRequest) {
|
|||
slot = &batchSlot{id: i}
|
||||
s.slots[i] = slot
|
||||
}
|
||||
p, err := request.open(slot)
|
||||
var peers []*prefixCache
|
||||
for _, other := range s.slots {
|
||||
if other != nil && other != slot && other.scoreCache != nil {
|
||||
peers = append(peers, other.scoreCache)
|
||||
}
|
||||
}
|
||||
p, err := request.open(slot, peers)
|
||||
if err != nil {
|
||||
request.done <- err
|
||||
return
|
||||
|
|
@ -83,8 +89,8 @@ func (r *Runner) forwardHidden(ctx context.Context, input *model.PreparedRequest
|
|||
return 0, err
|
||||
}
|
||||
cached := 0
|
||||
err = r.scheduleEvaluation(ctx, func(slot *batchSlot) (*evaluationState, error) {
|
||||
p := r.openEvaluation(slot, input, items, consume)
|
||||
err = r.scheduleEvaluation(ctx, func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
|
||||
p := r.openEvaluation(slot, input, items, consume, peers...)
|
||||
p.keepScratch = true
|
||||
cached = p.position
|
||||
return p, nil
|
||||
|
|
@ -92,7 +98,7 @@ func (r *Runner) forwardHidden(ctx context.Context, input *model.PreparedRequest
|
|||
return cached, err
|
||||
}
|
||||
|
||||
func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, items []mediaItem, consume func(*mlx.Array) error) *evaluationState {
|
||||
func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, items []mediaItem, consume func(*mlx.Array) error, peers ...*prefixCache) *evaluationState {
|
||||
p := &evaluationState{r: r, input: input, consume: consume}
|
||||
p.media = r.openMedia(request{Tokens: input.Tokens, MediaItems: items, Layout: input.Layout})
|
||||
if slot.scoreCache == nil {
|
||||
|
|
@ -104,7 +110,7 @@ func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, i
|
|||
}
|
||||
}
|
||||
if slot.scoreCache != nil {
|
||||
p.session = slot.scoreCache.beginScore(input.Tokens, items)
|
||||
p.session = slot.scoreCache.beginScore(input.Tokens, items, peers...)
|
||||
p.session.schedulePrefillSnapshots(scoreSnapshots(len(input.Tokens), input.ScorePrefix))
|
||||
p.hidden = slot.scoreHidden
|
||||
p.position = len(input.Tokens) - len(p.session.remaining)
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ func TestWholeForwardRows(t *testing.T) {
|
|||
}
|
||||
}
|
||||
done[id] = make(chan error, 1)
|
||||
r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot) (*evaluationState, error) {
|
||||
r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
|
||||
return r.openEvaluation(slot, input, items, func(h *mlx.Array) error { result[id] = h.Floats(); return nil }), nil
|
||||
}}
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ func TestEvaluationCancellationAndSlotReuse(t *testing.T) {
|
|||
if i == 0 {
|
||||
requestCtx = ctx
|
||||
}
|
||||
requests[i] = &evaluationRequest{ctx: requestCtx, done: make(chan error, 1), open: func(slot *batchSlot) (*evaluationState, error) {
|
||||
requests[i] = &evaluationRequest{ctx: requestCtx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
|
||||
input := &model.PreparedRequest{Tokens: []int32{int32(i + 1)}, Layout: float32(10)}
|
||||
return r.openEvaluation(slot, input, nil, func(h *mlx.Array) error {
|
||||
results[i] = h.Floats()
|
||||
|
|
@ -185,7 +185,7 @@ func TestCachedForwardBatchesAndRestores(t *testing.T) {
|
|||
cached := make([]int, 2)
|
||||
for id, tokens := range prompts {
|
||||
done[id] = make(chan error, 1)
|
||||
r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot) (*evaluationState, error) {
|
||||
r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
|
||||
p := r.openEvaluation(slot, &model.PreparedRequest{Tokens: tokens, ScorePrefix: 2 - id}, nil, func(h *mlx.Array) error { got[id] = h.Floats(); return nil })
|
||||
cached[id] = p.position
|
||||
return p, nil
|
||||
|
|
@ -255,3 +255,58 @@ func TestCachedForwardChunkBoundaries(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachedEvaluationReusesBusyPeer(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
m := &cachedRowTestModel{}
|
||||
r := &Runner{Model: m, parallel: 2}
|
||||
s := &batchScheduler{r: r, slots: make([]*batchSlot, 2)}
|
||||
defer func() {
|
||||
for _, slot := range s.slots {
|
||||
if slot != nil && slot.scoreCache != nil {
|
||||
slot.scoreCache.close()
|
||||
}
|
||||
}
|
||||
}()
|
||||
var got []float32
|
||||
open := func(tokens []int32) *evaluationRequest {
|
||||
return &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
|
||||
return r.openEvaluation(slot, &model.PreparedRequest{Tokens: tokens, ScorePrefix: 4}, nil, func(h *mlx.Array) error {
|
||||
got = h.Floats()
|
||||
return nil
|
||||
}, peers...), nil
|
||||
}}
|
||||
}
|
||||
prefix := []int32{1, 2, 3, 4, 5, 6, 7, 8}
|
||||
warm := open(prefix)
|
||||
s.admitEvaluation(warm)
|
||||
s.stepEvaluation(s.slots[0])
|
||||
if err := <-warm.done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Slot zero is occupied by another request before the fork arrives.
|
||||
s.admitEvaluation(open(append(slices.Clone(prefix), 30)))
|
||||
fork := open(append(slices.Clone(prefix), 20))
|
||||
s.admitEvaluation(fork)
|
||||
if got := s.slots[1].evaluation.state.position; got != len(prefix) {
|
||||
t.Fatalf("busy-peer cached tokens %d, want %d", got, len(prefix))
|
||||
}
|
||||
s.finishEvaluation(s.slots[0], context.Canceled)
|
||||
s.slots[0].scoreCache.close()
|
||||
s.stepEvaluation(s.slots[1])
|
||||
if err := <-fork.done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sum, squares float32
|
||||
var want []float32
|
||||
for _, n := range append(slices.Clone(prefix), 20) {
|
||||
sum += float32(n)
|
||||
squares += float32(n * n)
|
||||
want = append(want, sum+squares)
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("shared evaluation %v, want cumulative result %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -96,17 +96,18 @@ func (c *prefixCache) ensureRoot() {
|
|||
}
|
||||
|
||||
// begin prepares caches for a new request. It finds the nearest
|
||||
// matching cache or creates new caches if none match.
|
||||
func (c *prefixCache) begin(inputs []int32, items []mediaItem) *cacheSession {
|
||||
return c.beginAt(inputs, items, max(0, len(inputs)-1))
|
||||
// matching cache or creates new caches if none match. Peers belong to the same
|
||||
// runner and request kind, with identical cache layouts and draft lookahead.
|
||||
func (c *prefixCache) begin(inputs []int32, items []mediaItem, peers ...*prefixCache) *cacheSession {
|
||||
return c.beginAt(inputs, items, max(0, len(inputs)-1), peers...)
|
||||
}
|
||||
|
||||
// Scoring retains backbone outputs, so an exact hit needs no decode seed.
|
||||
func (c *prefixCache) beginScore(inputs []int32, items []mediaItem) *cacheSession {
|
||||
return c.beginAt(inputs, items, len(inputs))
|
||||
func (c *prefixCache) beginScore(inputs []int32, items []mediaItem, peers ...*prefixCache) *cacheSession {
|
||||
return c.beginAt(inputs, items, len(inputs), peers...)
|
||||
}
|
||||
|
||||
func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int) *cacheSession {
|
||||
func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int, peers ...*prefixCache) *cacheSession {
|
||||
c.ensureRoot()
|
||||
|
||||
effInputs := effectiveKeyTokens(inputs, items)
|
||||
|
|
@ -128,6 +129,30 @@ func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int) *
|
|||
|
||||
// switchToPath aligns caches to a common offset
|
||||
prefix := c.minCacheOffset()
|
||||
var source *prefixCache
|
||||
var shared []*trieNode
|
||||
for _, peer := range peers {
|
||||
path, end := peer.sharedPrefix(keys, min(maxReuse, len(keys)), items)
|
||||
if end > prefix {
|
||||
source, shared, prefix = peer, path, end
|
||||
}
|
||||
}
|
||||
if source != nil {
|
||||
c.rewind(0)
|
||||
c.restoreSnapshots(shared, prefix, 0)
|
||||
prefix = c.alignCaches()
|
||||
// The destination owns its trie and mutable state. Never install the
|
||||
// source's nodes or snapshots in a second store.
|
||||
c.activePath = []*trieNode{c.root}
|
||||
if prefix > 0 {
|
||||
c.pageOut(c.advancePath(c.root, keys[:prefix], prefix))
|
||||
shared[len(shared)-1].lastUsed = time.Now()
|
||||
}
|
||||
// Cross-cache restores may materialize the source's lazy snapshots.
|
||||
// Trim only after all borrowed snapshots have been consumed.
|
||||
source.enforceEvictionPolicy()
|
||||
originalMatched = max(originalMatched, prefix)
|
||||
}
|
||||
remaining := inputs[prefix:]
|
||||
|
||||
session := &cacheSession{
|
||||
|
|
@ -235,65 +260,15 @@ func (c *prefixCache) switchToPath(newPath []*trieNode, matched int) {
|
|||
ancestorOffset = c.activePath[commonLen-1].endOffset
|
||||
}
|
||||
|
||||
var pageInCount int
|
||||
|
||||
// Rewind each cache to the target offset or free it. When matched
|
||||
// falls within the ancestor's range (same-path case), we rewind
|
||||
// directly to the match point. Otherwise we rewind to the ancestor
|
||||
// and let page-in bring us forward to matched.
|
||||
rewindTarget := min(ancestorOffset, matched)
|
||||
for _, kv := range c.caches {
|
||||
if kv == nil {
|
||||
continue
|
||||
}
|
||||
if !kv.Restore(nil, rewindTarget) {
|
||||
kv.Free()
|
||||
}
|
||||
}
|
||||
|
||||
// Page in — walk the full new path, restoring from snapshots.
|
||||
// Freed caches naturally pick up the first available snapshot.
|
||||
// Caches already past a node skip it via offset check.
|
||||
pageIn:
|
||||
for _, node := range newPath {
|
||||
if !node.hasSnapshots() {
|
||||
continue
|
||||
}
|
||||
nodeTarget := min(node.endOffset, matched)
|
||||
for j, kv := range c.caches {
|
||||
if kv == nil {
|
||||
continue
|
||||
}
|
||||
if j >= len(node.snapshots) || node.snapshots[j] == nil {
|
||||
continue
|
||||
}
|
||||
if kv.Offset() >= nodeTarget {
|
||||
continue
|
||||
}
|
||||
if !kv.Restore(node.snapshots[j], nodeTarget) {
|
||||
// Restore failed — stop page-in and let alignment
|
||||
// bring all caches to a consistent offset.
|
||||
break pageIn
|
||||
}
|
||||
}
|
||||
if node.endOffset > ancestorOffset {
|
||||
pageInCount++
|
||||
logutil.Trace(fmt.Sprintf("page in: [%d, %d)", node.startOffset(), nodeTarget))
|
||||
}
|
||||
}
|
||||
|
||||
// Align all caches to the minimum offset.
|
||||
c.rewind(rewindTarget)
|
||||
pageInCount := c.restoreSnapshots(newPath, matched, ancestorOffset)
|
||||
c.activePath = newPath
|
||||
minOff := c.minCacheOffset()
|
||||
for _, kv := range c.caches {
|
||||
if kv != nil && kv.Offset() != minOff {
|
||||
if !kv.Restore(nil, minOff) {
|
||||
slog.Warn("failed to restore cache, freeing all caches", "offset", minOff)
|
||||
c.freeAll()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
minOff := c.alignCaches()
|
||||
|
||||
// If the live offset falls inside the last node, split it so the reused
|
||||
// head stays on the active path and only the unused tail can be evicted.
|
||||
|
|
@ -321,6 +296,78 @@ pageIn:
|
|||
}
|
||||
}
|
||||
|
||||
// sharedPrefix returns complete saved edges that another slot can restore.
|
||||
// Live state and partial edges are not borrowed: recurrent and wrapped sliding
|
||||
// caches can only resume at a saved boundary. Missing edges break KV continuity.
|
||||
func (c *prefixCache) sharedPrefix(keys []trieKey, maxReuse int, items []mediaItem) ([]*trieNode, int) {
|
||||
path, matched := findBestMatch(c.root, keys[:maxReuse])
|
||||
end, count := 0, 1
|
||||
for i := 1; i < len(path); i++ {
|
||||
node := path[i]
|
||||
if node.endOffset > matched || !hasAllSnapshots(node, c.caches) {
|
||||
break
|
||||
}
|
||||
if insideAtomicItem(items, node.endOffset) == nil {
|
||||
end, count = node.endOffset, i+1
|
||||
}
|
||||
}
|
||||
if end == 0 {
|
||||
return nil, 0
|
||||
}
|
||||
return path[:count], end
|
||||
}
|
||||
|
||||
func (c *prefixCache) rewind(offset int) {
|
||||
for _, kv := range c.caches {
|
||||
if kv != nil && !kv.Restore(nil, offset) {
|
||||
kv.Free()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *prefixCache) restoreSnapshots(path []*trieNode, matched, ancestorOffset int) int {
|
||||
count := 0
|
||||
for _, node := range path {
|
||||
if !node.hasSnapshots() {
|
||||
continue
|
||||
}
|
||||
nodeTarget := min(node.endOffset, matched)
|
||||
for j, kv := range c.caches {
|
||||
if kv == nil {
|
||||
continue
|
||||
}
|
||||
if j >= len(node.snapshots) || node.snapshots[j] == nil {
|
||||
continue
|
||||
}
|
||||
if kv.Offset() >= nodeTarget {
|
||||
continue
|
||||
}
|
||||
if !kv.Restore(node.snapshots[j], nodeTarget) {
|
||||
// Restore failed — stop page-in and let alignment
|
||||
// bring all caches to a consistent offset.
|
||||
return count
|
||||
}
|
||||
}
|
||||
if node.endOffset > ancestorOffset {
|
||||
count++
|
||||
logutil.Trace(fmt.Sprintf("page in: [%d, %d)", node.startOffset(), nodeTarget))
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (c *prefixCache) alignCaches() int {
|
||||
offset := c.minCacheOffset()
|
||||
for _, kv := range c.caches {
|
||||
if kv != nil && kv.Offset() != offset && !kv.Restore(nil, offset) {
|
||||
slog.Warn("failed to restore cache, freeing all caches", "offset", offset)
|
||||
c.freeAll()
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return offset
|
||||
}
|
||||
|
||||
// schedulePrefillSnapshots schedules every cache to capture snapshots as the
|
||||
// forward pass crosses the given absolute token offsets, so a single full-size
|
||||
// prefill records interior states without the caller breaking the batch.
|
||||
|
|
|
|||
317
mlxrunner/prefix_cache_shared_test.go
Normal file
317
mlxrunner/prefix_cache_shared_test.go
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
package mlxrunner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/ollama/ollama/mlx"
|
||||
"github.com/ollama/ollama/mlx/mlxtest"
|
||||
"github.com/ollama/ollama/mlxrunner/cache"
|
||||
"github.com/ollama/ollama/mlxrunner/model"
|
||||
)
|
||||
|
||||
func TestPrefixCacheSharesSavedPrefix(t *testing.T) {
|
||||
for name, newEnv := range map[string]func() *testEnv{
|
||||
"full": newTransformerEnv,
|
||||
"sliding": newSlidingWindowEnv,
|
||||
"recurrent": newRecurrentEnv,
|
||||
"stateless": newStatelessLayerEnv,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
source, destination := newEnv(), newEnv()
|
||||
defer destination.pc.close()
|
||||
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
||||
// Keep the source request active and ahead of its saved boundary.
|
||||
active := source.pc.begin(inputs, nil)
|
||||
active.schedulePrefillSnapshots([]int{4})
|
||||
feedAll(source.caches, inputs[:8])
|
||||
active.attachPrefillSnapshots()
|
||||
forkTokens := []int32{1, 2, 3, 4, 30, 31, 32}
|
||||
fork := destination.pc.begin(forkTokens, nil, source.pc)
|
||||
if got := len(forkTokens) - len(fork.remaining); got != 4 {
|
||||
t.Fatalf("reused %d tokens, want saved boundary 4", got)
|
||||
}
|
||||
destination.assertAllTokens(t, "restored fork", inputs[:4])
|
||||
source.assertAllTokens(t, "unmodified source", inputs[:8])
|
||||
// The source can finish and be destroyed without invalidating the fork.
|
||||
active.close()
|
||||
source.pc.close()
|
||||
feedAll(destination.caches, []int32{30, 31})
|
||||
destination.assertAllTokens(t, "independent fork", []int32{1, 2, 3, 4, 30, 31})
|
||||
fork.close()
|
||||
for _, snapshot := range source.tracker.all {
|
||||
if snapshot.closeCount != 1 {
|
||||
t.Fatalf("source snapshot closed %d times, want once", snapshot.closeCount)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefixCacheSharedPrefixSelection(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
local, first, second int
|
||||
want int
|
||||
}{
|
||||
{"longest peer", 0, 4, 6, 6},
|
||||
{"local wins", 8, 4, 6, 8},
|
||||
{"peer beats local", 4, 6, 2, 6},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
||||
local, first, second := newRecurrentEnv(), newRecurrentEnv(), newRecurrentEnv()
|
||||
for i, env := range []*testEnv{local, first, second} {
|
||||
defer env.pc.close()
|
||||
n := []int{tc.local, tc.first, tc.second}[i]
|
||||
if n > 0 {
|
||||
simulateRequest(t, env.pc, inputs[:n+1], nil, n)
|
||||
}
|
||||
}
|
||||
session := local.pc.begin(inputs, nil, first.pc, second.pc)
|
||||
if got := len(inputs) - len(session.remaining); got != tc.want {
|
||||
t.Fatalf("reused %d, want %d", got, tc.want)
|
||||
}
|
||||
local.assertAllTokens(t, "chosen prefix", inputs[:tc.want])
|
||||
session.close()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefixCacheSharedPrefixStopsAtMissingEdge(t *testing.T) {
|
||||
source, destination := newRecurrentEnv(), newRecurrentEnv()
|
||||
defer source.pc.close()
|
||||
defer destination.pc.close()
|
||||
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
||||
simulateRequest(t, source.pc, inputs, nil, 4, 8)
|
||||
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
|
||||
path[1].setSnapshots(nil, &source.pc.pagedOutBytes)
|
||||
session := destination.pc.begin(inputs, nil, source.pc)
|
||||
if !slices.Equal(session.remaining, inputs) {
|
||||
t.Fatalf("reused an incomplete snapshot path: remaining %v", session.remaining)
|
||||
}
|
||||
session.close()
|
||||
}
|
||||
|
||||
func TestSharedPrefixSurvivesIntermediateEviction(t *testing.T) {
|
||||
for name, newEnv := range map[string]func() *testEnv{
|
||||
"full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
source, destination := newEnv(), newEnv()
|
||||
defer source.pc.close()
|
||||
defer destination.pc.close()
|
||||
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
||||
simulateRequest(t, source.pc, inputs, nil, 4, 8)
|
||||
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
|
||||
if len(path) != 3 {
|
||||
t.Fatalf("saved path has %d nodes, want root and two checkpoints", len(path))
|
||||
}
|
||||
// Eviction merges the earlier edge into its child: continuous KV
|
||||
// history must remain reusable, even though the earlier checkpoint is gone.
|
||||
source.pc.evictNode(path[1])
|
||||
path, _ = findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
|
||||
if len(path) != 2 || path[1].endOffset != 8 || !hasAllSnapshots(path[1], source.caches) {
|
||||
t.Fatal("eviction did not preserve a complete merged checkpoint at 8")
|
||||
}
|
||||
fork := destination.pc.begin(inputs, nil, source.pc)
|
||||
defer fork.close()
|
||||
if got := len(inputs) - len(fork.remaining); got != 8 {
|
||||
t.Fatalf("reused %d tokens after intermediate eviction, want 8", got)
|
||||
}
|
||||
destination.assertAllTokens(t, "merged peer restore", inputs[:8])
|
||||
checkTrieInvariants(t, source.pc.root)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedPrefixWhileDonorHasPendingCaptures(t *testing.T) {
|
||||
for name, newEnv := range map[string]func() *testEnv{
|
||||
"full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
source, destination := newEnv(), newEnv()
|
||||
defer source.pc.close()
|
||||
defer destination.pc.close()
|
||||
old := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
||||
simulateRequest(t, source.pc, old, nil, 4, 8)
|
||||
// Give the fake snapshots a cost so borrowing triggers the donor's
|
||||
// normal eviction policy while its new captures are still pending.
|
||||
walkNodes(source.pc.root, func(n *trieNode) bool {
|
||||
for _, s := range n.snapshots {
|
||||
if s != nil {
|
||||
s.(*fakeSnapshot).byteSize = 32
|
||||
source.pc.pagedOutBytes += 32
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
next := []int32{1, 2, 3, 4, 30, 31, 32, 33, 34}
|
||||
active := source.pc.begin(next, nil)
|
||||
active.schedulePrefillSnapshots([]int{6, 8})
|
||||
feedAll(source.caches, next[4:6])
|
||||
source.pc.limitBytes = 1
|
||||
fork := destination.pc.begin(old, nil, source.pc)
|
||||
defer fork.close()
|
||||
if got := len(old) - len(fork.remaining); got != 8 {
|
||||
t.Fatalf("reused %d tokens from old conversation, want 8", got)
|
||||
}
|
||||
_, matched := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(old, nil)))
|
||||
if matched >= 8 {
|
||||
t.Fatal("donor eviction did not remove the borrowed old checkpoint")
|
||||
}
|
||||
feedAll(source.caches, next[6:8])
|
||||
active.attachPrefillSnapshots()
|
||||
active.close()
|
||||
source.assertAllTokens(t, "continued donor prefill", next[:8])
|
||||
destination.assertAllTokens(t, "independent old prefix", old[:8])
|
||||
if len(active.pendingSnapshots) != 0 {
|
||||
t.Fatal("donor did not drain its scheduled captures")
|
||||
}
|
||||
checkTrieInvariants(t, source.pc.root)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedAttentionPrefixSurvivesSourceWrites(t *testing.T) {
|
||||
for _, heads := range []int{1, 2} {
|
||||
mlxtest.RunSubtest(t, fmt.Sprint(heads), func(t *mlxtest.T) {
|
||||
newCache := func() *prefixCache {
|
||||
return newPrefixCache([]cache.Cache{cache.NewKVCache(), cache.NewRotatingKVCache(4)})
|
||||
}
|
||||
source, destination := newCache(), newCache()
|
||||
defer destination.close()
|
||||
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
||||
active := source.begin(inputs, nil)
|
||||
active.schedulePrefillSnapshots([]int{4})
|
||||
write := func(pc *prefixCache, values []float32) {
|
||||
mlx.Scoped(func() {
|
||||
x := mlx.FromValues(slices.Repeat(values, heads), 1, heads, len(values), 1)
|
||||
for _, layer := range pc.caches {
|
||||
layer.(cache.Attention).Update(nil, x, x)
|
||||
mlx.Eval(layer.State()...)
|
||||
}
|
||||
})
|
||||
}
|
||||
write(source, []float32{1, 2, 3, 4})
|
||||
active.attachPrefillSnapshots()
|
||||
write(source, []float32{5, 6, 7, 8})
|
||||
fork := destination.begin([]int32{1, 2, 3, 4, 20, 21}, nil, source)
|
||||
if got := len(fork.inputs) - len(fork.remaining); got != 4 {
|
||||
t.Fatalf("reused %d, want 4", got)
|
||||
}
|
||||
// Destroying donor snapshots and then its buffers must not change
|
||||
// the destination, including one-head views that are contiguous.
|
||||
source.close()
|
||||
for _, layer := range destination.caches {
|
||||
for _, a := range layer.State() {
|
||||
got := mlx.Contiguous(a, false).Floats()
|
||||
want := slices.Repeat([]float32{1, 2, 3, 4}, heads)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("restored %T state %v, want %v", layer, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
write(destination, []float32{20})
|
||||
fork.close()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedPrefixDraftLookahead(t *testing.T) {
|
||||
for _, changeNext := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(changeNext), func(t *testing.T) {
|
||||
source, destination := newHybridEnv(), newHybridEnv()
|
||||
defer source.pc.close()
|
||||
defer destination.pc.close()
|
||||
tokens := (&tokenStream{}).fresh(24)
|
||||
source.runRequest(t, tokens, []int32{50, 51})
|
||||
fork := slices.Clone(tokens)
|
||||
want := 19
|
||||
if changeNext {
|
||||
// State at19 depends on token19 through the next-token drafter.
|
||||
fork[19] = 100
|
||||
want = 15
|
||||
}
|
||||
session := destination.pc.begin(fork, nil, source.pc)
|
||||
if got := len(fork) - len(session.remaining); got != want {
|
||||
t.Fatalf("reused %d, want %d", got, want)
|
||||
}
|
||||
for _, c := range destination.pc.caches {
|
||||
assertTokens(t, "lookahead fork", c, tokens[:want])
|
||||
}
|
||||
session.close()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedPrefixMediaIdentity(t *testing.T) {
|
||||
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
||||
items := []mediaItem{{pos: 4, length: 3, fold: 1 << 31, item: &model.PreparedItem{}}}
|
||||
source := newRecurrentEnv()
|
||||
defer source.pc.close()
|
||||
active := source.pc.begin(inputs, items)
|
||||
active.schedulePrefillSnapshots([]int{4, 7})
|
||||
feedAll(source.caches, inputs[:8])
|
||||
active.attachPrefillSnapshots()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
fold uint32
|
||||
maxReuse, want int
|
||||
}{
|
||||
{"same image", 1 << 31, 8, 7},
|
||||
{"different image", 1<<31 | 1, 8, 4},
|
||||
{"inside atomic image", 1 << 31, 6, 4},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
destination := newRecurrentEnv()
|
||||
defer destination.pc.close()
|
||||
media := slices.Clone(items)
|
||||
media[0].fold = tc.fold
|
||||
session := destination.pc.beginAt(inputs, media, tc.maxReuse, source.pc)
|
||||
if got := len(inputs) - len(session.remaining); got != tc.want {
|
||||
t.Fatalf("reused %d, want %d", got, tc.want)
|
||||
}
|
||||
session.close()
|
||||
})
|
||||
}
|
||||
active.close()
|
||||
}
|
||||
|
||||
func TestSharedPrefixSurvivesCancelledFork(t *testing.T) {
|
||||
source, destination := newRecurrentEnv(), newRecurrentEnv()
|
||||
defer destination.pc.close()
|
||||
inputs := []int32{1, 2, 3, 4, 5}
|
||||
simulateRequest(t, source.pc, inputs, nil, 4)
|
||||
fork := destination.pc.begin(inputs, nil, source.pc)
|
||||
fork.close() // Cancel before forwarding any new tokens.
|
||||
source.pc.close()
|
||||
retry := destination.pc.begin(inputs, nil)
|
||||
defer retry.close()
|
||||
if got := len(inputs) - len(retry.remaining); got != 4 {
|
||||
t.Fatalf("retry reused %d tokens, want 4 without the source", got)
|
||||
}
|
||||
destination.assertAllTokens(t, "cancelled fork", inputs[:4])
|
||||
}
|
||||
|
||||
func TestSharedPrefixRestoreFailureRecomputes(t *testing.T) {
|
||||
source, destination := newStatelessLayerEnv(), newStatelessLayerEnv()
|
||||
defer source.pc.close()
|
||||
defer destination.pc.close()
|
||||
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
||||
simulateRequest(t, source.pc, inputs, nil, 4, 8)
|
||||
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
|
||||
// The last attention snapshot cannot continue from the earlier edge.
|
||||
// Recurrent state restores first, then cannot rewind to the attention
|
||||
// offset after that layer rejects its snapshot. All layers must reset.
|
||||
path[len(path)-1].snapshots[2].(*fakeSnapshot).from = 7
|
||||
fork := destination.pc.begin(inputs, nil, source.pc)
|
||||
if !slices.Equal(fork.remaining, inputs) {
|
||||
t.Fatalf("incomplete restore skipped input: remaining %v", fork.remaining)
|
||||
}
|
||||
destination.assertAllTokens(t, "failed restore", nil)
|
||||
feedAll(destination.caches, inputs[:8])
|
||||
fork.close()
|
||||
destination.assertAllTokens(t, "recomputed", inputs[:8])
|
||||
}
|
||||
|
|
@ -60,7 +60,7 @@ func TestSchedulerMixedRequests(t *testing.T) {
|
|||
generation := request{Ctx: ctx, Tokens: []int32{1, 1, 1}, Responses: make(chan CompletionResponse, 16), CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 8}}}
|
||||
var embedding []float32
|
||||
evaluations := 0
|
||||
evaluation := &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot) (*evaluationState, error) {
|
||||
evaluation := &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
|
||||
return r.openEvaluation(slot, &model.PreparedRequest{Tokens: []int32{2, 2}}, nil, func(hidden *mlx.Array) error {
|
||||
evaluations++
|
||||
embedding = meanPoolAndNormalize(hidden.Reshape(2, mtpTestVocab), 2)
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ func completionStatusError(err error) *api.StatusError {
|
|||
// entering the token decode loop. Its callback runs on the MLX worker.
|
||||
type evaluationRequest struct {
|
||||
ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner.
|
||||
open func(*batchSlot) (*evaluationState, error)
|
||||
open func(*batchSlot, []*prefixCache) (*evaluationState, error)
|
||||
done chan error
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -284,7 +284,13 @@ func (s *batchScheduler) admitGeneration(request request) {
|
|||
s.slots[best] = slot
|
||||
}
|
||||
req := &batchRequest{request: request}
|
||||
req.session = slot.cache.begin(request.Tokens, request.MediaItems)
|
||||
var peers []*prefixCache
|
||||
for _, other := range s.slots {
|
||||
if other != nil && other != slot && other.cache != nil {
|
||||
peers = append(peers, other.cache)
|
||||
}
|
||||
}
|
||||
req.session = slot.cache.begin(request.Tokens, request.MediaItems, peers...)
|
||||
req.media = s.r.openMedia(request)
|
||||
req.spec = slot.speculation.open(request, req.media.rowLayout())
|
||||
req.prefill = s.r.newPrefill(req.session, req.spec, req.media)
|
||||
|
|
|
|||
|
|
@ -842,3 +842,67 @@ func TestBatchSchedulerLearnsDraftDepth(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerationReusesBusyPeer(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
tok := newTestTokenizer(t, []int32{7})
|
||||
m := &schedulingModel{fakeMTPModel: fakeMTPModel{tok: tok}}
|
||||
r := &Runner{Model: m, Tokenizer: tok, Sampler: sample.New(4096), parallel: 2}
|
||||
defer r.Sampler.Free()
|
||||
s := &batchScheduler{r: r, slots: make([]*batchSlot, 2)}
|
||||
defer func() {
|
||||
for _, slot := range s.slots {
|
||||
if slot != nil && slot.cache != nil {
|
||||
slot.cache.close()
|
||||
}
|
||||
}
|
||||
}()
|
||||
makeRequest := func(ctx context.Context, count int) request {
|
||||
return request{
|
||||
Ctx: ctx, Tokens: slices.Repeat([]int32{1}, 32), Responses: make(chan CompletionResponse, 64),
|
||||
CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: count}}, SamplerOpts: sample.Options{Temperature: 0},
|
||||
}
|
||||
}
|
||||
sourceCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
source := makeRequest(sourceCtx, 32)
|
||||
s.admitGeneration(source)
|
||||
for range 8 {
|
||||
s.step()
|
||||
if len(source.Responses) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(source.Responses) == 0 {
|
||||
t.Fatal("source never started streaming")
|
||||
}
|
||||
fork := makeRequest(ctx, 2)
|
||||
s.admitGeneration(fork)
|
||||
if got := len(fork.Tokens) - len(s.slots[1].request.session.remaining); got != 28 {
|
||||
t.Fatalf("busy-peer reuse %d, want pre-thinking snapshot28", got)
|
||||
}
|
||||
cancel()
|
||||
var final CompletionResponse
|
||||
var text string
|
||||
for i := 0; i < 16 && !final.Done; i++ {
|
||||
s.step()
|
||||
for len(fork.Responses) > 0 {
|
||||
response := <-fork.Responses
|
||||
if response.Error != nil {
|
||||
t.Fatal(response.Error)
|
||||
}
|
||||
text += response.Content
|
||||
if response.Done {
|
||||
final = response
|
||||
}
|
||||
}
|
||||
}
|
||||
if !final.Done || final.EvalCount != 2 || final.PromptEvalCachedCount == nil || *final.PromptEvalCachedCount != 28 {
|
||||
t.Fatalf("fork completion: %+v", final)
|
||||
}
|
||||
if want := tok.Decode([]int32{1, 1}); text != want {
|
||||
t.Fatalf("fork output %q, want %q", text, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue