mlxrunner: reuse saved prefixes across request slots (#19677)

Reuse completed prefix checkpoints from another slot, including one still
serving a request. Restore into recipient-owned state and copy full-attention
history directly without materializing an intermediate donor snapshot.

Cover donor lifetime, pending captures, cancellation, media and drafting
boundaries, merged history, and cached evaluation.
This commit is contained in:
Daniel Hiltgen 2026-11-16 20:54:35 -08:00 • committed by GitHub
parent 312c3c8d0c
commit b8db8d5b5c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 675 additions and 79 deletions

View file

@ -246,13 +246,17 @@ func (c *KVCache) Restore(snapshot Snapshot, target int) bool {
return true
}
// Own the data before feeding it: appendKV mutates the buffer a lazy snapshot
// may still index into, so copy out first (no-op if already owned).
snap.copyOut()
// Rewind to snapshot start, then feed snapshot.
// Copy into this cache directly. A lazy peer snapshot can remain lazy:
// the slice holds its source arrays until our write has evaluated.
c.offset = snap.fromOffset
mlx.Scoped(func() { c.appendKV(snap.keys, snap.values) })
mlx.Scoped(func() {
keys, values := snap.keys, snap.values
if snap.cache != nil {
keys = snap.cache.keys.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(snap.fromOffset, snap.toOffset), mlx.Slice())
values = snap.cache.values.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(snap.fromOffset, snap.toOffset), mlx.Slice())
}
c.appendKV(keys, values)
})
// Clamp to target if needed (target may be less than full snapshot).
if target < c.offset {

View file

@ -1,6 +1,7 @@
package cache
import (
"fmt"
"testing"
"github.com/ollama/ollama/mlx"
@ -345,3 +346,99 @@ func TestKVRestoreLiveLazySnapshotIsOffsetMove(t *testing.T) {
snap.Close()
})
}
func TestPeerRestoreKeepsDonorSnapshotLazy(t *testing.T) {
for _, heads := range []int{1, 2} {
for _, reuse := range []bool{false, true} {
mlxtest.RunSubtest(t, fmt.Sprintf("heads%d/reuse%t", heads, reuse), func(t *mlxtest.T) {
source, destination := NewKVCache(), NewKVCache()
defer source.Free()
defer destination.Free()
k, v := distinctKV(0, 8, heads, 3)
source.Update(nil, k, v)
mlx.Eval(source.State()...)
snap := source.lazySnapshot(0, 4)
defer snap.Close()
if reuse {
k, v = distinctKV(100, 8, heads, 3)
destination.Update(nil, k, v)
mlx.Eval(destination.State()...)
if !destination.Restore(nil, 0) {
t.Fatal("destination rewind failed")
}
}
if !destination.Restore(snap, 4) {
t.Fatal("peer restore failed")
}
if snap.Size() != 0 {
t.Fatalf("peer read allocated %d donor snapshot bytes", snap.Size())
}
// Overwrite and free the donor before evaluating the destination graph.
if !source.Restore(nil, 0) {
t.Fatal("donor rewind failed")
}
k, v = distinctKV(200, 8, heads, 3)
source.Update(nil, k, v)
mlx.Eval(source.State()...)
source.Free()
for field, a := range destination.State() {
got := mlx.Contiguous(a, false).Floats()
for i, value := range got {
want := float32((i / 3) % 4)
if field == 1 {
want = -want
}
if value != want {
t.Fatalf("state[%d][%d]=%v, want %v after donor overwrite", field, i, value, want)
}
}
}
})
}
}
}
func TestPeerRestoreDoesNotRetainDonorAllocation(t *testing.T) {
mlxtest.Run(t, func(t *mlxtest.T) {
const heads, dims, tokens = 2, 16, 4
mlx.DefaultStream().Synchronize()
baseline := settledActiveMemory()
destination := NewKVCache()
defer destination.Free()
mlx.Scoped(func() {
source := NewKVCache()
defer source.Free()
keys, values := distinctKV(0, 16384, heads, dims)
source.Update(nil, keys, values)
mlx.Eval(source.State()...)
snapshot := source.lazySnapshot(0, tokens)
defer snapshot.Close()
if !destination.Restore(snapshot, tokens) {
t.Fatal("peer restore failed")
}
mlx.Eval(destination.State()...)
})
mlx.DefaultStream().Synchronize()
// Measure before reading the arrays: materializing a view for an assertion
// must not accidentally detach the donor allocation and conceal retention.
owned := 2 * destination.step * heads * dims * 4 // One K/V reservation, not the donor's capacity.
if retained := settledActiveMemory() - baseline; retained > owned+64*1024 {
t.Fatalf("recipient owns %d bytes but retains %d after donor destruction", owned, retained)
}
for field, state := range destination.State() {
values := mlx.Contiguous(state, false).Floats()
if len(values) != heads*tokens*dims {
t.Fatalf("state[%d] contains %d values, want %d", field, len(values), heads*tokens*dims)
}
for i, value := range values {
want := float32((i / dims) % tokens)
if field == 1 {
want = -want
}
if value != want {
t.Fatalf("state[%d][%d]=%v, want %v", field, i, value, want)
}
}
}
})
}

View file

@ -268,7 +268,7 @@ func (r *Runner) runEmbed(ctx context.Context, request embeddingRequest) (embedW
return fail(http.StatusBadRequest, err)
}
var embedding []float32
err = r.scheduleEvaluation(ctx, func(slot *batchSlot) (*evaluationState, error) {
err = r.scheduleEvaluation(ctx, func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
// Device queries and array operations stay on the MLX worker.
if budget := EmbedCapacityBudget(); budget > 0 {
if need := PredictedEmbedAllocationBytes(len(tokens), embedHiddenSize(r.Model)); need > budget {
@ -282,7 +282,7 @@ func (r *Runner) runEmbed(ctx context.Context, request embeddingRequest) (embedW
rows := hidden.Reshape(hidden.Dim(1), hidden.Dim(2))
embedding = meanPoolAndNormalize(rows, len(tokens))
return nil
}), nil
}, peers...), nil
})
if err != nil {
return embedWireResponse{}, err

View file

@ -15,7 +15,7 @@ type activeEvaluation struct {
state *evaluationState
}
func (r *Runner) scheduleEvaluation(ctx context.Context, open func(*batchSlot) (*evaluationState, error)) error {
func (r *Runner) scheduleEvaluation(ctx context.Context, open func(*batchSlot, []*prefixCache) (*evaluationState, error)) error {
request := &evaluationRequest{ctx: ctx, open: open, done: make(chan error, 1)}
select {
case <-ctx.Done():
@ -40,7 +40,13 @@ func (s *batchScheduler) admitEvaluation(request *evaluationRequest) {
slot = &batchSlot{id: i}
s.slots[i] = slot
}
p, err := request.open(slot)
var peers []*prefixCache
for _, other := range s.slots {
if other != nil && other != slot && other.scoreCache != nil {
peers = append(peers, other.scoreCache)
}
}
p, err := request.open(slot, peers)
if err != nil {
request.done <- err
return
@ -83,8 +89,8 @@ func (r *Runner) forwardHidden(ctx context.Context, input *model.PreparedRequest
return 0, err
}
cached := 0
err = r.scheduleEvaluation(ctx, func(slot *batchSlot) (*evaluationState, error) {
p := r.openEvaluation(slot, input, items, consume)
err = r.scheduleEvaluation(ctx, func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
p := r.openEvaluation(slot, input, items, consume, peers...)
p.keepScratch = true
cached = p.position
return p, nil
@ -92,7 +98,7 @@ func (r *Runner) forwardHidden(ctx context.Context, input *model.PreparedRequest
return cached, err
}
func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, items []mediaItem, consume func(*mlx.Array) error) *evaluationState {
func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, items []mediaItem, consume func(*mlx.Array) error, peers ...*prefixCache) *evaluationState {
p := &evaluationState{r: r, input: input, consume: consume}
p.media = r.openMedia(request{Tokens: input.Tokens, MediaItems: items, Layout: input.Layout})
if slot.scoreCache == nil {
@ -104,7 +110,7 @@ func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, i
}
}
if slot.scoreCache != nil {
p.session = slot.scoreCache.beginScore(input.Tokens, items)
p.session = slot.scoreCache.beginScore(input.Tokens, items, peers...)
p.session.schedulePrefillSnapshots(scoreSnapshots(len(input.Tokens), input.ScorePrefix))
p.hidden = slot.scoreHidden
p.position = len(input.Tokens) - len(p.session.remaining)

View file

@ -58,7 +58,7 @@ func TestWholeForwardRows(t *testing.T) {
}
}
done[id] = make(chan error, 1)
r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot) (*evaluationState, error) {
r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
return r.openEvaluation(slot, input, items, func(h *mlx.Array) error { result[id] = h.Floats(); return nil }), nil
}}
}
@ -95,7 +95,7 @@ func TestEvaluationCancellationAndSlotReuse(t *testing.T) {
if i == 0 {
requestCtx = ctx
}
requests[i] = &evaluationRequest{ctx: requestCtx, done: make(chan error, 1), open: func(slot *batchSlot) (*evaluationState, error) {
requests[i] = &evaluationRequest{ctx: requestCtx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
input := &model.PreparedRequest{Tokens: []int32{int32(i + 1)}, Layout: float32(10)}
return r.openEvaluation(slot, input, nil, func(h *mlx.Array) error {
results[i] = h.Floats()
@ -185,7 +185,7 @@ func TestCachedForwardBatchesAndRestores(t *testing.T) {
cached := make([]int, 2)
for id, tokens := range prompts {
done[id] = make(chan error, 1)
r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot) (*evaluationState, error) {
r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
p := r.openEvaluation(slot, &model.PreparedRequest{Tokens: tokens, ScorePrefix: 2 - id}, nil, func(h *mlx.Array) error { got[id] = h.Floats(); return nil })
cached[id] = p.position
return p, nil
@ -255,3 +255,58 @@ func TestCachedForwardChunkBoundaries(t *testing.T) {
})
}
}
func TestCachedEvaluationReusesBusyPeer(t *testing.T) {
ctx := t.Context()
mlxtest.Run(t, func(t *mlxtest.T) {
m := &cachedRowTestModel{}
r := &Runner{Model: m, parallel: 2}
s := &batchScheduler{r: r, slots: make([]*batchSlot, 2)}
defer func() {
for _, slot := range s.slots {
if slot != nil && slot.scoreCache != nil {
slot.scoreCache.close()
}
}
}()
var got []float32
open := func(tokens []int32) *evaluationRequest {
return &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
return r.openEvaluation(slot, &model.PreparedRequest{Tokens: tokens, ScorePrefix: 4}, nil, func(h *mlx.Array) error {
got = h.Floats()
return nil
}, peers...), nil
}}
}
prefix := []int32{1, 2, 3, 4, 5, 6, 7, 8}
warm := open(prefix)
s.admitEvaluation(warm)
s.stepEvaluation(s.slots[0])
if err := <-warm.done; err != nil {
t.Fatal(err)
}
// Slot zero is occupied by another request before the fork arrives.
s.admitEvaluation(open(append(slices.Clone(prefix), 30)))
fork := open(append(slices.Clone(prefix), 20))
s.admitEvaluation(fork)
if got := s.slots[1].evaluation.state.position; got != len(prefix) {
t.Fatalf("busy-peer cached tokens %d, want %d", got, len(prefix))
}
s.finishEvaluation(s.slots[0], context.Canceled)
s.slots[0].scoreCache.close()
s.stepEvaluation(s.slots[1])
if err := <-fork.done; err != nil {
t.Fatal(err)
}
var sum, squares float32
var want []float32
for _, n := range append(slices.Clone(prefix), 20) {
sum += float32(n)
squares += float32(n * n)
want = append(want, sum+squares)
}
if !slices.Equal(got, want) {
t.Fatalf("shared evaluation %v, want cumulative result %v", got, want)
}
})
}

View file

@ -96,17 +96,18 @@ func (c *prefixCache) ensureRoot() {
}
// begin prepares caches for a new request. It finds the nearest
// matching cache or creates new caches if none match.
func (c *prefixCache) begin(inputs []int32, items []mediaItem) *cacheSession {
return c.beginAt(inputs, items, max(0, len(inputs)-1))
// matching cache or creates new caches if none match. Peers belong to the same
// runner and request kind, with identical cache layouts and draft lookahead.
func (c *prefixCache) begin(inputs []int32, items []mediaItem, peers ...*prefixCache) *cacheSession {
return c.beginAt(inputs, items, max(0, len(inputs)-1), peers...)
}
// Scoring retains backbone outputs, so an exact hit needs no decode seed.
func (c *prefixCache) beginScore(inputs []int32, items []mediaItem) *cacheSession {
return c.beginAt(inputs, items, len(inputs))
func (c *prefixCache) beginScore(inputs []int32, items []mediaItem, peers ...*prefixCache) *cacheSession {
return c.beginAt(inputs, items, len(inputs), peers...)
}
func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int) *cacheSession {
func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int, peers ...*prefixCache) *cacheSession {
c.ensureRoot()
effInputs := effectiveKeyTokens(inputs, items)
@ -128,6 +129,30 @@ func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int) *
// switchToPath aligns caches to a common offset
prefix := c.minCacheOffset()
var source *prefixCache
var shared []*trieNode
for _, peer := range peers {
path, end := peer.sharedPrefix(keys, min(maxReuse, len(keys)), items)
if end > prefix {
source, shared, prefix = peer, path, end
}
}
if source != nil {
c.rewind(0)
c.restoreSnapshots(shared, prefix, 0)
prefix = c.alignCaches()
// The destination owns its trie and mutable state. Never install the
// source's nodes or snapshots in a second store.
c.activePath = []*trieNode{c.root}
if prefix > 0 {
c.pageOut(c.advancePath(c.root, keys[:prefix], prefix))
shared[len(shared)-1].lastUsed = time.Now()
}
// Cross-cache restores may materialize the source's lazy snapshots.
// Trim only after all borrowed snapshots have been consumed.
source.enforceEvictionPolicy()
originalMatched = max(originalMatched, prefix)
}
remaining := inputs[prefix:]
session := &cacheSession{
@ -235,65 +260,15 @@ func (c *prefixCache) switchToPath(newPath []*trieNode, matched int) {
ancestorOffset = c.activePath[commonLen-1].endOffset
}
var pageInCount int
// Rewind each cache to the target offset or free it. When matched
// falls within the ancestor's range (same-path case), we rewind
// directly to the match point. Otherwise we rewind to the ancestor
// and let page-in bring us forward to matched.
rewindTarget := min(ancestorOffset, matched)
for _, kv := range c.caches {
if kv == nil {
continue
}
if !kv.Restore(nil, rewindTarget) {
kv.Free()
}
}
// Page in — walk the full new path, restoring from snapshots.
// Freed caches naturally pick up the first available snapshot.
// Caches already past a node skip it via offset check.
pageIn:
for _, node := range newPath {
if !node.hasSnapshots() {
continue
}
nodeTarget := min(node.endOffset, matched)
for j, kv := range c.caches {
if kv == nil {
continue
}
if j >= len(node.snapshots) || node.snapshots[j] == nil {
continue
}
if kv.Offset() >= nodeTarget {
continue
}
if !kv.Restore(node.snapshots[j], nodeTarget) {
// Restore failed — stop page-in and let alignment
// bring all caches to a consistent offset.
break pageIn
}
}
if node.endOffset > ancestorOffset {
pageInCount++
logutil.Trace(fmt.Sprintf("page in: [%d, %d)", node.startOffset(), nodeTarget))
}
}
// Align all caches to the minimum offset.
c.rewind(rewindTarget)
pageInCount := c.restoreSnapshots(newPath, matched, ancestorOffset)
c.activePath = newPath
minOff := c.minCacheOffset()
for _, kv := range c.caches {
if kv != nil && kv.Offset() != minOff {
if !kv.Restore(nil, minOff) {
slog.Warn("failed to restore cache, freeing all caches", "offset", minOff)
c.freeAll()
break
}
}
}
minOff := c.alignCaches()
// If the live offset falls inside the last node, split it so the reused
// head stays on the active path and only the unused tail can be evicted.
@ -321,6 +296,78 @@ pageIn:
}
}
// sharedPrefix returns complete saved edges that another slot can restore.
// Live state and partial edges are not borrowed: recurrent and wrapped sliding
// caches can only resume at a saved boundary. Missing edges break KV continuity.
func (c *prefixCache) sharedPrefix(keys []trieKey, maxReuse int, items []mediaItem) ([]*trieNode, int) {
path, matched := findBestMatch(c.root, keys[:maxReuse])
end, count := 0, 1
for i := 1; i < len(path); i++ {
node := path[i]
if node.endOffset > matched || !hasAllSnapshots(node, c.caches) {
break
}
if insideAtomicItem(items, node.endOffset) == nil {
end, count = node.endOffset, i+1
}
}
if end == 0 {
return nil, 0
}
return path[:count], end
}
func (c *prefixCache) rewind(offset int) {
for _, kv := range c.caches {
if kv != nil && !kv.Restore(nil, offset) {
kv.Free()
}
}
}
func (c *prefixCache) restoreSnapshots(path []*trieNode, matched, ancestorOffset int) int {
count := 0
for _, node := range path {
if !node.hasSnapshots() {
continue
}
nodeTarget := min(node.endOffset, matched)
for j, kv := range c.caches {
if kv == nil {
continue
}
if j >= len(node.snapshots) || node.snapshots[j] == nil {
continue
}
if kv.Offset() >= nodeTarget {
continue
}
if !kv.Restore(node.snapshots[j], nodeTarget) {
// Restore failed — stop page-in and let alignment
// bring all caches to a consistent offset.
return count
}
}
if node.endOffset > ancestorOffset {
count++
logutil.Trace(fmt.Sprintf("page in: [%d, %d)", node.startOffset(), nodeTarget))
}
}
return count
}
func (c *prefixCache) alignCaches() int {
offset := c.minCacheOffset()
for _, kv := range c.caches {
if kv != nil && kv.Offset() != offset && !kv.Restore(nil, offset) {
slog.Warn("failed to restore cache, freeing all caches", "offset", offset)
c.freeAll()
return 0
}
}
return offset
}
// schedulePrefillSnapshots schedules every cache to capture snapshots as the
// forward pass crosses the given absolute token offsets, so a single full-size
// prefill records interior states without the caller breaking the batch.

View file

@ -0,0 +1,317 @@
package mlxrunner
import (
"fmt"
"slices"
"testing"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlx/mlxtest"
"github.com/ollama/ollama/mlxrunner/cache"
"github.com/ollama/ollama/mlxrunner/model"
)
func TestPrefixCacheSharesSavedPrefix(t *testing.T) {
for name, newEnv := range map[string]func() *testEnv{
"full": newTransformerEnv,
"sliding": newSlidingWindowEnv,
"recurrent": newRecurrentEnv,
"stateless": newStatelessLayerEnv,
} {
t.Run(name, func(t *testing.T) {
source, destination := newEnv(), newEnv()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
// Keep the source request active and ahead of its saved boundary.
active := source.pc.begin(inputs, nil)
active.schedulePrefillSnapshots([]int{4})
feedAll(source.caches, inputs[:8])
active.attachPrefillSnapshots()
forkTokens := []int32{1, 2, 3, 4, 30, 31, 32}
fork := destination.pc.begin(forkTokens, nil, source.pc)
if got := len(forkTokens) - len(fork.remaining); got != 4 {
t.Fatalf("reused %d tokens, want saved boundary 4", got)
}
destination.assertAllTokens(t, "restored fork", inputs[:4])
source.assertAllTokens(t, "unmodified source", inputs[:8])
// The source can finish and be destroyed without invalidating the fork.
active.close()
source.pc.close()
feedAll(destination.caches, []int32{30, 31})
destination.assertAllTokens(t, "independent fork", []int32{1, 2, 3, 4, 30, 31})
fork.close()
for _, snapshot := range source.tracker.all {
if snapshot.closeCount != 1 {
t.Fatalf("source snapshot closed %d times, want once", snapshot.closeCount)
}
}
})
}
}
func TestPrefixCacheSharedPrefixSelection(t *testing.T) {
for _, tc := range []struct {
name string
local, first, second int
want int
}{
{"longest peer", 0, 4, 6, 6},
{"local wins", 8, 4, 6, 8},
{"peer beats local", 4, 6, 2, 6},
} {
t.Run(tc.name, func(t *testing.T) {
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
local, first, second := newRecurrentEnv(), newRecurrentEnv(), newRecurrentEnv()
for i, env := range []*testEnv{local, first, second} {
defer env.pc.close()
n := []int{tc.local, tc.first, tc.second}[i]
if n > 0 {
simulateRequest(t, env.pc, inputs[:n+1], nil, n)
}
}
session := local.pc.begin(inputs, nil, first.pc, second.pc)
if got := len(inputs) - len(session.remaining); got != tc.want {
t.Fatalf("reused %d, want %d", got, tc.want)
}
local.assertAllTokens(t, "chosen prefix", inputs[:tc.want])
session.close()
})
}
}
func TestPrefixCacheSharedPrefixStopsAtMissingEdge(t *testing.T) {
source, destination := newRecurrentEnv(), newRecurrentEnv()
defer source.pc.close()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
simulateRequest(t, source.pc, inputs, nil, 4, 8)
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
path[1].setSnapshots(nil, &source.pc.pagedOutBytes)
session := destination.pc.begin(inputs, nil, source.pc)
if !slices.Equal(session.remaining, inputs) {
t.Fatalf("reused an incomplete snapshot path: remaining %v", session.remaining)
}
session.close()
}
func TestSharedPrefixSurvivesIntermediateEviction(t *testing.T) {
for name, newEnv := range map[string]func() *testEnv{
"full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv,
} {
t.Run(name, func(t *testing.T) {
source, destination := newEnv(), newEnv()
defer source.pc.close()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
simulateRequest(t, source.pc, inputs, nil, 4, 8)
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
if len(path) != 3 {
t.Fatalf("saved path has %d nodes, want root and two checkpoints", len(path))
}
// Eviction merges the earlier edge into its child: continuous KV
// history must remain reusable, even though the earlier checkpoint is gone.
source.pc.evictNode(path[1])
path, _ = findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
if len(path) != 2 || path[1].endOffset != 8 || !hasAllSnapshots(path[1], source.caches) {
t.Fatal("eviction did not preserve a complete merged checkpoint at 8")
}
fork := destination.pc.begin(inputs, nil, source.pc)
defer fork.close()
if got := len(inputs) - len(fork.remaining); got != 8 {
t.Fatalf("reused %d tokens after intermediate eviction, want 8", got)
}
destination.assertAllTokens(t, "merged peer restore", inputs[:8])
checkTrieInvariants(t, source.pc.root)
})
}
}
func TestSharedPrefixWhileDonorHasPendingCaptures(t *testing.T) {
for name, newEnv := range map[string]func() *testEnv{
"full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv,
} {
t.Run(name, func(t *testing.T) {
source, destination := newEnv(), newEnv()
defer source.pc.close()
defer destination.pc.close()
old := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
simulateRequest(t, source.pc, old, nil, 4, 8)
// Give the fake snapshots a cost so borrowing triggers the donor's
// normal eviction policy while its new captures are still pending.
walkNodes(source.pc.root, func(n *trieNode) bool {
for _, s := range n.snapshots {
if s != nil {
s.(*fakeSnapshot).byteSize = 32
source.pc.pagedOutBytes += 32
}
}
return true
})
next := []int32{1, 2, 3, 4, 30, 31, 32, 33, 34}
active := source.pc.begin(next, nil)
active.schedulePrefillSnapshots([]int{6, 8})
feedAll(source.caches, next[4:6])
source.pc.limitBytes = 1
fork := destination.pc.begin(old, nil, source.pc)
defer fork.close()
if got := len(old) - len(fork.remaining); got != 8 {
t.Fatalf("reused %d tokens from old conversation, want 8", got)
}
_, matched := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(old, nil)))
if matched >= 8 {
t.Fatal("donor eviction did not remove the borrowed old checkpoint")
}
feedAll(source.caches, next[6:8])
active.attachPrefillSnapshots()
active.close()
source.assertAllTokens(t, "continued donor prefill", next[:8])
destination.assertAllTokens(t, "independent old prefix", old[:8])
if len(active.pendingSnapshots) != 0 {
t.Fatal("donor did not drain its scheduled captures")
}
checkTrieInvariants(t, source.pc.root)
})
}
}
func TestSharedAttentionPrefixSurvivesSourceWrites(t *testing.T) {
for _, heads := range []int{1, 2} {
mlxtest.RunSubtest(t, fmt.Sprint(heads), func(t *mlxtest.T) {
newCache := func() *prefixCache {
return newPrefixCache([]cache.Cache{cache.NewKVCache(), cache.NewRotatingKVCache(4)})
}
source, destination := newCache(), newCache()
defer destination.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
active := source.begin(inputs, nil)
active.schedulePrefillSnapshots([]int{4})
write := func(pc *prefixCache, values []float32) {
mlx.Scoped(func() {
x := mlx.FromValues(slices.Repeat(values, heads), 1, heads, len(values), 1)
for _, layer := range pc.caches {
layer.(cache.Attention).Update(nil, x, x)
mlx.Eval(layer.State()...)
}
})
}
write(source, []float32{1, 2, 3, 4})
active.attachPrefillSnapshots()
write(source, []float32{5, 6, 7, 8})
fork := destination.begin([]int32{1, 2, 3, 4, 20, 21}, nil, source)
if got := len(fork.inputs) - len(fork.remaining); got != 4 {
t.Fatalf("reused %d, want 4", got)
}
// Destroying donor snapshots and then its buffers must not change
// the destination, including one-head views that are contiguous.
source.close()
for _, layer := range destination.caches {
for _, a := range layer.State() {
got := mlx.Contiguous(a, false).Floats()
want := slices.Repeat([]float32{1, 2, 3, 4}, heads)
if !slices.Equal(got, want) {
t.Fatalf("restored %T state %v, want %v", layer, got, want)
}
}
}
write(destination, []float32{20})
fork.close()
})
}
}
func TestSharedPrefixDraftLookahead(t *testing.T) {
for _, changeNext := range []bool{false, true} {
t.Run(fmt.Sprint(changeNext), func(t *testing.T) {
source, destination := newHybridEnv(), newHybridEnv()
defer source.pc.close()
defer destination.pc.close()
tokens := (&tokenStream{}).fresh(24)
source.runRequest(t, tokens, []int32{50, 51})
fork := slices.Clone(tokens)
want := 19
if changeNext {
// State at19 depends on token19 through the next-token drafter.
fork[19] = 100
want = 15
}
session := destination.pc.begin(fork, nil, source.pc)
if got := len(fork) - len(session.remaining); got != want {
t.Fatalf("reused %d, want %d", got, want)
}
for _, c := range destination.pc.caches {
assertTokens(t, "lookahead fork", c, tokens[:want])
}
session.close()
})
}
}
func TestSharedPrefixMediaIdentity(t *testing.T) {
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
items := []mediaItem{{pos: 4, length: 3, fold: 1 << 31, item: &model.PreparedItem{}}}
source := newRecurrentEnv()
defer source.pc.close()
active := source.pc.begin(inputs, items)
active.schedulePrefillSnapshots([]int{4, 7})
feedAll(source.caches, inputs[:8])
active.attachPrefillSnapshots()
for _, tc := range []struct {
name string
fold uint32
maxReuse, want int
}{
{"same image", 1 << 31, 8, 7},
{"different image", 1<<31 | 1, 8, 4},
{"inside atomic image", 1 << 31, 6, 4},
} {
t.Run(tc.name, func(t *testing.T) {
destination := newRecurrentEnv()
defer destination.pc.close()
media := slices.Clone(items)
media[0].fold = tc.fold
session := destination.pc.beginAt(inputs, media, tc.maxReuse, source.pc)
if got := len(inputs) - len(session.remaining); got != tc.want {
t.Fatalf("reused %d, want %d", got, tc.want)
}
session.close()
})
}
active.close()
}
func TestSharedPrefixSurvivesCancelledFork(t *testing.T) {
source, destination := newRecurrentEnv(), newRecurrentEnv()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5}
simulateRequest(t, source.pc, inputs, nil, 4)
fork := destination.pc.begin(inputs, nil, source.pc)
fork.close() // Cancel before forwarding any new tokens.
source.pc.close()
retry := destination.pc.begin(inputs, nil)
defer retry.close()
if got := len(inputs) - len(retry.remaining); got != 4 {
t.Fatalf("retry reused %d tokens, want 4 without the source", got)
}
destination.assertAllTokens(t, "cancelled fork", inputs[:4])
}
func TestSharedPrefixRestoreFailureRecomputes(t *testing.T) {
source, destination := newStatelessLayerEnv(), newStatelessLayerEnv()
defer source.pc.close()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
simulateRequest(t, source.pc, inputs, nil, 4, 8)
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
// The last attention snapshot cannot continue from the earlier edge.
// Recurrent state restores first, then cannot rewind to the attention
// offset after that layer rejects its snapshot. All layers must reset.
path[len(path)-1].snapshots[2].(*fakeSnapshot).from = 7
fork := destination.pc.begin(inputs, nil, source.pc)
if !slices.Equal(fork.remaining, inputs) {
t.Fatalf("incomplete restore skipped input: remaining %v", fork.remaining)
}
destination.assertAllTokens(t, "failed restore", nil)
feedAll(destination.caches, inputs[:8])
fork.close()
destination.assertAllTokens(t, "recomputed", inputs[:8])
}

View file

@ -60,7 +60,7 @@ func TestSchedulerMixedRequests(t *testing.T) {
generation := request{Ctx: ctx, Tokens: []int32{1, 1, 1}, Responses: make(chan CompletionResponse, 16), CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 8}}}
var embedding []float32
evaluations := 0
evaluation := &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot) (*evaluationState, error) {
evaluation := &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) {
return r.openEvaluation(slot, &model.PreparedRequest{Tokens: []int32{2, 2}}, nil, func(hidden *mlx.Array) error {
evaluations++
embedding = meanPoolAndNormalize(hidden.Reshape(2, mtpTestVocab), 2)

View file

@ -69,7 +69,7 @@ func completionStatusError(err error) *api.StatusError {
// entering the token decode loop. Its callback runs on the MLX worker.
type evaluationRequest struct {
ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner.
open func(*batchSlot) (*evaluationState, error)
open func(*batchSlot, []*prefixCache) (*evaluationState, error)
done chan error
}

View file

@ -284,7 +284,13 @@ func (s *batchScheduler) admitGeneration(request request) {
s.slots[best] = slot
}
req := &batchRequest{request: request}
req.session = slot.cache.begin(request.Tokens, request.MediaItems)
var peers []*prefixCache
for _, other := range s.slots {
if other != nil && other != slot && other.cache != nil {
peers = append(peers, other.cache)
}
}
req.session = slot.cache.begin(request.Tokens, request.MediaItems, peers...)
req.media = s.r.openMedia(request)
req.spec = slot.speculation.open(request, req.media.rowLayout())
req.prefill = s.r.newPrefill(req.session, req.spec, req.media)

View file

@ -842,3 +842,67 @@ func TestBatchSchedulerLearnsDraftDepth(t *testing.T) {
})
}
}
func TestGenerationReusesBusyPeer(t *testing.T) {
ctx := t.Context()
mlxtest.Run(t, func(t *mlxtest.T) {
tok := newTestTokenizer(t, []int32{7})
m := &schedulingModel{fakeMTPModel: fakeMTPModel{tok: tok}}
r := &Runner{Model: m, Tokenizer: tok, Sampler: sample.New(4096), parallel: 2}
defer r.Sampler.Free()
s := &batchScheduler{r: r, slots: make([]*batchSlot, 2)}
defer func() {
for _, slot := range s.slots {
if slot != nil && slot.cache != nil {
slot.cache.close()
}
}
}()
makeRequest := func(ctx context.Context, count int) request {
return request{
Ctx: ctx, Tokens: slices.Repeat([]int32{1}, 32), Responses: make(chan CompletionResponse, 64),
CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: count}}, SamplerOpts: sample.Options{Temperature: 0},
}
}
sourceCtx, cancel := context.WithCancel(ctx)
defer cancel()
source := makeRequest(sourceCtx, 32)
s.admitGeneration(source)
for range 8 {
s.step()
if len(source.Responses) > 0 {
break
}
}
if len(source.Responses) == 0 {
t.Fatal("source never started streaming")
}
fork := makeRequest(ctx, 2)
s.admitGeneration(fork)
if got := len(fork.Tokens) - len(s.slots[1].request.session.remaining); got != 28 {
t.Fatalf("busy-peer reuse %d, want pre-thinking snapshot28", got)
}
cancel()
var final CompletionResponse
var text string
for i := 0; i < 16 && !final.Done; i++ {
s.step()
for len(fork.Responses) > 0 {
response := <-fork.Responses
if response.Error != nil {
t.Fatal(response.Error)
}
text += response.Content
if response.Done {
final = response
}
}
}
if !final.Done || final.EvalCount != 2 || final.PromptEvalCachedCount == nil || *final.PromptEvalCachedCount != 28 {
t.Fatalf("fork completion: %+v", final)
}
if want := tok.Decode([]int32{1, 1}); text != want {
t.Fatalf("fork output %q, want %q", text, want)
}
})
}