ollama/mlxrunner/prefix_cache_shared_test.go
Daniel Hiltgen b8db8d5b5c
mlxrunner: reuse saved prefixes across request slots (#19677)
Reuse completed prefix checkpoints from another slot, including one still
serving a request. Restore into recipient-owned state and copy full-attention
history directly without materializing an intermediate donor snapshot.

Cover donor lifetime, pending captures, cancellation, media and drafting
boundaries, merged history, and cached evaluation.
2026-11-16 20:54:35 -08:00

317 lines
11 KiB
Go

package mlxrunner
import (
"fmt"
"slices"
"testing"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlx/mlxtest"
"github.com/ollama/ollama/mlxrunner/cache"
"github.com/ollama/ollama/mlxrunner/model"
)
func TestPrefixCacheSharesSavedPrefix(t *testing.T) {
for name, newEnv := range map[string]func() *testEnv{
"full": newTransformerEnv,
"sliding": newSlidingWindowEnv,
"recurrent": newRecurrentEnv,
"stateless": newStatelessLayerEnv,
} {
t.Run(name, func(t *testing.T) {
source, destination := newEnv(), newEnv()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
// Keep the source request active and ahead of its saved boundary.
active := source.pc.begin(inputs, nil)
active.schedulePrefillSnapshots([]int{4})
feedAll(source.caches, inputs[:8])
active.attachPrefillSnapshots()
forkTokens := []int32{1, 2, 3, 4, 30, 31, 32}
fork := destination.pc.begin(forkTokens, nil, source.pc)
if got := len(forkTokens) - len(fork.remaining); got != 4 {
t.Fatalf("reused %d tokens, want saved boundary 4", got)
}
destination.assertAllTokens(t, "restored fork", inputs[:4])
source.assertAllTokens(t, "unmodified source", inputs[:8])
// The source can finish and be destroyed without invalidating the fork.
active.close()
source.pc.close()
feedAll(destination.caches, []int32{30, 31})
destination.assertAllTokens(t, "independent fork", []int32{1, 2, 3, 4, 30, 31})
fork.close()
for _, snapshot := range source.tracker.all {
if snapshot.closeCount != 1 {
t.Fatalf("source snapshot closed %d times, want once", snapshot.closeCount)
}
}
})
}
}
func TestPrefixCacheSharedPrefixSelection(t *testing.T) {
for _, tc := range []struct {
name string
local, first, second int
want int
}{
{"longest peer", 0, 4, 6, 6},
{"local wins", 8, 4, 6, 8},
{"peer beats local", 4, 6, 2, 6},
} {
t.Run(tc.name, func(t *testing.T) {
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
local, first, second := newRecurrentEnv(), newRecurrentEnv(), newRecurrentEnv()
for i, env := range []*testEnv{local, first, second} {
defer env.pc.close()
n := []int{tc.local, tc.first, tc.second}[i]
if n > 0 {
simulateRequest(t, env.pc, inputs[:n+1], nil, n)
}
}
session := local.pc.begin(inputs, nil, first.pc, second.pc)
if got := len(inputs) - len(session.remaining); got != tc.want {
t.Fatalf("reused %d, want %d", got, tc.want)
}
local.assertAllTokens(t, "chosen prefix", inputs[:tc.want])
session.close()
})
}
}
func TestPrefixCacheSharedPrefixStopsAtMissingEdge(t *testing.T) {
source, destination := newRecurrentEnv(), newRecurrentEnv()
defer source.pc.close()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
simulateRequest(t, source.pc, inputs, nil, 4, 8)
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
path[1].setSnapshots(nil, &source.pc.pagedOutBytes)
session := destination.pc.begin(inputs, nil, source.pc)
if !slices.Equal(session.remaining, inputs) {
t.Fatalf("reused an incomplete snapshot path: remaining %v", session.remaining)
}
session.close()
}
func TestSharedPrefixSurvivesIntermediateEviction(t *testing.T) {
for name, newEnv := range map[string]func() *testEnv{
"full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv,
} {
t.Run(name, func(t *testing.T) {
source, destination := newEnv(), newEnv()
defer source.pc.close()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
simulateRequest(t, source.pc, inputs, nil, 4, 8)
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
if len(path) != 3 {
t.Fatalf("saved path has %d nodes, want root and two checkpoints", len(path))
}
// Eviction merges the earlier edge into its child: continuous KV
// history must remain reusable, even though the earlier checkpoint is gone.
source.pc.evictNode(path[1])
path, _ = findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
if len(path) != 2 || path[1].endOffset != 8 || !hasAllSnapshots(path[1], source.caches) {
t.Fatal("eviction did not preserve a complete merged checkpoint at 8")
}
fork := destination.pc.begin(inputs, nil, source.pc)
defer fork.close()
if got := len(inputs) - len(fork.remaining); got != 8 {
t.Fatalf("reused %d tokens after intermediate eviction, want 8", got)
}
destination.assertAllTokens(t, "merged peer restore", inputs[:8])
checkTrieInvariants(t, source.pc.root)
})
}
}
func TestSharedPrefixWhileDonorHasPendingCaptures(t *testing.T) {
for name, newEnv := range map[string]func() *testEnv{
"full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv,
} {
t.Run(name, func(t *testing.T) {
source, destination := newEnv(), newEnv()
defer source.pc.close()
defer destination.pc.close()
old := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
simulateRequest(t, source.pc, old, nil, 4, 8)
// Give the fake snapshots a cost so borrowing triggers the donor's
// normal eviction policy while its new captures are still pending.
walkNodes(source.pc.root, func(n *trieNode) bool {
for _, s := range n.snapshots {
if s != nil {
s.(*fakeSnapshot).byteSize = 32
source.pc.pagedOutBytes += 32
}
}
return true
})
next := []int32{1, 2, 3, 4, 30, 31, 32, 33, 34}
active := source.pc.begin(next, nil)
active.schedulePrefillSnapshots([]int{6, 8})
feedAll(source.caches, next[4:6])
source.pc.limitBytes = 1
fork := destination.pc.begin(old, nil, source.pc)
defer fork.close()
if got := len(old) - len(fork.remaining); got != 8 {
t.Fatalf("reused %d tokens from old conversation, want 8", got)
}
_, matched := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(old, nil)))
if matched >= 8 {
t.Fatal("donor eviction did not remove the borrowed old checkpoint")
}
feedAll(source.caches, next[6:8])
active.attachPrefillSnapshots()
active.close()
source.assertAllTokens(t, "continued donor prefill", next[:8])
destination.assertAllTokens(t, "independent old prefix", old[:8])
if len(active.pendingSnapshots) != 0 {
t.Fatal("donor did not drain its scheduled captures")
}
checkTrieInvariants(t, source.pc.root)
})
}
}
func TestSharedAttentionPrefixSurvivesSourceWrites(t *testing.T) {
for _, heads := range []int{1, 2} {
mlxtest.RunSubtest(t, fmt.Sprint(heads), func(t *mlxtest.T) {
newCache := func() *prefixCache {
return newPrefixCache([]cache.Cache{cache.NewKVCache(), cache.NewRotatingKVCache(4)})
}
source, destination := newCache(), newCache()
defer destination.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
active := source.begin(inputs, nil)
active.schedulePrefillSnapshots([]int{4})
write := func(pc *prefixCache, values []float32) {
mlx.Scoped(func() {
x := mlx.FromValues(slices.Repeat(values, heads), 1, heads, len(values), 1)
for _, layer := range pc.caches {
layer.(cache.Attention).Update(nil, x, x)
mlx.Eval(layer.State()...)
}
})
}
write(source, []float32{1, 2, 3, 4})
active.attachPrefillSnapshots()
write(source, []float32{5, 6, 7, 8})
fork := destination.begin([]int32{1, 2, 3, 4, 20, 21}, nil, source)
if got := len(fork.inputs) - len(fork.remaining); got != 4 {
t.Fatalf("reused %d, want 4", got)
}
// Destroying donor snapshots and then its buffers must not change
// the destination, including one-head views that are contiguous.
source.close()
for _, layer := range destination.caches {
for _, a := range layer.State() {
got := mlx.Contiguous(a, false).Floats()
want := slices.Repeat([]float32{1, 2, 3, 4}, heads)
if !slices.Equal(got, want) {
t.Fatalf("restored %T state %v, want %v", layer, got, want)
}
}
}
write(destination, []float32{20})
fork.close()
})
}
}
func TestSharedPrefixDraftLookahead(t *testing.T) {
for _, changeNext := range []bool{false, true} {
t.Run(fmt.Sprint(changeNext), func(t *testing.T) {
source, destination := newHybridEnv(), newHybridEnv()
defer source.pc.close()
defer destination.pc.close()
tokens := (&tokenStream{}).fresh(24)
source.runRequest(t, tokens, []int32{50, 51})
fork := slices.Clone(tokens)
want := 19
if changeNext {
// State at19 depends on token19 through the next-token drafter.
fork[19] = 100
want = 15
}
session := destination.pc.begin(fork, nil, source.pc)
if got := len(fork) - len(session.remaining); got != want {
t.Fatalf("reused %d, want %d", got, want)
}
for _, c := range destination.pc.caches {
assertTokens(t, "lookahead fork", c, tokens[:want])
}
session.close()
})
}
}
func TestSharedPrefixMediaIdentity(t *testing.T) {
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
items := []mediaItem{{pos: 4, length: 3, fold: 1 << 31, item: &model.PreparedItem{}}}
source := newRecurrentEnv()
defer source.pc.close()
active := source.pc.begin(inputs, items)
active.schedulePrefillSnapshots([]int{4, 7})
feedAll(source.caches, inputs[:8])
active.attachPrefillSnapshots()
for _, tc := range []struct {
name string
fold uint32
maxReuse, want int
}{
{"same image", 1 << 31, 8, 7},
{"different image", 1<<31 | 1, 8, 4},
{"inside atomic image", 1 << 31, 6, 4},
} {
t.Run(tc.name, func(t *testing.T) {
destination := newRecurrentEnv()
defer destination.pc.close()
media := slices.Clone(items)
media[0].fold = tc.fold
session := destination.pc.beginAt(inputs, media, tc.maxReuse, source.pc)
if got := len(inputs) - len(session.remaining); got != tc.want {
t.Fatalf("reused %d, want %d", got, tc.want)
}
session.close()
})
}
active.close()
}
func TestSharedPrefixSurvivesCancelledFork(t *testing.T) {
source, destination := newRecurrentEnv(), newRecurrentEnv()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5}
simulateRequest(t, source.pc, inputs, nil, 4)
fork := destination.pc.begin(inputs, nil, source.pc)
fork.close() // Cancel before forwarding any new tokens.
source.pc.close()
retry := destination.pc.begin(inputs, nil)
defer retry.close()
if got := len(inputs) - len(retry.remaining); got != 4 {
t.Fatalf("retry reused %d tokens, want 4 without the source", got)
}
destination.assertAllTokens(t, "cancelled fork", inputs[:4])
}
func TestSharedPrefixRestoreFailureRecomputes(t *testing.T) {
source, destination := newStatelessLayerEnv(), newStatelessLayerEnv()
defer source.pc.close()
defer destination.pc.close()
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
simulateRequest(t, source.pc, inputs, nil, 4, 8)
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
// The last attention snapshot cannot continue from the earlier edge.
// Recurrent state restores first, then cannot rewind to the attention
// offset after that layer rejects its snapshot. All layers must reset.
path[len(path)-1].snapshots[2].(*fakeSnapshot).from = 7
fork := destination.pc.begin(inputs, nil, source.pc)
if !slices.Equal(fork.remaining, inputs) {
t.Fatalf("incomplete restore skipped input: remaining %v", fork.remaining)
}
destination.assertAllTokens(t, "failed restore", nil)
feedAll(destination.caches, inputs[:8])
fork.close()
destination.assertAllTokens(t, "recomputed", inputs[:8])
}