mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
Reuse completed prefix checkpoints from another slot, including one still serving a request. Restore into recipient-owned state and copy full-attention history directly without materializing an intermediate donor snapshot. Cover donor lifetime, pending captures, cancellation, media and drafting boundaries, merged history, and cached evaluation.
317 lines
11 KiB
Go
317 lines
11 KiB
Go
package mlxrunner
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/ollama/ollama/mlx"
|
|
"github.com/ollama/ollama/mlx/mlxtest"
|
|
"github.com/ollama/ollama/mlxrunner/cache"
|
|
"github.com/ollama/ollama/mlxrunner/model"
|
|
)
|
|
|
|
func TestPrefixCacheSharesSavedPrefix(t *testing.T) {
|
|
for name, newEnv := range map[string]func() *testEnv{
|
|
"full": newTransformerEnv,
|
|
"sliding": newSlidingWindowEnv,
|
|
"recurrent": newRecurrentEnv,
|
|
"stateless": newStatelessLayerEnv,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
source, destination := newEnv(), newEnv()
|
|
defer destination.pc.close()
|
|
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
// Keep the source request active and ahead of its saved boundary.
|
|
active := source.pc.begin(inputs, nil)
|
|
active.schedulePrefillSnapshots([]int{4})
|
|
feedAll(source.caches, inputs[:8])
|
|
active.attachPrefillSnapshots()
|
|
forkTokens := []int32{1, 2, 3, 4, 30, 31, 32}
|
|
fork := destination.pc.begin(forkTokens, nil, source.pc)
|
|
if got := len(forkTokens) - len(fork.remaining); got != 4 {
|
|
t.Fatalf("reused %d tokens, want saved boundary 4", got)
|
|
}
|
|
destination.assertAllTokens(t, "restored fork", inputs[:4])
|
|
source.assertAllTokens(t, "unmodified source", inputs[:8])
|
|
// The source can finish and be destroyed without invalidating the fork.
|
|
active.close()
|
|
source.pc.close()
|
|
feedAll(destination.caches, []int32{30, 31})
|
|
destination.assertAllTokens(t, "independent fork", []int32{1, 2, 3, 4, 30, 31})
|
|
fork.close()
|
|
for _, snapshot := range source.tracker.all {
|
|
if snapshot.closeCount != 1 {
|
|
t.Fatalf("source snapshot closed %d times, want once", snapshot.closeCount)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPrefixCacheSharedPrefixSelection(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
local, first, second int
|
|
want int
|
|
}{
|
|
{"longest peer", 0, 4, 6, 6},
|
|
{"local wins", 8, 4, 6, 8},
|
|
{"peer beats local", 4, 6, 2, 6},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
local, first, second := newRecurrentEnv(), newRecurrentEnv(), newRecurrentEnv()
|
|
for i, env := range []*testEnv{local, first, second} {
|
|
defer env.pc.close()
|
|
n := []int{tc.local, tc.first, tc.second}[i]
|
|
if n > 0 {
|
|
simulateRequest(t, env.pc, inputs[:n+1], nil, n)
|
|
}
|
|
}
|
|
session := local.pc.begin(inputs, nil, first.pc, second.pc)
|
|
if got := len(inputs) - len(session.remaining); got != tc.want {
|
|
t.Fatalf("reused %d, want %d", got, tc.want)
|
|
}
|
|
local.assertAllTokens(t, "chosen prefix", inputs[:tc.want])
|
|
session.close()
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPrefixCacheSharedPrefixStopsAtMissingEdge(t *testing.T) {
|
|
source, destination := newRecurrentEnv(), newRecurrentEnv()
|
|
defer source.pc.close()
|
|
defer destination.pc.close()
|
|
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
simulateRequest(t, source.pc, inputs, nil, 4, 8)
|
|
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
|
|
path[1].setSnapshots(nil, &source.pc.pagedOutBytes)
|
|
session := destination.pc.begin(inputs, nil, source.pc)
|
|
if !slices.Equal(session.remaining, inputs) {
|
|
t.Fatalf("reused an incomplete snapshot path: remaining %v", session.remaining)
|
|
}
|
|
session.close()
|
|
}
|
|
|
|
func TestSharedPrefixSurvivesIntermediateEviction(t *testing.T) {
|
|
for name, newEnv := range map[string]func() *testEnv{
|
|
"full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
source, destination := newEnv(), newEnv()
|
|
defer source.pc.close()
|
|
defer destination.pc.close()
|
|
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
simulateRequest(t, source.pc, inputs, nil, 4, 8)
|
|
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
|
|
if len(path) != 3 {
|
|
t.Fatalf("saved path has %d nodes, want root and two checkpoints", len(path))
|
|
}
|
|
// Eviction merges the earlier edge into its child: continuous KV
|
|
// history must remain reusable, even though the earlier checkpoint is gone.
|
|
source.pc.evictNode(path[1])
|
|
path, _ = findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
|
|
if len(path) != 2 || path[1].endOffset != 8 || !hasAllSnapshots(path[1], source.caches) {
|
|
t.Fatal("eviction did not preserve a complete merged checkpoint at 8")
|
|
}
|
|
fork := destination.pc.begin(inputs, nil, source.pc)
|
|
defer fork.close()
|
|
if got := len(inputs) - len(fork.remaining); got != 8 {
|
|
t.Fatalf("reused %d tokens after intermediate eviction, want 8", got)
|
|
}
|
|
destination.assertAllTokens(t, "merged peer restore", inputs[:8])
|
|
checkTrieInvariants(t, source.pc.root)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSharedPrefixWhileDonorHasPendingCaptures(t *testing.T) {
|
|
for name, newEnv := range map[string]func() *testEnv{
|
|
"full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
source, destination := newEnv(), newEnv()
|
|
defer source.pc.close()
|
|
defer destination.pc.close()
|
|
old := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
simulateRequest(t, source.pc, old, nil, 4, 8)
|
|
// Give the fake snapshots a cost so borrowing triggers the donor's
|
|
// normal eviction policy while its new captures are still pending.
|
|
walkNodes(source.pc.root, func(n *trieNode) bool {
|
|
for _, s := range n.snapshots {
|
|
if s != nil {
|
|
s.(*fakeSnapshot).byteSize = 32
|
|
source.pc.pagedOutBytes += 32
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
next := []int32{1, 2, 3, 4, 30, 31, 32, 33, 34}
|
|
active := source.pc.begin(next, nil)
|
|
active.schedulePrefillSnapshots([]int{6, 8})
|
|
feedAll(source.caches, next[4:6])
|
|
source.pc.limitBytes = 1
|
|
fork := destination.pc.begin(old, nil, source.pc)
|
|
defer fork.close()
|
|
if got := len(old) - len(fork.remaining); got != 8 {
|
|
t.Fatalf("reused %d tokens from old conversation, want 8", got)
|
|
}
|
|
_, matched := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(old, nil)))
|
|
if matched >= 8 {
|
|
t.Fatal("donor eviction did not remove the borrowed old checkpoint")
|
|
}
|
|
feedAll(source.caches, next[6:8])
|
|
active.attachPrefillSnapshots()
|
|
active.close()
|
|
source.assertAllTokens(t, "continued donor prefill", next[:8])
|
|
destination.assertAllTokens(t, "independent old prefix", old[:8])
|
|
if len(active.pendingSnapshots) != 0 {
|
|
t.Fatal("donor did not drain its scheduled captures")
|
|
}
|
|
checkTrieInvariants(t, source.pc.root)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSharedAttentionPrefixSurvivesSourceWrites(t *testing.T) {
|
|
for _, heads := range []int{1, 2} {
|
|
mlxtest.RunSubtest(t, fmt.Sprint(heads), func(t *mlxtest.T) {
|
|
newCache := func() *prefixCache {
|
|
return newPrefixCache([]cache.Cache{cache.NewKVCache(), cache.NewRotatingKVCache(4)})
|
|
}
|
|
source, destination := newCache(), newCache()
|
|
defer destination.close()
|
|
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
active := source.begin(inputs, nil)
|
|
active.schedulePrefillSnapshots([]int{4})
|
|
write := func(pc *prefixCache, values []float32) {
|
|
mlx.Scoped(func() {
|
|
x := mlx.FromValues(slices.Repeat(values, heads), 1, heads, len(values), 1)
|
|
for _, layer := range pc.caches {
|
|
layer.(cache.Attention).Update(nil, x, x)
|
|
mlx.Eval(layer.State()...)
|
|
}
|
|
})
|
|
}
|
|
write(source, []float32{1, 2, 3, 4})
|
|
active.attachPrefillSnapshots()
|
|
write(source, []float32{5, 6, 7, 8})
|
|
fork := destination.begin([]int32{1, 2, 3, 4, 20, 21}, nil, source)
|
|
if got := len(fork.inputs) - len(fork.remaining); got != 4 {
|
|
t.Fatalf("reused %d, want 4", got)
|
|
}
|
|
// Destroying donor snapshots and then its buffers must not change
|
|
// the destination, including one-head views that are contiguous.
|
|
source.close()
|
|
for _, layer := range destination.caches {
|
|
for _, a := range layer.State() {
|
|
got := mlx.Contiguous(a, false).Floats()
|
|
want := slices.Repeat([]float32{1, 2, 3, 4}, heads)
|
|
if !slices.Equal(got, want) {
|
|
t.Fatalf("restored %T state %v, want %v", layer, got, want)
|
|
}
|
|
}
|
|
}
|
|
write(destination, []float32{20})
|
|
fork.close()
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSharedPrefixDraftLookahead(t *testing.T) {
|
|
for _, changeNext := range []bool{false, true} {
|
|
t.Run(fmt.Sprint(changeNext), func(t *testing.T) {
|
|
source, destination := newHybridEnv(), newHybridEnv()
|
|
defer source.pc.close()
|
|
defer destination.pc.close()
|
|
tokens := (&tokenStream{}).fresh(24)
|
|
source.runRequest(t, tokens, []int32{50, 51})
|
|
fork := slices.Clone(tokens)
|
|
want := 19
|
|
if changeNext {
|
|
// State at19 depends on token19 through the next-token drafter.
|
|
fork[19] = 100
|
|
want = 15
|
|
}
|
|
session := destination.pc.begin(fork, nil, source.pc)
|
|
if got := len(fork) - len(session.remaining); got != want {
|
|
t.Fatalf("reused %d, want %d", got, want)
|
|
}
|
|
for _, c := range destination.pc.caches {
|
|
assertTokens(t, "lookahead fork", c, tokens[:want])
|
|
}
|
|
session.close()
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSharedPrefixMediaIdentity(t *testing.T) {
|
|
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
items := []mediaItem{{pos: 4, length: 3, fold: 1 << 31, item: &model.PreparedItem{}}}
|
|
source := newRecurrentEnv()
|
|
defer source.pc.close()
|
|
active := source.pc.begin(inputs, items)
|
|
active.schedulePrefillSnapshots([]int{4, 7})
|
|
feedAll(source.caches, inputs[:8])
|
|
active.attachPrefillSnapshots()
|
|
for _, tc := range []struct {
|
|
name string
|
|
fold uint32
|
|
maxReuse, want int
|
|
}{
|
|
{"same image", 1 << 31, 8, 7},
|
|
{"different image", 1<<31 | 1, 8, 4},
|
|
{"inside atomic image", 1 << 31, 6, 4},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
destination := newRecurrentEnv()
|
|
defer destination.pc.close()
|
|
media := slices.Clone(items)
|
|
media[0].fold = tc.fold
|
|
session := destination.pc.beginAt(inputs, media, tc.maxReuse, source.pc)
|
|
if got := len(inputs) - len(session.remaining); got != tc.want {
|
|
t.Fatalf("reused %d, want %d", got, tc.want)
|
|
}
|
|
session.close()
|
|
})
|
|
}
|
|
active.close()
|
|
}
|
|
|
|
func TestSharedPrefixSurvivesCancelledFork(t *testing.T) {
|
|
source, destination := newRecurrentEnv(), newRecurrentEnv()
|
|
defer destination.pc.close()
|
|
inputs := []int32{1, 2, 3, 4, 5}
|
|
simulateRequest(t, source.pc, inputs, nil, 4)
|
|
fork := destination.pc.begin(inputs, nil, source.pc)
|
|
fork.close() // Cancel before forwarding any new tokens.
|
|
source.pc.close()
|
|
retry := destination.pc.begin(inputs, nil)
|
|
defer retry.close()
|
|
if got := len(inputs) - len(retry.remaining); got != 4 {
|
|
t.Fatalf("retry reused %d tokens, want 4 without the source", got)
|
|
}
|
|
destination.assertAllTokens(t, "cancelled fork", inputs[:4])
|
|
}
|
|
|
|
func TestSharedPrefixRestoreFailureRecomputes(t *testing.T) {
|
|
source, destination := newStatelessLayerEnv(), newStatelessLayerEnv()
|
|
defer source.pc.close()
|
|
defer destination.pc.close()
|
|
inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
simulateRequest(t, source.pc, inputs, nil, 4, 8)
|
|
path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil)))
|
|
// The last attention snapshot cannot continue from the earlier edge.
|
|
// Recurrent state restores first, then cannot rewind to the attention
|
|
// offset after that layer rejects its snapshot. All layers must reset.
|
|
path[len(path)-1].snapshots[2].(*fakeSnapshot).from = 7
|
|
fork := destination.pc.begin(inputs, nil, source.pc)
|
|
if !slices.Equal(fork.remaining, inputs) {
|
|
t.Fatalf("incomplete restore skipped input: remaining %v", fork.remaining)
|
|
}
|
|
destination.assertAllTokens(t, "failed restore", nil)
|
|
feedAll(destination.caches, inputs[:8])
|
|
fork.close()
|
|
destination.assertAllTokens(t, "recomputed", inputs[:8])
|
|
}
|