From b8db8d5b5c8daedbbef7bb9ace2221a38bc384c2 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Mon, 16 Nov 2026 20:54:35 -0800 Subject: [PATCH] mlxrunner: reuse saved prefixes across request slots (#19677) Reuse completed prefix checkpoints from another slot, including one still serving a request. Restore into recipient-owned state and copy full-attention history directly without materializing an intermediate donor snapshot. Cover donor lifetime, pending captures, cancellation, media and drafting boundaries, merged history, and cached evaluation. --- mlxrunner/cache/kvcache.go | 16 +- mlxrunner/cache/lazy_test.go | 97 ++++++++ mlxrunner/embed.go | 4 +- mlxrunner/evaluation.go | 18 +- mlxrunner/evaluation_test.go | 61 ++++- mlxrunner/prefix_cache.go | 165 +++++++++----- mlxrunner/prefix_cache_shared_test.go | 317 ++++++++++++++++++++++++++ mlxrunner/request_test.go | 2 +- mlxrunner/runner.go | 2 +- mlxrunner/scheduler.go | 8 +- mlxrunner/scheduler_test.go | 64 ++++++ 11 files changed, 675 insertions(+), 79 deletions(-) create mode 100644 mlxrunner/prefix_cache_shared_test.go diff --git a/mlxrunner/cache/kvcache.go b/mlxrunner/cache/kvcache.go index ffa126465..0eadac24f 100644 --- a/mlxrunner/cache/kvcache.go +++ b/mlxrunner/cache/kvcache.go @@ -246,13 +246,17 @@ func (c *KVCache) Restore(snapshot Snapshot, target int) bool { return true } - // Own the data before feeding it: appendKV mutates the buffer a lazy snapshot - // may still index into, so copy out first (no-op if already owned). - snap.copyOut() - - // Rewind to snapshot start, then feed snapshot. + // Copy into this cache directly. A lazy peer snapshot can remain lazy: + // the slice holds its source arrays until our write has evaluated. c.offset = snap.fromOffset - mlx.Scoped(func() { c.appendKV(snap.keys, snap.values) }) + mlx.Scoped(func() { + keys, values := snap.keys, snap.values + if snap.cache != nil { + keys = snap.cache.keys.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(snap.fromOffset, snap.toOffset), mlx.Slice()) + values = snap.cache.values.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(snap.fromOffset, snap.toOffset), mlx.Slice()) + } + c.appendKV(keys, values) + }) // Clamp to target if needed (target may be less than full snapshot). if target < c.offset { diff --git a/mlxrunner/cache/lazy_test.go b/mlxrunner/cache/lazy_test.go index 309093f02..fe37da4b1 100644 --- a/mlxrunner/cache/lazy_test.go +++ b/mlxrunner/cache/lazy_test.go @@ -1,6 +1,7 @@ package cache import ( + "fmt" "testing" "github.com/ollama/ollama/mlx" @@ -345,3 +346,99 @@ func TestKVRestoreLiveLazySnapshotIsOffsetMove(t *testing.T) { snap.Close() }) } + +func TestPeerRestoreKeepsDonorSnapshotLazy(t *testing.T) { + for _, heads := range []int{1, 2} { + for _, reuse := range []bool{false, true} { + mlxtest.RunSubtest(t, fmt.Sprintf("heads%d/reuse%t", heads, reuse), func(t *mlxtest.T) { + source, destination := NewKVCache(), NewKVCache() + defer source.Free() + defer destination.Free() + k, v := distinctKV(0, 8, heads, 3) + source.Update(nil, k, v) + mlx.Eval(source.State()...) + snap := source.lazySnapshot(0, 4) + defer snap.Close() + if reuse { + k, v = distinctKV(100, 8, heads, 3) + destination.Update(nil, k, v) + mlx.Eval(destination.State()...) + if !destination.Restore(nil, 0) { + t.Fatal("destination rewind failed") + } + } + if !destination.Restore(snap, 4) { + t.Fatal("peer restore failed") + } + if snap.Size() != 0 { + t.Fatalf("peer read allocated %d donor snapshot bytes", snap.Size()) + } + // Overwrite and free the donor before evaluating the destination graph. + if !source.Restore(nil, 0) { + t.Fatal("donor rewind failed") + } + k, v = distinctKV(200, 8, heads, 3) + source.Update(nil, k, v) + mlx.Eval(source.State()...) + source.Free() + for field, a := range destination.State() { + got := mlx.Contiguous(a, false).Floats() + for i, value := range got { + want := float32((i / 3) % 4) + if field == 1 { + want = -want + } + if value != want { + t.Fatalf("state[%d][%d]=%v, want %v after donor overwrite", field, i, value, want) + } + } + } + }) + } + } +} + +func TestPeerRestoreDoesNotRetainDonorAllocation(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + const heads, dims, tokens = 2, 16, 4 + mlx.DefaultStream().Synchronize() + baseline := settledActiveMemory() + destination := NewKVCache() + defer destination.Free() + mlx.Scoped(func() { + source := NewKVCache() + defer source.Free() + keys, values := distinctKV(0, 16384, heads, dims) + source.Update(nil, keys, values) + mlx.Eval(source.State()...) + snapshot := source.lazySnapshot(0, tokens) + defer snapshot.Close() + if !destination.Restore(snapshot, tokens) { + t.Fatal("peer restore failed") + } + mlx.Eval(destination.State()...) + }) + mlx.DefaultStream().Synchronize() + // Measure before reading the arrays: materializing a view for an assertion + // must not accidentally detach the donor allocation and conceal retention. + owned := 2 * destination.step * heads * dims * 4 // One K/V reservation, not the donor's capacity. + if retained := settledActiveMemory() - baseline; retained > owned+64*1024 { + t.Fatalf("recipient owns %d bytes but retains %d after donor destruction", owned, retained) + } + for field, state := range destination.State() { + values := mlx.Contiguous(state, false).Floats() + if len(values) != heads*tokens*dims { + t.Fatalf("state[%d] contains %d values, want %d", field, len(values), heads*tokens*dims) + } + for i, value := range values { + want := float32((i / dims) % tokens) + if field == 1 { + want = -want + } + if value != want { + t.Fatalf("state[%d][%d]=%v, want %v", field, i, value, want) + } + } + } + }) +} diff --git a/mlxrunner/embed.go b/mlxrunner/embed.go index 6bde7dfc1..39e77def2 100644 --- a/mlxrunner/embed.go +++ b/mlxrunner/embed.go @@ -268,7 +268,7 @@ func (r *Runner) runEmbed(ctx context.Context, request embeddingRequest) (embedW return fail(http.StatusBadRequest, err) } var embedding []float32 - err = r.scheduleEvaluation(ctx, func(slot *batchSlot) (*evaluationState, error) { + err = r.scheduleEvaluation(ctx, func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) { // Device queries and array operations stay on the MLX worker. if budget := EmbedCapacityBudget(); budget > 0 { if need := PredictedEmbedAllocationBytes(len(tokens), embedHiddenSize(r.Model)); need > budget { @@ -282,7 +282,7 @@ func (r *Runner) runEmbed(ctx context.Context, request embeddingRequest) (embedW rows := hidden.Reshape(hidden.Dim(1), hidden.Dim(2)) embedding = meanPoolAndNormalize(rows, len(tokens)) return nil - }), nil + }, peers...), nil }) if err != nil { return embedWireResponse{}, err diff --git a/mlxrunner/evaluation.go b/mlxrunner/evaluation.go index 370e589dc..beb99dbc5 100644 --- a/mlxrunner/evaluation.go +++ b/mlxrunner/evaluation.go @@ -15,7 +15,7 @@ type activeEvaluation struct { state *evaluationState } -func (r *Runner) scheduleEvaluation(ctx context.Context, open func(*batchSlot) (*evaluationState, error)) error { +func (r *Runner) scheduleEvaluation(ctx context.Context, open func(*batchSlot, []*prefixCache) (*evaluationState, error)) error { request := &evaluationRequest{ctx: ctx, open: open, done: make(chan error, 1)} select { case <-ctx.Done(): @@ -40,7 +40,13 @@ func (s *batchScheduler) admitEvaluation(request *evaluationRequest) { slot = &batchSlot{id: i} s.slots[i] = slot } - p, err := request.open(slot) + var peers []*prefixCache + for _, other := range s.slots { + if other != nil && other != slot && other.scoreCache != nil { + peers = append(peers, other.scoreCache) + } + } + p, err := request.open(slot, peers) if err != nil { request.done <- err return @@ -83,8 +89,8 @@ func (r *Runner) forwardHidden(ctx context.Context, input *model.PreparedRequest return 0, err } cached := 0 - err = r.scheduleEvaluation(ctx, func(slot *batchSlot) (*evaluationState, error) { - p := r.openEvaluation(slot, input, items, consume) + err = r.scheduleEvaluation(ctx, func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) { + p := r.openEvaluation(slot, input, items, consume, peers...) p.keepScratch = true cached = p.position return p, nil @@ -92,7 +98,7 @@ func (r *Runner) forwardHidden(ctx context.Context, input *model.PreparedRequest return cached, err } -func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, items []mediaItem, consume func(*mlx.Array) error) *evaluationState { +func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, items []mediaItem, consume func(*mlx.Array) error, peers ...*prefixCache) *evaluationState { p := &evaluationState{r: r, input: input, consume: consume} p.media = r.openMedia(request{Tokens: input.Tokens, MediaItems: items, Layout: input.Layout}) if slot.scoreCache == nil { @@ -104,7 +110,7 @@ func (r *Runner) openEvaluation(slot *batchSlot, input *model.PreparedRequest, i } } if slot.scoreCache != nil { - p.session = slot.scoreCache.beginScore(input.Tokens, items) + p.session = slot.scoreCache.beginScore(input.Tokens, items, peers...) p.session.schedulePrefillSnapshots(scoreSnapshots(len(input.Tokens), input.ScorePrefix)) p.hidden = slot.scoreHidden p.position = len(input.Tokens) - len(p.session.remaining) diff --git a/mlxrunner/evaluation_test.go b/mlxrunner/evaluation_test.go index 52f91f1ce..cceeeab9d 100644 --- a/mlxrunner/evaluation_test.go +++ b/mlxrunner/evaluation_test.go @@ -58,7 +58,7 @@ func TestWholeForwardRows(t *testing.T) { } } done[id] = make(chan error, 1) - r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot) (*evaluationState, error) { + r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) { return r.openEvaluation(slot, input, items, func(h *mlx.Array) error { result[id] = h.Floats(); return nil }), nil }} } @@ -95,7 +95,7 @@ func TestEvaluationCancellationAndSlotReuse(t *testing.T) { if i == 0 { requestCtx = ctx } - requests[i] = &evaluationRequest{ctx: requestCtx, done: make(chan error, 1), open: func(slot *batchSlot) (*evaluationState, error) { + requests[i] = &evaluationRequest{ctx: requestCtx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) { input := &model.PreparedRequest{Tokens: []int32{int32(i + 1)}, Layout: float32(10)} return r.openEvaluation(slot, input, nil, func(h *mlx.Array) error { results[i] = h.Floats() @@ -185,7 +185,7 @@ func TestCachedForwardBatchesAndRestores(t *testing.T) { cached := make([]int, 2) for id, tokens := range prompts { done[id] = make(chan error, 1) - r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot) (*evaluationState, error) { + r.Requests <- &evaluationRequest{ctx: context.Background(), done: done[id], open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) { p := r.openEvaluation(slot, &model.PreparedRequest{Tokens: tokens, ScorePrefix: 2 - id}, nil, func(h *mlx.Array) error { got[id] = h.Floats(); return nil }) cached[id] = p.position return p, nil @@ -255,3 +255,58 @@ func TestCachedForwardChunkBoundaries(t *testing.T) { }) } } + +func TestCachedEvaluationReusesBusyPeer(t *testing.T) { + ctx := t.Context() + mlxtest.Run(t, func(t *mlxtest.T) { + m := &cachedRowTestModel{} + r := &Runner{Model: m, parallel: 2} + s := &batchScheduler{r: r, slots: make([]*batchSlot, 2)} + defer func() { + for _, slot := range s.slots { + if slot != nil && slot.scoreCache != nil { + slot.scoreCache.close() + } + } + }() + var got []float32 + open := func(tokens []int32) *evaluationRequest { + return &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) { + return r.openEvaluation(slot, &model.PreparedRequest{Tokens: tokens, ScorePrefix: 4}, nil, func(h *mlx.Array) error { + got = h.Floats() + return nil + }, peers...), nil + }} + } + prefix := []int32{1, 2, 3, 4, 5, 6, 7, 8} + warm := open(prefix) + s.admitEvaluation(warm) + s.stepEvaluation(s.slots[0]) + if err := <-warm.done; err != nil { + t.Fatal(err) + } + // Slot zero is occupied by another request before the fork arrives. + s.admitEvaluation(open(append(slices.Clone(prefix), 30))) + fork := open(append(slices.Clone(prefix), 20)) + s.admitEvaluation(fork) + if got := s.slots[1].evaluation.state.position; got != len(prefix) { + t.Fatalf("busy-peer cached tokens %d, want %d", got, len(prefix)) + } + s.finishEvaluation(s.slots[0], context.Canceled) + s.slots[0].scoreCache.close() + s.stepEvaluation(s.slots[1]) + if err := <-fork.done; err != nil { + t.Fatal(err) + } + var sum, squares float32 + var want []float32 + for _, n := range append(slices.Clone(prefix), 20) { + sum += float32(n) + squares += float32(n * n) + want = append(want, sum+squares) + } + if !slices.Equal(got, want) { + t.Fatalf("shared evaluation %v, want cumulative result %v", got, want) + } + }) +} diff --git a/mlxrunner/prefix_cache.go b/mlxrunner/prefix_cache.go index 3fd04af36..e1fd0e99b 100644 --- a/mlxrunner/prefix_cache.go +++ b/mlxrunner/prefix_cache.go @@ -96,17 +96,18 @@ func (c *prefixCache) ensureRoot() { } // begin prepares caches for a new request. It finds the nearest -// matching cache or creates new caches if none match. -func (c *prefixCache) begin(inputs []int32, items []mediaItem) *cacheSession { - return c.beginAt(inputs, items, max(0, len(inputs)-1)) +// matching cache or creates new caches if none match. Peers belong to the same +// runner and request kind, with identical cache layouts and draft lookahead. +func (c *prefixCache) begin(inputs []int32, items []mediaItem, peers ...*prefixCache) *cacheSession { + return c.beginAt(inputs, items, max(0, len(inputs)-1), peers...) } // Scoring retains backbone outputs, so an exact hit needs no decode seed. -func (c *prefixCache) beginScore(inputs []int32, items []mediaItem) *cacheSession { - return c.beginAt(inputs, items, len(inputs)) +func (c *prefixCache) beginScore(inputs []int32, items []mediaItem, peers ...*prefixCache) *cacheSession { + return c.beginAt(inputs, items, len(inputs), peers...) } -func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int) *cacheSession { +func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int, peers ...*prefixCache) *cacheSession { c.ensureRoot() effInputs := effectiveKeyTokens(inputs, items) @@ -128,6 +129,30 @@ func (c *prefixCache) beginAt(inputs []int32, items []mediaItem, maxReuse int) * // switchToPath aligns caches to a common offset prefix := c.minCacheOffset() + var source *prefixCache + var shared []*trieNode + for _, peer := range peers { + path, end := peer.sharedPrefix(keys, min(maxReuse, len(keys)), items) + if end > prefix { + source, shared, prefix = peer, path, end + } + } + if source != nil { + c.rewind(0) + c.restoreSnapshots(shared, prefix, 0) + prefix = c.alignCaches() + // The destination owns its trie and mutable state. Never install the + // source's nodes or snapshots in a second store. + c.activePath = []*trieNode{c.root} + if prefix > 0 { + c.pageOut(c.advancePath(c.root, keys[:prefix], prefix)) + shared[len(shared)-1].lastUsed = time.Now() + } + // Cross-cache restores may materialize the source's lazy snapshots. + // Trim only after all borrowed snapshots have been consumed. + source.enforceEvictionPolicy() + originalMatched = max(originalMatched, prefix) + } remaining := inputs[prefix:] session := &cacheSession{ @@ -235,65 +260,15 @@ func (c *prefixCache) switchToPath(newPath []*trieNode, matched int) { ancestorOffset = c.activePath[commonLen-1].endOffset } - var pageInCount int - // Rewind each cache to the target offset or free it. When matched // falls within the ancestor's range (same-path case), we rewind // directly to the match point. Otherwise we rewind to the ancestor // and let page-in bring us forward to matched. rewindTarget := min(ancestorOffset, matched) - for _, kv := range c.caches { - if kv == nil { - continue - } - if !kv.Restore(nil, rewindTarget) { - kv.Free() - } - } - - // Page in — walk the full new path, restoring from snapshots. - // Freed caches naturally pick up the first available snapshot. - // Caches already past a node skip it via offset check. -pageIn: - for _, node := range newPath { - if !node.hasSnapshots() { - continue - } - nodeTarget := min(node.endOffset, matched) - for j, kv := range c.caches { - if kv == nil { - continue - } - if j >= len(node.snapshots) || node.snapshots[j] == nil { - continue - } - if kv.Offset() >= nodeTarget { - continue - } - if !kv.Restore(node.snapshots[j], nodeTarget) { - // Restore failed — stop page-in and let alignment - // bring all caches to a consistent offset. - break pageIn - } - } - if node.endOffset > ancestorOffset { - pageInCount++ - logutil.Trace(fmt.Sprintf("page in: [%d, %d)", node.startOffset(), nodeTarget)) - } - } - - // Align all caches to the minimum offset. + c.rewind(rewindTarget) + pageInCount := c.restoreSnapshots(newPath, matched, ancestorOffset) c.activePath = newPath - minOff := c.minCacheOffset() - for _, kv := range c.caches { - if kv != nil && kv.Offset() != minOff { - if !kv.Restore(nil, minOff) { - slog.Warn("failed to restore cache, freeing all caches", "offset", minOff) - c.freeAll() - break - } - } - } + minOff := c.alignCaches() // If the live offset falls inside the last node, split it so the reused // head stays on the active path and only the unused tail can be evicted. @@ -321,6 +296,78 @@ pageIn: } } +// sharedPrefix returns complete saved edges that another slot can restore. +// Live state and partial edges are not borrowed: recurrent and wrapped sliding +// caches can only resume at a saved boundary. Missing edges break KV continuity. +func (c *prefixCache) sharedPrefix(keys []trieKey, maxReuse int, items []mediaItem) ([]*trieNode, int) { + path, matched := findBestMatch(c.root, keys[:maxReuse]) + end, count := 0, 1 + for i := 1; i < len(path); i++ { + node := path[i] + if node.endOffset > matched || !hasAllSnapshots(node, c.caches) { + break + } + if insideAtomicItem(items, node.endOffset) == nil { + end, count = node.endOffset, i+1 + } + } + if end == 0 { + return nil, 0 + } + return path[:count], end +} + +func (c *prefixCache) rewind(offset int) { + for _, kv := range c.caches { + if kv != nil && !kv.Restore(nil, offset) { + kv.Free() + } + } +} + +func (c *prefixCache) restoreSnapshots(path []*trieNode, matched, ancestorOffset int) int { + count := 0 + for _, node := range path { + if !node.hasSnapshots() { + continue + } + nodeTarget := min(node.endOffset, matched) + for j, kv := range c.caches { + if kv == nil { + continue + } + if j >= len(node.snapshots) || node.snapshots[j] == nil { + continue + } + if kv.Offset() >= nodeTarget { + continue + } + if !kv.Restore(node.snapshots[j], nodeTarget) { + // Restore failed — stop page-in and let alignment + // bring all caches to a consistent offset. + return count + } + } + if node.endOffset > ancestorOffset { + count++ + logutil.Trace(fmt.Sprintf("page in: [%d, %d)", node.startOffset(), nodeTarget)) + } + } + return count +} + +func (c *prefixCache) alignCaches() int { + offset := c.minCacheOffset() + for _, kv := range c.caches { + if kv != nil && kv.Offset() != offset && !kv.Restore(nil, offset) { + slog.Warn("failed to restore cache, freeing all caches", "offset", offset) + c.freeAll() + return 0 + } + } + return offset +} + // schedulePrefillSnapshots schedules every cache to capture snapshots as the // forward pass crosses the given absolute token offsets, so a single full-size // prefill records interior states without the caller breaking the batch. diff --git a/mlxrunner/prefix_cache_shared_test.go b/mlxrunner/prefix_cache_shared_test.go new file mode 100644 index 000000000..8a48bdbcf --- /dev/null +++ b/mlxrunner/prefix_cache_shared_test.go @@ -0,0 +1,317 @@ +package mlxrunner + +import ( + "fmt" + "slices" + "testing" + + "github.com/ollama/ollama/mlx" + "github.com/ollama/ollama/mlx/mlxtest" + "github.com/ollama/ollama/mlxrunner/cache" + "github.com/ollama/ollama/mlxrunner/model" +) + +func TestPrefixCacheSharesSavedPrefix(t *testing.T) { + for name, newEnv := range map[string]func() *testEnv{ + "full": newTransformerEnv, + "sliding": newSlidingWindowEnv, + "recurrent": newRecurrentEnv, + "stateless": newStatelessLayerEnv, + } { + t.Run(name, func(t *testing.T) { + source, destination := newEnv(), newEnv() + defer destination.pc.close() + inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9} + // Keep the source request active and ahead of its saved boundary. + active := source.pc.begin(inputs, nil) + active.schedulePrefillSnapshots([]int{4}) + feedAll(source.caches, inputs[:8]) + active.attachPrefillSnapshots() + forkTokens := []int32{1, 2, 3, 4, 30, 31, 32} + fork := destination.pc.begin(forkTokens, nil, source.pc) + if got := len(forkTokens) - len(fork.remaining); got != 4 { + t.Fatalf("reused %d tokens, want saved boundary 4", got) + } + destination.assertAllTokens(t, "restored fork", inputs[:4]) + source.assertAllTokens(t, "unmodified source", inputs[:8]) + // The source can finish and be destroyed without invalidating the fork. + active.close() + source.pc.close() + feedAll(destination.caches, []int32{30, 31}) + destination.assertAllTokens(t, "independent fork", []int32{1, 2, 3, 4, 30, 31}) + fork.close() + for _, snapshot := range source.tracker.all { + if snapshot.closeCount != 1 { + t.Fatalf("source snapshot closed %d times, want once", snapshot.closeCount) + } + } + }) + } +} + +func TestPrefixCacheSharedPrefixSelection(t *testing.T) { + for _, tc := range []struct { + name string + local, first, second int + want int + }{ + {"longest peer", 0, 4, 6, 6}, + {"local wins", 8, 4, 6, 8}, + {"peer beats local", 4, 6, 2, 6}, + } { + t.Run(tc.name, func(t *testing.T) { + inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9} + local, first, second := newRecurrentEnv(), newRecurrentEnv(), newRecurrentEnv() + for i, env := range []*testEnv{local, first, second} { + defer env.pc.close() + n := []int{tc.local, tc.first, tc.second}[i] + if n > 0 { + simulateRequest(t, env.pc, inputs[:n+1], nil, n) + } + } + session := local.pc.begin(inputs, nil, first.pc, second.pc) + if got := len(inputs) - len(session.remaining); got != tc.want { + t.Fatalf("reused %d, want %d", got, tc.want) + } + local.assertAllTokens(t, "chosen prefix", inputs[:tc.want]) + session.close() + }) + } +} + +func TestPrefixCacheSharedPrefixStopsAtMissingEdge(t *testing.T) { + source, destination := newRecurrentEnv(), newRecurrentEnv() + defer source.pc.close() + defer destination.pc.close() + inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9} + simulateRequest(t, source.pc, inputs, nil, 4, 8) + path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil))) + path[1].setSnapshots(nil, &source.pc.pagedOutBytes) + session := destination.pc.begin(inputs, nil, source.pc) + if !slices.Equal(session.remaining, inputs) { + t.Fatalf("reused an incomplete snapshot path: remaining %v", session.remaining) + } + session.close() +} + +func TestSharedPrefixSurvivesIntermediateEviction(t *testing.T) { + for name, newEnv := range map[string]func() *testEnv{ + "full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv, + } { + t.Run(name, func(t *testing.T) { + source, destination := newEnv(), newEnv() + defer source.pc.close() + defer destination.pc.close() + inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9} + simulateRequest(t, source.pc, inputs, nil, 4, 8) + path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil))) + if len(path) != 3 { + t.Fatalf("saved path has %d nodes, want root and two checkpoints", len(path)) + } + // Eviction merges the earlier edge into its child: continuous KV + // history must remain reusable, even though the earlier checkpoint is gone. + source.pc.evictNode(path[1]) + path, _ = findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil))) + if len(path) != 2 || path[1].endOffset != 8 || !hasAllSnapshots(path[1], source.caches) { + t.Fatal("eviction did not preserve a complete merged checkpoint at 8") + } + fork := destination.pc.begin(inputs, nil, source.pc) + defer fork.close() + if got := len(inputs) - len(fork.remaining); got != 8 { + t.Fatalf("reused %d tokens after intermediate eviction, want 8", got) + } + destination.assertAllTokens(t, "merged peer restore", inputs[:8]) + checkTrieInvariants(t, source.pc.root) + }) + } +} + +func TestSharedPrefixWhileDonorHasPendingCaptures(t *testing.T) { + for name, newEnv := range map[string]func() *testEnv{ + "full": newTransformerEnv, "sliding": newSlidingWindowEnv, "recurrent": newRecurrentEnv, + } { + t.Run(name, func(t *testing.T) { + source, destination := newEnv(), newEnv() + defer source.pc.close() + defer destination.pc.close() + old := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9} + simulateRequest(t, source.pc, old, nil, 4, 8) + // Give the fake snapshots a cost so borrowing triggers the donor's + // normal eviction policy while its new captures are still pending. + walkNodes(source.pc.root, func(n *trieNode) bool { + for _, s := range n.snapshots { + if s != nil { + s.(*fakeSnapshot).byteSize = 32 + source.pc.pagedOutBytes += 32 + } + } + return true + }) + next := []int32{1, 2, 3, 4, 30, 31, 32, 33, 34} + active := source.pc.begin(next, nil) + active.schedulePrefillSnapshots([]int{6, 8}) + feedAll(source.caches, next[4:6]) + source.pc.limitBytes = 1 + fork := destination.pc.begin(old, nil, source.pc) + defer fork.close() + if got := len(old) - len(fork.remaining); got != 8 { + t.Fatalf("reused %d tokens from old conversation, want 8", got) + } + _, matched := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(old, nil))) + if matched >= 8 { + t.Fatal("donor eviction did not remove the borrowed old checkpoint") + } + feedAll(source.caches, next[6:8]) + active.attachPrefillSnapshots() + active.close() + source.assertAllTokens(t, "continued donor prefill", next[:8]) + destination.assertAllTokens(t, "independent old prefix", old[:8]) + if len(active.pendingSnapshots) != 0 { + t.Fatal("donor did not drain its scheduled captures") + } + checkTrieInvariants(t, source.pc.root) + }) + } +} + +func TestSharedAttentionPrefixSurvivesSourceWrites(t *testing.T) { + for _, heads := range []int{1, 2} { + mlxtest.RunSubtest(t, fmt.Sprint(heads), func(t *mlxtest.T) { + newCache := func() *prefixCache { + return newPrefixCache([]cache.Cache{cache.NewKVCache(), cache.NewRotatingKVCache(4)}) + } + source, destination := newCache(), newCache() + defer destination.close() + inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9} + active := source.begin(inputs, nil) + active.schedulePrefillSnapshots([]int{4}) + write := func(pc *prefixCache, values []float32) { + mlx.Scoped(func() { + x := mlx.FromValues(slices.Repeat(values, heads), 1, heads, len(values), 1) + for _, layer := range pc.caches { + layer.(cache.Attention).Update(nil, x, x) + mlx.Eval(layer.State()...) + } + }) + } + write(source, []float32{1, 2, 3, 4}) + active.attachPrefillSnapshots() + write(source, []float32{5, 6, 7, 8}) + fork := destination.begin([]int32{1, 2, 3, 4, 20, 21}, nil, source) + if got := len(fork.inputs) - len(fork.remaining); got != 4 { + t.Fatalf("reused %d, want 4", got) + } + // Destroying donor snapshots and then its buffers must not change + // the destination, including one-head views that are contiguous. + source.close() + for _, layer := range destination.caches { + for _, a := range layer.State() { + got := mlx.Contiguous(a, false).Floats() + want := slices.Repeat([]float32{1, 2, 3, 4}, heads) + if !slices.Equal(got, want) { + t.Fatalf("restored %T state %v, want %v", layer, got, want) + } + } + } + write(destination, []float32{20}) + fork.close() + }) + } +} + +func TestSharedPrefixDraftLookahead(t *testing.T) { + for _, changeNext := range []bool{false, true} { + t.Run(fmt.Sprint(changeNext), func(t *testing.T) { + source, destination := newHybridEnv(), newHybridEnv() + defer source.pc.close() + defer destination.pc.close() + tokens := (&tokenStream{}).fresh(24) + source.runRequest(t, tokens, []int32{50, 51}) + fork := slices.Clone(tokens) + want := 19 + if changeNext { + // State at19 depends on token19 through the next-token drafter. + fork[19] = 100 + want = 15 + } + session := destination.pc.begin(fork, nil, source.pc) + if got := len(fork) - len(session.remaining); got != want { + t.Fatalf("reused %d, want %d", got, want) + } + for _, c := range destination.pc.caches { + assertTokens(t, "lookahead fork", c, tokens[:want]) + } + session.close() + }) + } +} + +func TestSharedPrefixMediaIdentity(t *testing.T) { + inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9} + items := []mediaItem{{pos: 4, length: 3, fold: 1 << 31, item: &model.PreparedItem{}}} + source := newRecurrentEnv() + defer source.pc.close() + active := source.pc.begin(inputs, items) + active.schedulePrefillSnapshots([]int{4, 7}) + feedAll(source.caches, inputs[:8]) + active.attachPrefillSnapshots() + for _, tc := range []struct { + name string + fold uint32 + maxReuse, want int + }{ + {"same image", 1 << 31, 8, 7}, + {"different image", 1<<31 | 1, 8, 4}, + {"inside atomic image", 1 << 31, 6, 4}, + } { + t.Run(tc.name, func(t *testing.T) { + destination := newRecurrentEnv() + defer destination.pc.close() + media := slices.Clone(items) + media[0].fold = tc.fold + session := destination.pc.beginAt(inputs, media, tc.maxReuse, source.pc) + if got := len(inputs) - len(session.remaining); got != tc.want { + t.Fatalf("reused %d, want %d", got, tc.want) + } + session.close() + }) + } + active.close() +} + +func TestSharedPrefixSurvivesCancelledFork(t *testing.T) { + source, destination := newRecurrentEnv(), newRecurrentEnv() + defer destination.pc.close() + inputs := []int32{1, 2, 3, 4, 5} + simulateRequest(t, source.pc, inputs, nil, 4) + fork := destination.pc.begin(inputs, nil, source.pc) + fork.close() // Cancel before forwarding any new tokens. + source.pc.close() + retry := destination.pc.begin(inputs, nil) + defer retry.close() + if got := len(inputs) - len(retry.remaining); got != 4 { + t.Fatalf("retry reused %d tokens, want 4 without the source", got) + } + destination.assertAllTokens(t, "cancelled fork", inputs[:4]) +} + +func TestSharedPrefixRestoreFailureRecomputes(t *testing.T) { + source, destination := newStatelessLayerEnv(), newStatelessLayerEnv() + defer source.pc.close() + defer destination.pc.close() + inputs := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9} + simulateRequest(t, source.pc, inputs, nil, 4, 8) + path, _ := findBestMatch(source.pc.root, source.pc.key(effectiveKeyTokens(inputs, nil))) + // The last attention snapshot cannot continue from the earlier edge. + // Recurrent state restores first, then cannot rewind to the attention + // offset after that layer rejects its snapshot. All layers must reset. + path[len(path)-1].snapshots[2].(*fakeSnapshot).from = 7 + fork := destination.pc.begin(inputs, nil, source.pc) + if !slices.Equal(fork.remaining, inputs) { + t.Fatalf("incomplete restore skipped input: remaining %v", fork.remaining) + } + destination.assertAllTokens(t, "failed restore", nil) + feedAll(destination.caches, inputs[:8]) + fork.close() + destination.assertAllTokens(t, "recomputed", inputs[:8]) +} diff --git a/mlxrunner/request_test.go b/mlxrunner/request_test.go index 04e103b82..2e53912b5 100644 --- a/mlxrunner/request_test.go +++ b/mlxrunner/request_test.go @@ -60,7 +60,7 @@ func TestSchedulerMixedRequests(t *testing.T) { generation := request{Ctx: ctx, Tokens: []int32{1, 1, 1}, Responses: make(chan CompletionResponse, 16), CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 8}}} var embedding []float32 evaluations := 0 - evaluation := &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot) (*evaluationState, error) { + evaluation := &evaluationRequest{ctx: ctx, done: make(chan error, 1), open: func(slot *batchSlot, peers []*prefixCache) (*evaluationState, error) { return r.openEvaluation(slot, &model.PreparedRequest{Tokens: []int32{2, 2}}, nil, func(hidden *mlx.Array) error { evaluations++ embedding = meanPoolAndNormalize(hidden.Reshape(2, mtpTestVocab), 2) diff --git a/mlxrunner/runner.go b/mlxrunner/runner.go index 47fa453be..8e37e0b0f 100644 --- a/mlxrunner/runner.go +++ b/mlxrunner/runner.go @@ -69,7 +69,7 @@ func completionStatusError(err error) *api.StatusError { // entering the token decode loop. Its callback runs on the MLX worker. type evaluationRequest struct { ctx context.Context //nolint:containedctx // Queued requests carry caller cancellation to the runner. - open func(*batchSlot) (*evaluationState, error) + open func(*batchSlot, []*prefixCache) (*evaluationState, error) done chan error } diff --git a/mlxrunner/scheduler.go b/mlxrunner/scheduler.go index 0b4d9c581..ccb67929a 100644 --- a/mlxrunner/scheduler.go +++ b/mlxrunner/scheduler.go @@ -284,7 +284,13 @@ func (s *batchScheduler) admitGeneration(request request) { s.slots[best] = slot } req := &batchRequest{request: request} - req.session = slot.cache.begin(request.Tokens, request.MediaItems) + var peers []*prefixCache + for _, other := range s.slots { + if other != nil && other != slot && other.cache != nil { + peers = append(peers, other.cache) + } + } + req.session = slot.cache.begin(request.Tokens, request.MediaItems, peers...) req.media = s.r.openMedia(request) req.spec = slot.speculation.open(request, req.media.rowLayout()) req.prefill = s.r.newPrefill(req.session, req.spec, req.media) diff --git a/mlxrunner/scheduler_test.go b/mlxrunner/scheduler_test.go index d8faf44a3..73dbabf29 100644 --- a/mlxrunner/scheduler_test.go +++ b/mlxrunner/scheduler_test.go @@ -842,3 +842,67 @@ func TestBatchSchedulerLearnsDraftDepth(t *testing.T) { }) } } + +func TestGenerationReusesBusyPeer(t *testing.T) { + ctx := t.Context() + mlxtest.Run(t, func(t *mlxtest.T) { + tok := newTestTokenizer(t, []int32{7}) + m := &schedulingModel{fakeMTPModel: fakeMTPModel{tok: tok}} + r := &Runner{Model: m, Tokenizer: tok, Sampler: sample.New(4096), parallel: 2} + defer r.Sampler.Free() + s := &batchScheduler{r: r, slots: make([]*batchSlot, 2)} + defer func() { + for _, slot := range s.slots { + if slot != nil && slot.cache != nil { + slot.cache.close() + } + } + }() + makeRequest := func(ctx context.Context, count int) request { + return request{ + Ctx: ctx, Tokens: slices.Repeat([]int32{1}, 32), Responses: make(chan CompletionResponse, 64), + CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: count}}, SamplerOpts: sample.Options{Temperature: 0}, + } + } + sourceCtx, cancel := context.WithCancel(ctx) + defer cancel() + source := makeRequest(sourceCtx, 32) + s.admitGeneration(source) + for range 8 { + s.step() + if len(source.Responses) > 0 { + break + } + } + if len(source.Responses) == 0 { + t.Fatal("source never started streaming") + } + fork := makeRequest(ctx, 2) + s.admitGeneration(fork) + if got := len(fork.Tokens) - len(s.slots[1].request.session.remaining); got != 28 { + t.Fatalf("busy-peer reuse %d, want pre-thinking snapshot28", got) + } + cancel() + var final CompletionResponse + var text string + for i := 0; i < 16 && !final.Done; i++ { + s.step() + for len(fork.Responses) > 0 { + response := <-fork.Responses + if response.Error != nil { + t.Fatal(response.Error) + } + text += response.Content + if response.Done { + final = response + } + } + } + if !final.Done || final.EvalCount != 2 || final.PromptEvalCachedCount == nil || *final.PromptEvalCachedCount != 28 { + t.Fatalf("fork completion: %+v", final) + } + if want := tok.Decode([]int32{1, 1}); text != want { + t.Fatalf("fork output %q, want %q", text, want) + } + }) +}