From d63b702609c4300a2ca07be2bdb0adb9834725e5 Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Sat, 7 Nov 2026 16:36:53 -0800 Subject: [PATCH] Add ripple preview support to the MLX runner (#19592) Add the Ripple model, renderer, parser and import registration. Support per-layer NoPE in DFlash drafts. Fuse MLP activation, value blending and attention gating. Add reference fixtures and model/runner regression tests. --- create/metadata.go | 4 + create/metadata_test.go | 17 + .../model/architectures/architectures.go | 1 + mlxrunner/model/dflash/cache_test.go | 294 ++++ mlxrunner/model/dflash/dflash.go | 37 +- mlxrunner/model/dflash/dflash2_test.go | 83 + mlxrunner/model/dflash/dflash_test.go | 46 + mlxrunner/model/ripple/forward.go | 156 ++ mlxrunner/model/ripple/ripple.go | 318 ++++ mlxrunner/model/ripple/ripple_test.go | 1393 +++++++++++++++++ mlxrunner/ripple_test.go | 681 ++++++++ mlxrunner/testdata/generate_qwen35_yarn.py | 194 --- mlxrunner/verify_test.go | 6 + model/parsers/parsers.go | 2 + model/parsers/ripple.go | 184 +++ model/parsers/ripple_test.go | 228 +++ model/renderers/renderer.go | 2 + model/renderers/ripple.go | 88 ++ model/renderers/ripple_test.go | 292 ++++ .../testdata/ripple_chat_template.jinja | 52 + 20 files changed, 3878 insertions(+), 200 deletions(-) create mode 100644 mlxrunner/model/ripple/forward.go create mode 100644 mlxrunner/model/ripple/ripple.go create mode 100644 mlxrunner/model/ripple/ripple_test.go create mode 100644 mlxrunner/ripple_test.go delete mode 100644 mlxrunner/testdata/generate_qwen35_yarn.py create mode 100644 model/parsers/ripple.go create mode 100644 model/parsers/ripple_test.go create mode 100644 model/renderers/ripple.go create mode 100644 model/renderers/ripple_test.go create mode 100644 model/renderers/testdata/ripple_chat_template.jinja diff --git a/create/metadata.go b/create/metadata.go index bce1c7742..84e2d7336 100644 --- a/create/metadata.go +++ b/create/metadata.go @@ -282,6 +282,8 @@ func parserNameForConfig(modelDir string, cfg sourceModelConfig, chatTemplate st func parserNameForIdentifier(modelDir, s, chatTemplate string) (string, error) { s = strings.ToLower(s) switch { + case s == "ricursiveforcausallm" || s == "ricursive": + return "ripple", nil case strings.Contains(s, "flint"): return "flint", nil case strings.HasPrefix(s, "lfm3"): @@ -322,6 +324,8 @@ func rendererNameForConfig(modelDir string, cfg sourceModelConfig, chatTemplate func rendererNameForIdentifier(modelDir, s, chatTemplate string) (string, error) { s = strings.ToLower(s) switch { + case s == "ricursiveforcausallm" || s == "ricursive": + return "ripple", nil case strings.Contains(s, "flint"): return "flint", nil case strings.HasPrefix(s, "lfm3"): diff --git a/create/metadata_test.go b/create/metadata_test.go index d3872d1af..ccf3d6382 100644 --- a/create/metadata_test.go +++ b/create/metadata_test.go @@ -102,6 +102,23 @@ func TestInferSafetensorsConfigFamilies(t *testing.T) { wantRenderer string wantCaps []string }{ + { + name: "ripple architecture", + config: `{"architectures":["RicursiveForCausalLM"],"model_type":"ricursive"}`, + chatTemplate: `{% for message in messages %}{{ message.content }}{% endfor %}`, + wantParser: "ripple", + wantRenderer: "ripple", + wantCaps: []string{"completion", "tools", "thinking"}, + }, + { + name: "ripple model type", + config: `{"model_type":"ricursive"}`, + chatTemplate: `{{ reasoning_content }}{{ content }}`, + standalone: true, + wantParser: "ripple", + wantRenderer: "ripple", + wantCaps: []string{"completion", "tools", "thinking"}, + }, { name: "qwen3", config: `{"architectures":["Qwen3ForCausalLM"]}`, diff --git a/mlxrunner/model/architectures/architectures.go b/mlxrunner/model/architectures/architectures.go index 3b9d5d9a5..625324b7a 100644 --- a/mlxrunner/model/architectures/architectures.go +++ b/mlxrunner/model/architectures/architectures.go @@ -23,5 +23,6 @@ import ( _ "github.com/ollama/ollama/mlxrunner/model/qwen3_5" _ "github.com/ollama/ollama/mlxrunner/model/qwen3_5_moe" _ "github.com/ollama/ollama/mlxrunner/model/qwen4_exp" + _ "github.com/ollama/ollama/mlxrunner/model/ripple" _ "github.com/ollama/ollama/mlxrunner/model/strands" ) diff --git a/mlxrunner/model/dflash/cache_test.go b/mlxrunner/model/dflash/cache_test.go index b99d15fc8..1d4ff56f8 100644 --- a/mlxrunner/model/dflash/cache_test.go +++ b/mlxrunner/model/dflash/cache_test.go @@ -3,12 +3,14 @@ package dflash import ( "fmt" "math" + "slices" "testing" "github.com/ollama/ollama/mlx" "github.com/ollama/ollama/mlx/mlxtest" "github.com/ollama/ollama/mlxrunner/batch" "github.com/ollama/ollama/mlxrunner/cache" + "github.com/ollama/ollama/mlxrunner/model" "github.com/ollama/ollama/mlxrunner/nn" ) @@ -120,6 +122,298 @@ func TestRepeatedBlockRollback(t *testing.T) { } } +// The scalar oracle uses nonzero queries and checks both cache K/V and output, +// so skipping only context K, block K, or block Q cannot satisfy the contract. +func TestDraftPerLayerNoRoPE(t *testing.T) { + for _, policy := range []struct { + name, field string + noRoPE [2]bool + }{ + {"omitted", "", [2]bool{}}, + {"empty", `,"no_rope_layers":[]`, [2]bool{}}, + {"full_nope", `,"no_rope_layers":[1]`, [2]bool{false, true}}, + {"sliding_nope", `,"no_rope_layers":[0]`, [2]bool{true, false}}, + } { + for _, path := range []struct { + name string + context, block bool + }{ + {"context_only", true, false}, + {"block_only", false, true}, + {"combined", true, true}, + } { + mlxtest.RunSubtest(t, policy.name+"/"+path.name, func(t *mlxtest.T) { + mlx.Scoped(func() { + m := newDraftRotaryTestModel(t, policy.field) + caches := m.NewCaches() + for _, c := range caches { + defer c.Free() + } + var oracle draftRotaryOracle + // Seed the real caches without invoking either rotary path. + // Block-only must not fail because context maintenance is broken. + seed := []float32{0.25, -0.5, 0.75, 0.25, -0.5, 1} + for i, c := range caches { + c.(cache.Attention).Update(&batch.Batch{SeqOffsets: []int32{0}}, + mlx.Zeros(mlx.DTypeFloat32, 1, 1, 3, 2), mlx.FromValues(seed, 1, 1, 3, 2)) + oracle.keys[i] = make([]float64, len(seed)) + for _, v := range seed { + oracle.values[i] = append(oracle.values[i], float64(v)) + } + } + for round := range 2 { + start := caches[0].Offset() + b := &batch.Batch{SeqOffsets: []int32{int32(start)}} + var context, block []float64 + if path.context { + context = []float64{1, 2, -3, 1} + b.Hidden = mlx.FromValues([]float32{1, 2, -3, 1}, 1, 2, 2) + } + if path.block { + block = []float64{1, 2, -2, 1, 0.5, -1} + b.InputIDs = mlx.FromValues([]int32{0, 1, 2}, 1, 3) + b.SeqQueryLens = []int32{3} + } + want := oracle.forward(context, block, start, policy.noRoPE) + hidden, aux := m.Forward(b, nil, caches) + label := fmt.Sprintf("round %d offset %d", round, start) + if !path.block { + if hidden != nil || aux != nil { + t.Fatal("context-only forward returned block hidden states") + } + } else { + if hidden == nil || aux != hidden { + t.Fatal("block forward did not return shared hidden/aux states") + } + checkDraftRotaryArray(t, label+" hidden", hidden, want) + } + for i, c := range caches { + if wantOffset := len(oracle.keys[i]) / 2; c.Offset() != wantOffset { + t.Fatalf("%s layer %d cache offset=%d want=%d", label, i, c.Offset(), wantOffset) + } + state := c.State() + if len(state) != 2 { + t.Fatalf("%s layer %d state roots=%d want=2", label, i, len(state)) + } + checkDraftRotaryArray(t, fmt.Sprintf("%s layer %d keys", label, i), state[0], oracle.keys[i]) + checkDraftRotaryArray(t, fmt.Sprintf("%s layer %d values", label, i), state[1], oracle.values[i]) + } + } + }) + }) + } + } +} + +type windowedDraftTarget struct { + *draftRotaryTarget + window int +} + +func (t *windowedDraftTarget) DraftWindow() int { return t.window } + +func checkDraftWindowValues(t *mlxtest.T, label string, array *mlx.Array, want []float64) { + t.Helper() + array = mlx.Contiguous(array, false).AsType(mlx.DTypeFloat32) + mlx.Eval(array) + got := array.Floats() + wantFloat := make([]float32, len(want)) + for i, v := range want { + wantFloat[i] = float32(v) + } + slices.Sort(got) + slices.Sort(wantFloat) + if !slices.Equal(got, wantFloat) { + t.Fatalf("%s values = %v, want %v", label, got, wantFloat) + } +} + +func TestDraftWindowCache(t *testing.T) { + for _, tc := range []struct { + name string + target draftTarget + initial, wrapped [][]float64 + }{ + { + name: "default", target: &draftRotaryTarget{}, + initial: [][]float64{{2, 3}, {0, 1, 2, 3}}, + wrapped: [][]float64{{3, 4}, {0, 1, 2, 3, 4}}, + }, + { + name: "bounded", target: &windowedDraftTarget{draftRotaryTarget: &draftRotaryTarget{}, window: 3}, + initial: [][]float64{{2, 3}, {1, 2, 3}}, + wrapped: [][]float64{{3, 4}, {2, 3, 4}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + m := &Model{ + Config: &Config{SlidingWindow: 2}, + Layers: []*Layer{{IsSliding: true}, {IsSliding: false}}, + target: tc.target, + } + caches := m.NewCaches() + defer func() { + for _, c := range caches { + c.Free() + } + }() + initial := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{4}} + values := mlx.FromValues([]float32{0, 1, 2, 3}, 1, 1, 4, 1) + for i, c := range caches { + c.(cache.Attention).Update(initial, values, values) + checkDraftWindowValues(t, fmt.Sprintf("initial cache %d", i), c.(cache.Attention).View(initial).V(), tc.initial[i]) + c.PrepareSnapshots([]int{4}) + next := &batch.Batch{SeqOffsets: []int32{4}, SeqQueryLens: []int32{1}} + value := mlx.FromValue(float32(4)) + value = mlx.Reshape(value, 1, 1, 1, 1) + c.(cache.Attention).Update(next, value, value) + checkDraftWindowValues(t, fmt.Sprintf("wrapped cache %d", i), c.(cache.Attention).View(next).V(), tc.wrapped[i]) + snapshots := c.TakeSnapshots() + if len(snapshots) != 1 { + t.Fatalf("cache %d snapshots = %d", i, len(snapshots)) + } + if !c.Restore(snapshots[0], 4) { + t.Fatalf("cache %d could not restore", i) + } + if snapshots[0] != nil { + snapshots[0].Close() + } + if c.Offset() != 4 { + t.Fatalf("cache %d offset = %d", i, c.Offset()) + } + checkDraftWindowValues(t, fmt.Sprintf("restored cache %d", i), c.(cache.Attention).View(initial).V(), tc.initial[i]) + } + }) + }) + } +} + +type draftRotaryTarget struct { + model.Model + embeddings *mlx.Array +} + +func (m *draftRotaryTarget) TokenEmbeddings(ids *mlx.Array) *mlx.Array { + return m.embeddings.TakeAxis(ids, 0) +} + +func (*draftRotaryTarget) RawLogits(x *mlx.Array) *mlx.Array { return x } +func (*draftRotaryTarget) SetAuxHiddenLayers([]int) {} +func (*draftRotaryTarget) NumLayers() int { return 2 } + +func newDraftRotaryTestModel(t *mlxtest.T, field string) *Model { + t.Helper() + cfg, err := parseConfig(draftRotaryTestConfig(field)) + if err != nil { + t.Fatal(err) + } + m := &Model{ + Config: cfg, Layers: make([]*Layer, cfg.NumHiddenLayers), tensorPrefix: "draft.", + target: &draftRotaryTarget{embeddings: mlx.FromValues([]float32{1, 2, -2, 1, 0.5, -1}, 3, 2)}, + } + identity := mlx.FromValues([]float32{1, 0, 0, 1}, 2, 2) + ones := mlx.FromValues([]float32{1, 1}, 2) + zero := mlx.Zeros(mlx.DTypeFloat32, 2, 2) + tensors := map[string]*mlx.Array{ + "draft.fc.weight": identity, "draft.hidden_norm.weight": ones, "draft.norm.weight": ones, + } + for i := range m.Layers { + prefix := fmt.Sprintf("draft.layers.%d.", i) + for _, name := range []string{"q_proj", "k_proj", "v_proj", "o_proj"} { + tensors[prefix+"self_attn."+name+".weight"] = identity + } + for _, name := range []string{"input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm"} { + tensors[prefix+name+".weight"] = ones + } + for _, name := range []string{"gate_proj", "up_proj", "down_proj"} { + tensors[prefix+"mlp."+name+".weight"] = zero + } + } + if err := m.LoadWeights(tensors); err != nil { + t.Fatal(err) + } + return m +} + +func checkDraftRotaryArray(t *mlxtest.T, label string, array *mlx.Array, want []float64) { + t.Helper() + // Cache state can be a strided view; Floats requires contiguous storage. + array = mlx.Contiguous(array, false).AsType(mlx.DTypeFloat32) + mlx.Eval(array) + got := array.Floats() + if len(got) != len(want) { + t.Fatalf("%s length=%d want=%d", label, len(got), len(want)) + } + for i, v := range got { + if math.IsNaN(float64(v)) || math.Abs(float64(v)-want[i]) > 2e-3*math.Max(1, math.Abs(want[i])) { + t.Errorf("%s[%d]: got %g want %g", label, i, v, want[i]) + return + } + } +} + +// This two-wide scalar model has identity projections, unit norms, zero MLPs, +// and a window larger than its history. It does not call production RoPE/SDPA. +type draftRotaryOracle struct { + keys, values [2][]float64 +} + +func draftRotaryNorm(x []float64) []float64 { + out := make([]float64, len(x)) + for r := 0; r < len(x); r += 2 { + inv := 1 / math.Sqrt((x[r]*x[r]+x[r+1]*x[r+1])/2+1e-6) + out[r], out[r+1] = x[r]*inv, x[r+1]*inv + } + return out +} + +func draftRotaryRotate(x []float64, start int) []float64 { + out := slices.Clone(x) + for r := 0; r < len(x); r += 2 { + s, c := math.Sincos(float64(start + r/2)) + out[r], out[r+1] = x[r]*c-x[r+1]*s, x[r]*s+x[r+1]*c + } + return out +} + +func (o *draftRotaryOracle) forward(context, block []float64, start int, noRoPE [2]bool) []float64 { + contextValues := draftRotaryNorm(context) + x := slices.Clone(block) + for layer := range o.keys { + contextKeys := draftRotaryNorm(contextValues) + values := draftRotaryNorm(x) + q := draftRotaryNorm(values) + k := slices.Clone(q) + if !noRoPE[layer] { + contextKeys = draftRotaryRotate(contextKeys, start) + q = draftRotaryRotate(q, start+len(context)/2) + k = draftRotaryRotate(k, start+len(context)/2) + } + o.keys[layer] = append(o.keys[layer], contextKeys...) + o.keys[layer] = append(o.keys[layer], k...) + o.values[layer] = append(o.values[layer], contextValues...) + o.values[layer] = append(o.values[layer], values...) + for r := 0; r < len(x); r += 2 { + scores := make([]float64, len(o.keys[layer])/2) + for j := range scores { + scores[j] = (q[r]*o.keys[layer][2*j] + q[r+1]*o.keys[layer][2*j+1]) / math.Sqrt(2) + } + maxScore := slices.Max(scores) + total := 0.0 + for j, score := range scores { + scores[j] = math.Exp(score - maxScore) + total += scores[j] + } + for j, score := range scores { + x[r] += score / total * o.values[layer][2*j] + x[r+1] += score / total * o.values[layer][2*j+1] + } + } + } + return draftRotaryNorm(x) +} + // meanAttention is one head of zero queries over width-wide rows: every output // row is the mean of the value rows it may see, and values copy their input. func meanAttention(width int, window int32) (*Config, *Attention) { diff --git a/mlxrunner/model/dflash/dflash.go b/mlxrunner/model/dflash/dflash.go index 5056844ae..04744f98a 100644 --- a/mlxrunner/model/dflash/dflash.go +++ b/mlxrunner/model/dflash/dflash.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "math" + "slices" "github.com/ollama/ollama/mlx" "github.com/ollama/ollama/mlxrunner/batch" @@ -49,6 +50,7 @@ type Config struct { Scale float32 SlidingWindow int32 LayerTypes []string + NoRoPELayers []int BlockSize int MaskTokenID int32 @@ -143,6 +145,7 @@ type Attention struct { OProj nn.LinearLayer QNorm *nn.RMSNorm KNorm *nn.RMSNorm + NoRoPE bool // ctxInputNorm applies the layer's input norm to context rows (laguna). ctxInputNorm *nn.RMSNorm @@ -183,6 +186,7 @@ func parseConfig(data []byte) (*Config, error) { VocabSize int32 `json:"vocab_size"` HasOwnLMHead bool `json:"has_own_lm_head"` LayerTypes []string `json:"layer_types"` + NoRoPELayers []int `json:"no_rope_layers"` SlidingWindow int32 `json:"sliding_window"` RopeInterleaved *bool `json:"rope_interleaved"` } @@ -200,6 +204,7 @@ func parseConfig(data []byte) (*Config, error) { RopeTheta: raw.RopeTheta, SlidingWindow: raw.SlidingWindow, LayerTypes: raw.LayerTypes, + NoRoPELayers: raw.NoRoPELayers, BlockSize: raw.DFlashConfig.BlockSize, VocabSize: raw.VocabSize, HasOwnLMHead: raw.HasOwnLMHead, @@ -217,6 +222,11 @@ func parseConfig(data []byte) (*Config, error) { if cfg.NumHiddenLayers > maxDraftLayers { return nil, fmt.Errorf("dflash num_hidden_layers %d exceeds supported maximum %d", cfg.NumHiddenLayers, maxDraftLayers) } + for _, id := range cfg.NoRoPELayers { + if id < 0 || id >= int(cfg.NumHiddenLayers) { + return nil, fmt.Errorf("dflash no_rope_layers index %d out of range for %d layers", id, cfg.NumHiddenLayers) + } + } // The fused projection holds the query, key and value rows. if (int64(cfg.NumAttentionHeads)+2*int64(cfg.NumKeyValueHeads))*int64(cfg.HeadDim) > math.MaxInt32 { return nil, fmt.Errorf("dflash attention projection dimensions exceed int32") @@ -391,8 +401,9 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error { layer := &Layer{ IsSliding: m.LayerTypes[i] == "sliding_attention", Attention: &Attention{ - GProj: linears.Make(layerPrefix + ".self_attn.g_proj"), - OProj: linears.Make(layerPrefix + ".self_attn.o_proj"), + NoRoPE: slices.Contains(m.NoRoPELayers, i), + GProj: linears.Make(layerPrefix + ".self_attn.g_proj"), + OProj: linears.Make(layerPrefix + ".self_attn.o_proj"), }, MLP: &MLP{ DownProj: linears.Make(layerPrefix + ".mlp.down_proj"), @@ -583,9 +594,19 @@ func (m *Model) BlockParams() (int, int32) { return m.BlockSize, m.MaskTokenID } // NewCaches builds the per-layer context caches. func (m *Model) NewCaches() []cache.Cache { caches := make([]cache.Cache, len(m.Layers)) + draftWindow := 0 + if target, ok := m.target.(interface{ DraftWindow() int }); ok { + draftWindow = target.DraftWindow() + } for i, layer := range m.Layers { if layer.IsSliding { - caches[i] = cache.NewRotatingKVCache(int(m.SlidingWindow)) + window := int(m.SlidingWindow) + if draftWindow > 0 { + window = min(window, draftWindow) + } + caches[i] = cache.NewRotatingKVCache(window) + } else if draftWindow > 0 { + caches[i] = cache.NewRotatingKVCache(draftWindow) } else { caches[i] = cache.NewKVCache() } @@ -681,7 +702,9 @@ func (a *Attention) contextKV(hctx *mlx.Array, positions *mlx.Array, cfg *Config k = a.KNorm.Forward(k, cfg.RMSNormEps) k = mlx.Transpose(k, 0, 2, 1, 3) v = mlx.Transpose(v, 0, 2, 1, 3) - k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions) + if !a.NoRoPE { + k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions) + } return k, v } @@ -703,8 +726,10 @@ func (a *Attention) Forward(x, ctxK, ctxV *mlx.Array, c cache.Cache, bb *batch.B k = mlx.Transpose(k, 0, 2, 1, 3) v = mlx.Transpose(v, 0, 2, 1, 3) - q = mlx.RoPEWithBase(q, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions) - k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions) + if !a.NoRoPE { + q = mlx.RoPEWithBase(q, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions) + k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions) + } if ctxK != nil { k = ctxK.Concatenate(2, k) diff --git a/mlxrunner/model/dflash/dflash2_test.go b/mlxrunner/model/dflash/dflash2_test.go index 13296a2a7..07840a59d 100644 --- a/mlxrunner/model/dflash/dflash2_test.go +++ b/mlxrunner/model/dflash/dflash2_test.go @@ -241,6 +241,89 @@ func TestSelectorConditionalProbabilities(t *testing.T) { } } +func TestFourTapConvolutionOracle(t *testing.T) { + for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} { + for _, length := range []int{1, 2, 3, 4, 8, 16} { + mlxtest.RunSubtest(t, fmt.Sprintf("%v length %d", dtype, length), func(t *mlxtest.T) { + const width, group, taps = 32, 16, 4 + values := func(n int) []float32 { + v := make([]float32, n) + for i := range v { + v[i] = float32(i%7-3) / 8 + } + return v + } + h, d, s := values(length*width), values(length*taps*width/group), values(taps*width) + want := convolutionOracle(h, d, s, 1, length, width, taps, width/group, func(v float32) float32 { + return rounded(v, dtype == mlx.DTypeBFloat16) + }) + inputs := []*mlx.Array{mlx.FromValues(h, 1, length, width).AsType(dtype), mlx.FromValues(d, 1, length, taps, width/group).AsType(dtype), mlx.FromValues(s, taps, width).AsType(dtype)} + for _, got := range []*mlx.Array{convolveGraph(inputs[0], inputs[1], inputs[2]), convolve(inputs[0], inputs[1], inputs[2])} { + got = mlx.Contiguous(got.AsType(mlx.DTypeFloat32), false) + mlx.Eval(got) + if !slices.Equal(got.Floats(), want) { + t.Fatal("four-tap convolution differs from scalar oracle") + } + } + }) + } + } +} + +func TestSelectorTop32Probabilities(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + const vocab, rank, k, length = 40, 256, 32, 3 + pred, succ, projection := make([]float32, vocab*rank), make([]float32, vocab*rank), make([]float32, rank) + for id := range vocab { + for j := range rank { + pred[id*rank+j] = float32(id%3-1) / 32 + succ[id*rank+j] = float32(id%5-2) / 64 + } + } + for j := range projection { + projection[j] = 0.5 + } + logits := make([]float32, length*vocab) + for i := range logits { + logits[i] = -float32(i%vocab) / 8 + } + m := &model2{Selector: &candidateSelector{ + Projection: nn.NewLinear(mlx.FromValues(projection, rank, 1).AsType(mlx.DTypeBFloat16), nil), + Predecessor: mlx.FromValues(pred, vocab, rank).AsType(mlx.DTypeBFloat16), + Successor: mlx.FromValues(succ, vocab, rank).AsType(mlx.DTypeBFloat16), TopK: k, + }} + sampler := sample.New(16) + defer sampler.Free() + sampler.Add(0, sample.Options{Temperature: 1, UseSeed: true, Seed: 42}, nil) + tokens, ids, probs := m.Select(mlx.FromValues([]float32{1, 1, 1}, 1, length, 1).AsType(mlx.DTypeBFloat16), mlx.FromValues(logits, 1, length, vocab).AsType(mlx.DTypeBFloat16), mlx.FromValues([]int32{0}, 1), func(ids, scores *mlx.Array) (*mlx.Array, *mlx.Array) { + tokens, distribution := sampler.SampleCandidates(0, ids, scores) + return tokens, distribution.Probs + }) + mlx.Eval(tokens, ids, probs) + path, support, q := tokens.Ints(), ids.Ints(), mlx.Contiguous(probs, false).Floats() + previous := 0 + for row := range length { + weights, total := make([]float64, k), float64(0) + for id := range k { + weights[id] = math.Exp(-float64(id)/8 + float64((previous%3-1)*(id%5-2))/16) + total += weights[id] + } + for j, id := range support[row*k : (row+1)*k] { + if id < 0 || id >= k { + t.Fatalf("unexpected candidate %d", id) + } + if want := weights[id] / total; math.Abs(float64(q[row*k+j])-want) > 1e-6 { + t.Fatalf("row %d candidate %d probability=%g want=%g", row, id, q[row*k+j], want) + } + } + if !slices.Contains(support[row*k:(row+1)*k], path[row]) { + t.Fatal("selected token outside proposal support") + } + previous = int(path[row]) + } + }) +} + func TestDFlash2ConfigValidation(t *testing.T) { valid := Config{HiddenSize: 5120, ConvGroupSize: 16, ConvKernelSize: 2, BlockSize: 8, SelectorRank: 256, SelectorTopK: 16, VocabSize: 248320} for _, tt := range []struct { diff --git a/mlxrunner/model/dflash/dflash_test.go b/mlxrunner/model/dflash/dflash_test.go index 42654a485..0cc503f43 100644 --- a/mlxrunner/model/dflash/dflash_test.go +++ b/mlxrunner/model/dflash/dflash_test.go @@ -2,9 +2,11 @@ package dflash import ( "encoding/json" + "fmt" "maps" "math" "slices" + "strings" "testing" ) @@ -50,6 +52,50 @@ func TestDraftConfigBoundsAndCausality(t *testing.T) { } } +// Keep the policy in JSON so these tests exercise parsing and loader wiring. +func draftRotaryTestConfig(noRoPEField string) []byte { + return []byte(fmt.Sprintf(`{ + "hidden_size": 2, "num_hidden_layers": 2, + "num_attention_heads": 1, "num_key_value_heads": 1, "head_dim": 2, + "rms_norm_eps": 0.000001, "rope_theta": 10000, + "layer_types": ["sliding_attention", "full_attention"], "sliding_window": 64, + "dflash_config": {"block_size": 4, "mask_token_id": 0, "target_layer_ids": [0], "causal": false} + %s + }`, noRoPEField)) +} + +func TestDraftConfigNoRoPELayers(t *testing.T) { + for _, tt := range []struct { + name, field string + valid bool + }{ + {"omitted", "", true}, + {"empty", `,"no_rope_layers":[]`, true}, + {"first", `,"no_rope_layers":[0]`, true}, + {"last", `,"no_rope_layers":[1]`, true}, + {"unordered", `,"no_rope_layers":[1,0]`, true}, + {"negative", `,"no_rope_layers":[-1]`, false}, + {"past_last", `,"no_rope_layers":[2]`, false}, + {"invalid_after_valid", `,"no_rope_layers":[0,2]`, false}, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := parseConfig(draftRotaryTestConfig(tt.field)) + if tt.valid { + if err != nil { + t.Fatal(err) + } + return + } + if err == nil { + t.Fatalf("parseConfig accepted invalid %s", tt.field) + } + if !strings.Contains(err.Error(), "no_rope_layers") { + t.Fatalf("error %q does not identify no_rope_layers", err) + } + }) + } +} + func FuzzDraftConfig(f *testing.F) { f.Add([]byte(`{"hidden_size":32,"num_hidden_layers":2,"num_attention_heads":4,"num_key_value_heads":2,"head_dim":8,"is_causal":false,"vocab_size":64,"dflash_config":{"block_size":8,"mask_token_id":7,"target_layer_ids":[1,3],"conv_group_size":16,"conv_kernel_size":2,"selector_rank":4,"selector_top_k":2}}`)) f.Add([]byte(`{"num_hidden_layers":-1}`)) diff --git a/mlxrunner/model/ripple/forward.go b/mlxrunner/model/ripple/forward.go new file mode 100644 index 000000000..957446331 --- /dev/null +++ b/mlxrunner/model/ripple/forward.go @@ -0,0 +1,156 @@ +package ripple + +import ( + "math" + + "github.com/ollama/ollama/mlx" + "github.com/ollama/ollama/mlxrunner/batch" + "github.com/ollama/ollama/mlxrunner/cache" + "github.com/ollama/ollama/mlxrunner/nn" +) + +func (m *Model) Forward(b *batch.Batch, caches []cache.Cache) (hidden, auxHidden *mlx.Array) { + if caches != nil && len(caches) < len(m.Layers) { + panic("ripple: cache count must cover target layers") + } + if caches != nil && b.InputIDs.Dim(0) != 1 { + panic("ripple: cached batches require one sequence") + } + h := scaleEmbedding(m.EmbedTokens.Forward(b.InputIDs), m.EmbedScale) + seed := h + positions := mlx.Add(mlx.Reshape(mlx.FromValues(b.SeqOffsets, len(b.SeqOffsets)), int32(len(b.SeqOffsets)), 1), mlx.Arange(0, float64(b.InputIDs.Dim(1)), 1, mlx.DTypeInt32)) + cos, sin := rotaryTables(positions, m.HeadDim, m.RopeTheta, h.DType()) + var features []*mlx.Array + for i, layer := range m.Layers { + var c cache.Cache + if caches != nil { + c = caches[i] + } + h = layer.Forward(h, seed, b, c, cos, sin, m.LayerTypes[i] == "sliding_attention", m.Config) + if next := len(features); next < len(m.auxHiddenLayers) && i == m.auxHiddenLayers[next] { + features = append(features, h) + } + } + h = rmsNorm(h, m.Norm, m.RMSNormEps) + if len(features) > 0 { + return h, mlx.Concatenate(features, -1) + } + return h, h +} + +func (l *Layer) Forward(x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array { + u := rmsNorm(x, l.InputNorm, cfg.RMSNormEps) + a := l.Attention.Forward(u, seed, b, c, cos, sin, local, cfg) + h := mlx.Add(x, rmsNorm(a, l.PostAttentionNorm, cfg.RMSNormEps)) + f := l.MLP.Forward(rmsNorm(h, l.PreFFNNorm, cfg.RMSNormEps), cfg.FFNClampThreshold) + return mlx.Add(h, rmsNorm(f, l.PostFFNNorm, cfg.RMSNormEps)) +} + +func (a *Attention) Forward(x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array { + B, L := int32(x.Dim(0)), int32(x.Dim(1)) + q := mlx.Reshape(a.QProj.Forward(x), B, L, cfg.NumAttentionHeads, cfg.HeadDim) + k := mlx.Reshape(a.KProj.Forward(x), B, L, cfg.NumKeyValueHeads, cfg.HeadDim) + v := mlx.Reshape(a.VProj.Forward(blendValues(seed, x, a.ValueResLambda)), B, L, cfg.NumKeyValueHeads, cfg.HeadDim) + q = mlx.Transpose(rmsNorm(q, a.QNorm, cfg.RMSNormEps), 0, 2, 1, 3) + k = mlx.Transpose(rmsNorm(k, a.KNorm, cfg.RMSNormEps), 0, 2, 1, 3) + v = mlx.Transpose(v, 0, 2, 1, 3) + if local { + q, k = applyRotary(q, cos, sin), applyRotary(k, cos, sin) + } + mask := nn.CausalMask() + var kv nn.SDPAOption + if c != nil { + kv = nn.WithKVHistory(c.(cache.Attention).Update(b, k, v)) + } else { + kv = nn.WithKV(k, v, b.SeqQueryLens) + if local { + mask = mask.Intersect(nn.SlidingWindowMask(b, int(L), cfg.SlidingWindow, k.DType())) + } + } + out := nn.ScaledDotProductAttention(b, q, cfg.Scale, kv, nn.WithMask(mask)) + out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim) + return a.OProj.Forward(gatedAttention(out, a.GProj.Forward(x))) +} + +// MLP is the clipped SwiGLU feed-forward block. +type MLP struct { + GateProj, UpProj, DownProj nn.LinearLayer +} + +var ffnActivation = mlx.Compile2("SwiGLUF32", func(gate, up *mlx.Array) *mlx.Array { + f := gate.AsType(mlx.DTypeFloat32) + silu := mlx.Mul(f, mlx.Sigmoid(f)).AsType(gate.DType()) + return mlx.Mul(silu, up) +}, mlx.Shapeless()) + +var clippedFFNActivation = mlx.Compile3("ClippedSwiGLUF32", func(gate, up, threshold *mlx.Array) *mlx.Array { + gate = mlx.Minimum(mlx.Maximum(gate, mlx.FromValue(float32(math.Inf(-1)))), threshold).AsType(gate.DType()) + up = mlx.Minimum(mlx.Maximum(up, threshold.Negative()), threshold).AsType(up.DType()) + return ffnActivation(gate, up) +}, mlx.Shapeless()) + +func (m *MLP) Forward(x *mlx.Array, threshold *float32) *mlx.Array { + gate, up := m.GateProj.Forward(x), m.UpProj.Forward(x) + var activated *mlx.Array + if threshold != nil && *threshold > 0 { + activated = clippedFFNActivation(gate, up, mlx.FromValue(*threshold)) + } else { + activated = ffnActivation(gate, up) + } + return m.DownProj.Forward(activated) +} + +func rotaryTables(positions *mlx.Array, dim int32, theta float32, dtype mlx.DType) (*mlx.Array, *mlx.Array) { + inv := make([]float32, dim/2) + for i := range inv { + inv[i] = 1 / float32(math.Pow(float64(theta), float64(float32(2*i)/float32(dim)))) + } + freq := mlx.Mul(mlx.ExpandDims(positions.AsType(mlx.DTypeFloat32), -1), mlx.FromValues(inv, len(inv))) + freq = mlx.Concatenate([]*mlx.Array{freq, freq}, -1) + return mlx.Cos(freq).AsType(dtype), mlx.Sin(freq).AsType(dtype) +} + +func applyRotary(x, cos, sin *mlx.Array) *mlx.Array { + f := x.AsType(mlx.DTypeFloat32) + half := x.Dim(-1) / 2 + first := f.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(0, half)) + second := f.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(half, x.Dim(-1))) + rotated := mlx.Concatenate([]*mlx.Array{second.Negative(), first}, -1) + c := mlx.ExpandDims(cos.AsType(mlx.DTypeFloat32), 1) + s := mlx.ExpandDims(sin.AsType(mlx.DTypeFloat32), 1) + return mlx.Add(mlx.Mul(f, c), mlx.Mul(rotated, s)).AsType(x.DType()) +} + +func rmsNorm(x, weight *mlx.Array, eps float32) *mlx.Array { + // Round the normalized activation before multiplying the learned weight. + return mlx.Mul(mlx.RMSNormFn(x, nil, eps), weight) +} + +func scaleEmbedding(x *mlx.Array, scale *float64) *mlx.Array { + if scale == nil { + return x + } + return mlx.MulScalar(x.AsType(mlx.DTypeFloat32), float32(*scale)).AsType(x.DType()) +} + +func attentionGate(x *mlx.Array) *mlx.Array { + return mlx.Sigmoid(x.AsType(mlx.DTypeFloat32)).AsType(x.DType()) +} + +var gatedAttention = mlx.Compile2("AttentionOutputGate", func(out, gate *mlx.Array) *mlx.Array { + return mlx.Mul(out, attentionGate(gate)) +}, mlx.Shapeless()) + +func blendValues(seed, x, lambda *mlx.Array) *mlx.Array { + // Sigmoid rounds to the parameter dtype; lerp has one activation rounding. + a := mlx.Sigmoid(lambda.AsType(mlx.DTypeFloat32)).AsType(lambda.DType()).AsType(x.DType()).AsType(mlx.DTypeFloat32) + return interpolateValues(seed, x, a) +} + +var interpolateValues = mlx.Compile3("ValueBlend", func(seed, x, a *mlx.Array) *mlx.Array { + start, end := seed.AsType(mlx.DTypeFloat32), x.AsType(mlx.DTypeFloat32) + delta := mlx.Sub(end, start) + low := mlx.Add(start, mlx.Mul(a, delta)) + high := mlx.Sub(end, mlx.Mul(mlx.Sub(mlx.FromValue(float32(1)), a), delta)) + return mlx.Where(a.Less(mlx.FromValue(float32(0.5))), low, high).AsType(x.DType()) +}, mlx.Shapeless()) diff --git a/mlxrunner/model/ripple/ripple.go b/mlxrunner/model/ripple/ripple.go new file mode 100644 index 000000000..feb99527e --- /dev/null +++ b/mlxrunner/model/ripple/ripple.go @@ -0,0 +1,318 @@ +// Package ripple implements the Ricursive transformer. +package ripple + +import ( + "encoding/json" + "fmt" + "math" + "slices" + "strings" + + "github.com/ollama/ollama/mlx" + "github.com/ollama/ollama/mlx/quant" + "github.com/ollama/ollama/mlxrunner/cache" + "github.com/ollama/ollama/mlxrunner/nn" + "github.com/ollama/ollama/mlxrunner/tokenizer" + + "github.com/ollama/ollama/mlxrunner/model" +) + +func init() { model.Register("RicursiveForCausalLM", newModel) } + +// Model retains all loaded arrays; value-residual seeds belong to Forward. +type Model struct { + *Config + EmbedTokens nn.EmbeddingLayer + Layers []*Layer + Norm *mlx.Array + LMHead nn.LinearLayer + tok *tokenizer.Tokenizer + auxHiddenLayers []int + draftTensorPrefix string +} + +type Layer struct { + Attention *Attention + MLP *MLP + InputNorm, PostAttentionNorm, PreFFNNorm, PostFFNNorm *mlx.Array +} + +type Attention struct { + QProj, KProj, VProj, OProj, GProj nn.LinearLayer + QNorm, KNorm, ValueResLambda *mlx.Array +} + +func newModel(root *model.Root) (model.Model, error) { + cfg, err := loadConfig(root) + if err != nil { + return nil, err + } + configData, err := root.Manifest.ReadConfig("config.json") + if err != nil { + return nil, err + } + tokData, err := root.Manifest.ReadConfig("tokenizer.json") + if err != nil { + return nil, fmt.Errorf("load tokenizer: %w", err) + } + tc := &tokenizer.TokenizerConfig{ConfigJSON: configData} + tc.GenerationConfigJSON, _ = root.Manifest.ReadConfig("generation_config.json") + tc.TokenizerConfigJSON, _ = root.Manifest.ReadConfig("tokenizer_config.json") + tok, err := tokenizer.LoadFromBytesWithConfig(tokData, tc) + if err != nil { + return nil, fmt.Errorf("parse tokenizer: %w", err) + } + m := &Model{Config: cfg, Layers: make([]*Layer, cfg.NumHiddenLayers), tok: tok} + if root.Draft != nil { + m.draftTensorPrefix = root.Draft.TensorPrefix + if m.draftTensorPrefix == "" { + m.draftTensorPrefix = "draft." + } + } + return m, nil +} + +func (m *Model) Tokenizer() *tokenizer.Tokenizer { return m.tok } +func (m *Model) MaxContextLength() int { return int(m.MaxPositionEmbeddings) } +func (m *Model) Unembed(x *mlx.Array) *mlx.Array { return m.LMHead.Forward(x) } +func (m *Model) NumLayers() int { return len(m.Layers) } +func (m *Model) SetAuxHiddenLayers(layers []int) { m.auxHiddenLayers = slices.Clone(layers) } +func (m *Model) TokenEmbeddings(ids *mlx.Array) *mlx.Array { return m.EmbedTokens.Forward(ids) } +func (m *Model) RawLogits(hidden *mlx.Array) *mlx.Array { return m.LMHead.Forward(hidden) } + +// DraftWindow bounds draft attention to its training context. +func (*Model) DraftWindow() int { return 16384 } + +func (m *Model) NewCaches() []cache.Cache { + out := make([]cache.Cache, len(m.LayerTypes)) + for i, kind := range m.LayerTypes { + if kind == "sliding_attention" { + out[i] = cache.NewRotatingKVCache(m.SlidingWindow) + } else { + out[i] = cache.NewKVCache() + } + } + return out +} + +func (m *Model) weightShapes() map[string][]int { + h, d, f, v := int(m.HiddenSize), int(m.HeadDim), int(m.IntermediateSize), int(m.VocabSize) + q, k := int(m.NumAttentionHeads)*d, int(m.NumKeyValueHeads)*d + shapes := map[string][]int{"model.embed_tokens.weight": {v, h}, "lm_head.weight": {v, h}, "model.norm.weight": {h}} + for i := range m.NumHiddenLayers { + p := fmt.Sprintf("model.layers.%d.", i) + for name, shape := range map[string][]int{ + "input_layernorm.weight": {h}, "post_attention_layernorm.weight": {h}, "pre_feedforward_layernorm.weight": {h}, "post_feedforward_layernorm.weight": {h}, + "self_attn.q_proj.weight": {q, h}, "self_attn.k_proj.weight": {k, h}, "self_attn.v_proj.weight": {k, h}, "self_attn.o_proj.weight": {h, q}, "self_attn.g_proj.weight": {q, h}, + "self_attn.q_norm.weight": {d}, "self_attn.k_norm.weight": {d}, "self_attn.value_res_lambda": {1}, + "mlp.gate_proj.weight": {f, h}, "mlp.up_proj.weight": {f, h}, "mlp.down_proj.weight": {h, f}, + } { + shapes[p+name] = shape + } + } + return shapes +} + +func floatTensor(a *mlx.Array) bool { + return a.DType() == mlx.DTypeFloat32 || a.DType() == mlx.DTypeBFloat16 || a.DType() == mlx.DTypeFloat16 +} + +func (m *Model) validateWeights(tensors map[string]*mlx.Array) error { + allowed := make(map[string]bool) + for name, shape := range m.weightShapes() { + w := tensors[name] + if w == nil { + return fmt.Errorf("missing weight: %s", name) + } + allowed[name] = true + scales := tensors[name+"_scale"] + if scales == nil { + if !floatTensor(w) || !slices.Equal(w.Dims(), shape) { + return fmt.Errorf("invalid weight %s: dtype %v shape %v, want float %v", name, w.DType(), w.Dims(), shape) + } + if m.TensorQuant[name] != nil { + return fmt.Errorf("missing scale for quantized weight %s", name) + } + continue + } + if len(shape) != 2 { + return fmt.Errorf("cannot quantize %s", name) + } + if tq := m.TensorQuant[name]; tq != nil && quant.Canonical(tq.QuantType) == "" { + return fmt.Errorf("unsupported quantization for %s: %q", name, tq.QuantType) + } + gs, bits, mode := model.ResolveLinearQuantParams(m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant, name, w, scales) + if gs <= 0 || (bits != 4 && bits != 8) || (mode != "affine" && mode != "nvfp4" && mode != "mxfp4" && mode != "mxfp8") { + return fmt.Errorf("invalid quantization for %s", name) + } + if (mode == "nvfp4" && (gs != 16 || bits != 4)) || (mode == "mxfp4" && (gs != 32 || bits != 4)) || (mode == "mxfp8" && (gs != 32 || bits != 8)) { + return fmt.Errorf("invalid quantization group/bits for %s", name) + } + packed, ss := []int{shape[0], shape[1] * bits / 32}, []int{shape[0], shape[1] / gs} + if shape[1]%gs != 0 || shape[1]*bits%32 != 0 || w.DType() != mlx.DTypeUint32 || !slices.Equal(w.Dims(), packed) || !slices.Equal(scales.Dims(), ss) { + return fmt.Errorf("invalid quantized shape for %s: weight %v scale %v, want %v %v", name, w.Dims(), scales.Dims(), packed, ss) + } + allowed[name+"_scale"] = true + if mode == "affine" { + qb := tensors[name+"_qbias"] + if !floatTensor(scales) || qb == nil || !floatTensor(qb) || !slices.Equal(qb.Dims(), ss) { + return fmt.Errorf("invalid affine scale/bias for %s", name) + } + allowed[name+"_qbias"] = true + } else if scales.DType() != mlx.DTypeUint8 { + return fmt.Errorf("invalid packed scale dtype for %s", name) + } + for _, suffix := range []string{".global_scale", "_scale_2", ".input_scale", ".input_global_scale"} { + if a := tensors[name+suffix]; a != nil { + if mode != "nvfp4" || !floatTensor(a) || a.Size() != 1 { + return fmt.Errorf("invalid %s%s", name, suffix) + } + allowed[name+suffix] = true + } + } + } + for name := range tensors { + if !allowed[name] && !(m.draftTensorPrefix != "" && strings.HasPrefix(name, m.draftTensorPrefix)) { + return fmt.Errorf("unexpected weight: %s", name) + } + } + return nil +} + +func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error { + if err := m.validateWeights(tensors); err != nil { + return err + } + linears := model.NewLinearFactory(tensors, m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant) + m.EmbedTokens = model.MakeEmbeddingLayer(tensors, "model.embed_tokens", m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant) + m.LMHead = linears.Make("lm_head") + m.Norm = tensors["model.norm.weight"] + for i := range m.Layers { + p := fmt.Sprintf("model.layers.%d.", i) + m.Layers[i] = &Layer{ + InputNorm: tensors[p+"input_layernorm.weight"], PostAttentionNorm: tensors[p+"post_attention_layernorm.weight"], PreFFNNorm: tensors[p+"pre_feedforward_layernorm.weight"], PostFFNNorm: tensors[p+"post_feedforward_layernorm.weight"], + Attention: &Attention{QProj: linears.Make(p + "self_attn.q_proj"), KProj: linears.Make(p + "self_attn.k_proj"), VProj: linears.Make(p + "self_attn.v_proj"), OProj: linears.Make(p + "self_attn.o_proj"), GProj: linears.Make(p + "self_attn.g_proj"), QNorm: tensors[p+"self_attn.q_norm.weight"], KNorm: tensors[p+"self_attn.k_norm.weight"], ValueResLambda: tensors[p+"self_attn.value_res_lambda"]}, + MLP: &MLP{GateProj: linears.Make(p + "mlp.gate_proj"), UpProj: linears.Make(p + "mlp.up_proj"), DownProj: linears.Make(p + "mlp.down_proj")}, + } + } + return nil +} + +type Config struct { + HiddenSize int32 `json:"hidden_size"` + IntermediateSize int32 `json:"intermediate_size"` + NumHiddenLayers int32 `json:"num_hidden_layers"` + NumAttentionHeads int32 `json:"num_attention_heads"` + NumKeyValueHeads int32 `json:"num_key_value_heads"` + HeadDim int32 `json:"head_dim"` + VocabSize int32 `json:"vocab_size"` + MaxPositionEmbeddings int32 `json:"max_position_embeddings"` + RMSNormEps float32 `json:"rms_norm_eps"` + RopeTheta float32 `json:"rope_theta"` + SlidingWindow int `json:"sliding_window"` + LayerTypes []string `json:"layer_types"` + HiddenAct string `json:"hidden_act"` + UseGatedAttention bool `json:"use_gated_attention"` + GateFullPrecision bool `json:"gate_full_precision"` + UseValRes bool `json:"use_valres"` + TieWordEmbeddings bool `json:"tie_word_embeddings"` + AttentionBias bool `json:"attention_bias"` + MLPBias bool `json:"mlp_bias"` + FFNClampThreshold *float32 `json:"ffn_clamp_threshold"` + EmbedScale *float64 `json:"embed_scale"` + RopeScaling json.RawMessage `json:"rope_scaling"` + RopeParameters json.RawMessage `json:"rope_parameters"` + + QuantGroupSize int `json:"-"` + QuantBits int `json:"-"` + QuantMode string `json:"-"` + TensorQuant map[string]*model.TensorQuantInfo `json:"-"` + Scale float32 `json:"-"` +} + +func loadConfig(root *model.Root) (*Config, error) { + data, err := root.Manifest.ReadConfig("config.json") + if err != nil { + return nil, fmt.Errorf("load config: %w", err) + } + cfg := &Config{HiddenAct: "silu", RMSNormEps: 1e-6, RopeTheta: 10000, SlidingWindow: 1024, UseGatedAttention: true, GateFullPrecision: true} + if err := json.Unmarshal(data, cfg); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + for _, v := range []struct { + name string + value int32 + }{ + {"hidden_size", cfg.HiddenSize}, + {"intermediate_size", cfg.IntermediateSize}, + {"num_hidden_layers", cfg.NumHiddenLayers}, + {"num_attention_heads", cfg.NumAttentionHeads}, + {"num_key_value_heads", cfg.NumKeyValueHeads}, + {"head_dim", cfg.HeadDim}, + {"vocab_size", cfg.VocabSize}, + {"max_position_embeddings", cfg.MaxPositionEmbeddings}, + } { + if v.value <= 0 { + return nil, fmt.Errorf("invalid %s: %d", v.name, v.value) + } + } + if cfg.HeadDim%2 != 0 { + return nil, fmt.Errorf("head_dim must be even") + } + if cfg.NumAttentionHeads%cfg.NumKeyValueHeads != 0 { + return nil, fmt.Errorf("num_attention_heads must be divisible by num_key_value_heads") + } + if int64(cfg.NumAttentionHeads)*int64(cfg.HeadDim) > math.MaxInt32 { + return nil, fmt.Errorf("num_attention_heads * head_dim overflows") + } + if cfg.RMSNormEps <= 0 { + return nil, fmt.Errorf("rms_norm_eps must be positive") + } + if cfg.RopeTheta <= 0 { + return nil, fmt.Errorf("rope_theta must be positive") + } + if cfg.SlidingWindow <= 0 { + return nil, fmt.Errorf("sliding_window must be positive") + } + if len(cfg.LayerTypes) != int(cfg.NumHiddenLayers) { + return nil, fmt.Errorf("layer_types must have num_hidden_layers entries") + } + for i, kind := range cfg.LayerTypes { + if kind != "sliding_attention" && kind != "full_attention" { + return nil, fmt.Errorf("unsupported layer_types[%d]: %q", i, kind) + } + } + if cfg.HiddenAct != "silu" { + return nil, fmt.Errorf("unsupported hidden_act: %q", cfg.HiddenAct) + } + for _, v := range []struct { + name string + unsupported bool + }{ + {"use_gated_attention", !cfg.UseGatedAttention}, + {"gate_full_precision", !cfg.GateFullPrecision}, + {"use_valres", !cfg.UseValRes}, + {"tie_word_embeddings", cfg.TieWordEmbeddings}, + {"attention_bias", cfg.AttentionBias}, + {"mlp_bias", cfg.MLPBias}, + {"rope_scaling", len(cfg.RopeScaling) > 0 && string(cfg.RopeScaling) != "null"}, + {"rope_parameters", len(cfg.RopeParameters) > 0 && string(cfg.RopeParameters) != "null"}, + } { + if v.unsupported { + return nil, fmt.Errorf("unsupported %s", v.name) + } + } + if cfg.FFNClampThreshold != nil && *cfg.FFNClampThreshold < 0 { + return nil, fmt.Errorf("ffn_clamp_threshold must be nonnegative") + } + if cfg.EmbedScale != nil && (math.IsInf(*cfg.EmbedScale, 0) || math.IsNaN(*cfg.EmbedScale) || math.Abs(*cfg.EmbedScale) > math.MaxFloat32) { + return nil, fmt.Errorf("invalid embed_scale") + } + cfg.Scale = float32(1 / math.Sqrt(float64(cfg.HeadDim))) + cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode = model.QuantizationParams(root.QuantType()) + if gs := root.GroupSize(); gs > 0 { + cfg.QuantGroupSize = gs + } + cfg.TensorQuant = root.AllTensorQuant() + return cfg, nil +} diff --git a/mlxrunner/model/ripple/ripple_test.go b/mlxrunner/model/ripple/ripple_test.go new file mode 100644 index 000000000..fa13f73ce --- /dev/null +++ b/mlxrunner/model/ripple/ripple_test.go @@ -0,0 +1,1393 @@ +package ripple + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/json" + "fmt" + "maps" + "math" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/ollama/ollama/manifest" + "github.com/ollama/ollama/mlx" + "github.com/ollama/ollama/mlx/mlxtest" + "github.com/ollama/ollama/mlxrunner/batch" + "github.com/ollama/ollama/mlxrunner/cache" + "github.com/ollama/ollama/mlxrunner/model" + "github.com/ollama/ollama/mlxrunner/nn" + modeltypes "github.com/ollama/ollama/types/model" +) + +func TestMain(m *testing.M) { + if err := os.Setenv("MLX_ENABLE_TF32", "0"); err != nil { + panic(err) + } + os.Exit(m.Run()) +} + +func fixtureConfig(t *testing.T) map[string]any { + t.Helper() + return map[string]any{ + "architectures": []string{"RicursiveForCausalLM"}, + "vocab_size": 19, "hidden_size": 8, "intermediate_size": 12, + "num_hidden_layers": 3, "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 4, + "layer_types": []string{"sliding_attention", "full_attention", "sliding_attention"}, + "sliding_window": 3, "rms_norm_eps": 1e-6, "rope_theta": 10000.0, + "use_valres": true, "use_gated_attention": true, "gate_full_precision": true, + "ffn_clamp_threshold": 20.0, "embed_scale": 71.55417527999327, + "max_position_embeddings": 2097152, "tie_word_embeddings": false, + } +} + +func testRoot(t *testing.T, cfg map[string]any) *model.Root { + t.Helper() + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + root := &model.Root{Manifest: &manifest.Manifest{}} + for name, data := range map[string]string{ + "config.json": string(data), + "tokenizer.json": `{"model":{"type":"BPE","vocab":{"a":0},"merges":[]}}`, + } { + layer, err := manifest.NewLayer(strings.NewReader(data), manifest.MediaTypeImageJSON) + if err != nil { + t.Fatal(err) + } + layer.Name = name + root.Manifest.Layers = append(root.Manifest.Layers, layer) + } + return root +} + +type referenceTensor struct { + Data []float32 `json:"data"` + Shape []int `json:"shape"` + DType string `json:"dtype"` +} + +func (r referenceTensor) array() *mlx.Array { + var a *mlx.Array + if len(r.Shape) == 0 { + a = mlx.FromValue(r.Data[0]) + } else { + a = mlx.FromValues(r.Data, r.Shape...) + } + if r.DType == "bfloat16" { + a = a.AsType(mlx.DTypeBFloat16) + } + return a +} + +type reference struct { + Weights map[string]map[string]referenceTensor +} + +func readReference(t *testing.T) reference { + t.Helper() + m := &Model{Config: &Config{ + HiddenSize: 8, IntermediateSize: 12, NumHiddenLayers: 3, + NumAttentionHeads: 4, NumKeyValueHeads: 2, HeadDim: 4, VocabSize: 19, + }} + r := reference{Weights: make(map[string]map[string]referenceTensor)} + for _, dtype := range []string{"float32", "bfloat16"} { + weights := make(map[string]referenceTensor) + for name, shape := range m.weightShapes() { + size := 1 + for _, dim := range shape { + size *= dim + } + values := make([]float32, size) + hash := sha256.Sum256([]byte(name)) + phase := int(binary.BigEndian.Uint32(hash[:4]) % 101) + if strings.HasSuffix(name, "value_res_lambda") { + layer := strings.Split(name, ".")[2] + values[0] = map[string]float32{"0": -1.25, "1": 0.75, "2": 1.5}[layer] + } else { + for i := range values { + v := float32((i*37+phase)%101-50) / 50 + switch { + case strings.Contains(name, "norm.weight"): + scaled := v * 0.375 + v = 1 + scaled + case strings.Contains(name, "embed_tokens"): + v *= 0.045 + case strings.Contains(name, ".mlp.gate_proj") || strings.Contains(name, ".mlp.up_proj"): + v *= 18 + default: + v *= 0.3 + } + values[i] = v + } + } + if dtype == "bfloat16" { + for i, v := range values { + bits := math.Float32bits(v) + values[i] = math.Float32frombits((bits + 0x7fff + ((bits >> 16) & 1)) & 0xffff0000) + } + } + weights[name] = referenceTensor{Data: values, Shape: shape, DType: dtype} + } + r.Weights[dtype] = weights + } + return r +} + +func checkTensor(t *mlxtest.T, label string, got *mlx.Array, want referenceTensor, atol, rtol float64) { + t.Helper() + if !slices.Equal(got.Dims(), want.Shape) { + t.Fatalf("%s shape=%v want %v", label, got.Dims(), want.Shape) + } + a := mlx.Contiguous(got.AsType(mlx.DTypeFloat32), false) + mlx.Eval(a) + values := a.Floats() + for i, v := range values { + d := math.Abs(float64(v - want.Data[i])) + bound := atol + rtol*math.Abs(float64(want.Data[i])) + if math.IsNaN(float64(v)) || d > bound { + t.Fatalf("%s[%d]=%.9g want %.9g (abs error %.9g > %.9g)", label, i, v, want.Data[i], d, bound) + } + } +} + +type activationLinear struct { + value *mlx.Array + width int32 +} + +func (l activationLinear) Forward(x *mlx.Array) *mlx.Array { + if l.value != nil { + return l.value + } + return x +} + +func (l activationLinear) OutputDim() int32 { return l.width } + +func clipFFN(gate, up *mlx.Array, threshold *float32) (*mlx.Array, *mlx.Array) { + if threshold != nil && *threshold > 0 { + gate = mlx.Clamp(gate, float32(math.Inf(-1)), *threshold).AsType(gate.DType()) + up = mlx.Clamp(up, -*threshold, *threshold).AsType(up.DType()) + } + return gate, up +} + +func eagerFFN(gate, up *mlx.Array, threshold *float32) *mlx.Array { + gate, up = clipFFN(gate, up, threshold) + f := gate.AsType(mlx.DTypeFloat32) + silu := mlx.Mul(f, mlx.Sigmoid(f)).AsType(gate.DType()) + return mlx.Mul(silu, up) +} + +func TestMLPActivationFusionMemory(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.EnableCompile() + const width, rows = 1024, 256 + gateValues, upValues := make([]float32, rows*width), make([]float32, rows*width) + for i := range gateValues { + gateValues[i] = float32(i%513-256) / 8 + upValues[i] = float32((i*7)%513-256) / 8 + } + gate := mlx.FromValues(gateValues, 1, rows, width) + up := mlx.FromValues(upValues, 1, rows, width) + mlx.Eval(gate, up) + m := MLP{GateProj: activationLinear{gate, width}, UpProj: activationLinear{up, width}, DownProj: activationLinear{width: width}} + threshold := float32(20) + eager := func() *mlx.Array { return eagerFFN(gate, up, &threshold) } + actual := func() *mlx.Array { return m.Forward(gate, &threshold) } + for _, fn := range []func() *mlx.Array{eager, actual} { + mlx.Scoped(func() { mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} }) }) + } + measure := func(fn func() *mlx.Array) int { + mlx.DefaultStream().Synchronize() + mlx.ClearCache() + baseline := mlx.ActiveMemory() + mlx.ResetPeakMemory() + var peak int + mlx.Scoped(func() { + mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} }) + mlx.DefaultStream().Synchronize() + peak = mlx.PeakMemory() - baseline + }) + return peak + } + want, got := measure(eager), measure(actual) + t.Logf("activation temporary peak: eager=%d actual=%d", want, got) + if got >= want { + t.Fatalf("MLP activation peak %d did not improve on eager %d", got, want) + } + }) +} + +func TestMLPActivationPrecision(t *testing.T) { + for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} { + t.Run(fmt.Sprint(dtype), func(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.EnableCompile() + zero, small, large := float32(0), float32(2), float32(20) + for _, rows := range []int{1, 8, 3, 1} { + mlx.Scoped(func() { + const width = 256 + gateValues, upValues := make([]float32, rows*width), make([]float32, rows*width) + for i := range gateValues { + gateValues[i] = float32(i%1025-512) / 16 + upValues[i] = float32((i*29)%701-350) / 8 + } + copy(gateValues, []float32{-40, -20.125, -20, -19.875, -2.015625, -2, -1.984375, -0.125, 0, 0.125, 1.984375, 2, 2.015625, 19.875, 20, 20.125, 40}) + gate := mlx.FromValues(gateValues, 1, rows, width).AsType(dtype) + up := mlx.FromValues(upValues, 1, rows, width).AsType(dtype) + m := MLP{GateProj: activationLinear{gate, width}, UpProj: activationLinear{up, width}, DownProj: activationLinear{width: width}} + for _, threshold := range []*float32{nil, &zero, &small, &large, &small} { + want := eagerFFN(gate, up, threshold) + got := m.Forward(gate, threshold) + label := fmt.Sprintf("rows=%d unclipped", rows) + if threshold != nil { + label = fmt.Sprintf("rows=%d threshold=%g", rows, *threshold) + } + checkExactArray(t, label, got, want) + } + }) + } + }) + }) + } +} + +func eagerValueBlend(seed, x, lambda *mlx.Array) *mlx.Array { + a := mlx.Sigmoid(lambda.AsType(mlx.DTypeFloat32)).AsType(lambda.DType()).AsType(x.DType()).AsType(mlx.DTypeFloat32) + start, end := seed.AsType(mlx.DTypeFloat32), x.AsType(mlx.DTypeFloat32) + delta := mlx.Sub(end, start) + low := mlx.Add(start, mlx.Mul(a, delta)) + high := mlx.Sub(end, mlx.Mul(mlx.Sub(mlx.FromValue(float32(1)), a), delta)) + return mlx.Where(a.Less(mlx.FromValue(float32(0.5))), low, high).AsType(x.DType()) +} + +func TestValueBlendFusionMemory(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.EnableCompile() + const width, rows = 1024, 256 + seedValues, xValues := make([]float32, rows*width), make([]float32, rows*width) + for i := range seedValues { + seedValues[i] = float32(i%257-128) / 64 + xValues[i] = float32((i*7)%257-128) / 64 + } + seed := mlx.FromValues(seedValues, 1, rows, width) + x := mlx.FromValues(xValues, 1, rows, width) + lambda := mlx.FromValues([]float32{-0.5}, 1) + mlx.Eval(seed, x, lambda) + eager := func() *mlx.Array { return eagerValueBlend(seed, x, lambda) } + actual := func() *mlx.Array { return blendValues(seed, x, lambda) } + for _, fn := range []func() *mlx.Array{eager, actual} { + mlx.Scoped(func() { mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} }) }) + } + measure := func(fn func() *mlx.Array) int { + mlx.DefaultStream().Synchronize() + mlx.ClearCache() + baseline := mlx.ActiveMemory() + mlx.ResetPeakMemory() + var peak int + mlx.Scoped(func() { + mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} }) + mlx.DefaultStream().Synchronize() + peak = mlx.PeakMemory() - baseline + }) + return peak + } + want, got := measure(eager), measure(actual) + t.Logf("value-blend temporary peak: eager=%d actual=%d", want, got) + if got >= want { + t.Fatalf("value-blend peak %d did not improve on eager %d", got, want) + } + }) +} + +func TestValueBlendFusionPrecision(t *testing.T) { + for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} { + for _, parameterType := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} { + t.Run(fmt.Sprintf("%v/parameter_%v", dtype, parameterType), func(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.EnableCompile() + var maxError float64 + for _, rows := range []int{1, 8, 3, 1} { + mlx.Scoped(func() { + const width = 256 + seedValues, xValues := make([]float32, rows*width), make([]float32, rows*width) + for i := range seedValues { + seedValues[i] = float32(i%257-128) / 64 + xValues[i] = float32((i*11)%257-128) / 64 + } + seed := mlx.FromValues(seedValues, 1, rows, width).AsType(dtype) + x := mlx.FromValues(xValues, 1, rows, width).AsType(dtype) + for _, value := range []float32{-20, -4, -0.0078125, 0, 0.0078125, 4, 20} { + lambda := mlx.FromValues([]float32{value}, 1).AsType(parameterType) + want, got := eagerValueBlend(seed, x, lambda), blendValues(seed, x, lambda) + if got.DType() != dtype || !slices.Equal(got.Dims(), want.Dims()) { + t.Fatal("value blend changed dtype or shape") + } + wantValues := mlx.Contiguous(want.AsType(mlx.DTypeFloat32), false).Floats() + gotValues := mlx.Contiguous(got.AsType(mlx.DTypeFloat32), false).Floats() + atol := 1e-6 + if dtype == mlx.DTypeBFloat16 { + atol = 0 + } + for i, v := range gotValues { + delta := math.Abs(float64(v - wantValues[i])) + maxError = math.Max(maxError, delta) + if math.IsNaN(delta) || delta > atol { + t.Fatalf("rows=%d lambda=%g element=%d: error=%g exceeds %g", rows, value, i, delta, atol) + } + } + } + }) + } + t.Logf("maximum absolute difference=%g", maxError) + }) + }) + } + } +} + +// Pre-fusion attention path for the allocation comparison. +func eagerAttentionForGateTest(a *Attention, x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array { + B, L := int32(x.Dim(0)), int32(x.Dim(1)) + q := mlx.Reshape(a.QProj.Forward(x), B, L, cfg.NumAttentionHeads, cfg.HeadDim) + k := mlx.Reshape(a.KProj.Forward(x), B, L, cfg.NumKeyValueHeads, cfg.HeadDim) + v := mlx.Reshape(a.VProj.Forward(blendValues(seed, x, a.ValueResLambda)), B, L, cfg.NumKeyValueHeads, cfg.HeadDim) + q = mlx.Transpose(rmsNorm(q, a.QNorm, cfg.RMSNormEps), 0, 2, 1, 3) + k = mlx.Transpose(rmsNorm(k, a.KNorm, cfg.RMSNormEps), 0, 2, 1, 3) + v = mlx.Transpose(v, 0, 2, 1, 3) + if local { + q, k = applyRotary(q, cos, sin), applyRotary(k, cos, sin) + } + mask := nn.CausalMask() + var kv nn.SDPAOption + if c != nil { + kv = nn.WithKVHistory(c.(cache.Attention).Update(b, k, v)) + } else { + kv = nn.WithKV(k, v, b.SeqQueryLens) + if local { + mask = mask.Intersect(nn.SlidingWindowMask(b, int(L), cfg.SlidingWindow, k.DType())) + } + } + out := nn.ScaledDotProductAttention(b, q, cfg.Scale, kv, nn.WithMask(mask)) + out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim) + return a.OProj.Forward(mlx.Mul(out, attentionGate(a.GProj.Forward(x)))) +} + +func TestAttentionGateFusionMemory(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.ClearCache() + const heads, kvHeads, headDim = 80, 8, 128 + const width, kvWidth = heads * headDim, kvHeads * headDim + qValues, gateValues := make([]float32, width), make([]float32, width) + vValues, normValues := make([]float32, kvWidth), make([]float32, headDim) + for i := range qValues { + qValues[i] = float32(i%17-8) / 32 + gateValues[i] = float32(i%65-32) / 4 + } + for i := range vValues { + vValues[i] = float32(i%13-6) / 8 + } + for i := range normValues { + normValues[i] = 1 + } + q := mlx.FromValues(qValues, 1, 1, width) + k := mlx.Zeros(mlx.DTypeFloat32, 1, 1, kvWidth) + v := mlx.FromValues(vValues, 1, 1, kvWidth) + gate := mlx.FromValues(gateValues, 1, 1, width) + norm := mlx.FromValues(normValues, headDim) + // Keep the unrelated value blend scalar in this allocation probe. + x := mlx.FromValues([]float32{1}, 1, 1, 1) + lambda := mlx.FromValues([]float32{0}, 1) + mlx.Eval(q, k, v, gate, norm, x, lambda) + a := Attention{ + QProj: activationLinear{q, width}, KProj: activationLinear{k, kvWidth}, VProj: activationLinear{v, kvWidth}, + GProj: activationLinear{gate, width}, OProj: activationLinear{width: width}, + QNorm: norm, KNorm: norm, ValueResLambda: lambda, + } + cfg := Config{NumAttentionHeads: heads, NumKeyValueHeads: kvHeads, HeadDim: headDim, RMSNormEps: 1e-6, Scale: 1 / float32(math.Sqrt(headDim))} + b := inputBatch([]float32{1}, 0) + eager := func() *mlx.Array { return eagerAttentionForGateTest(&a, x, x, b, nil, nil, nil, false, &cfg) } + actual := func() *mlx.Array { return a.Forward(x, x, b, nil, nil, nil, false, &cfg) } + mlx.EnableCompile() + for _, fn := range []func() *mlx.Array{eager, actual} { + mlx.Scoped(func() { mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} }) }) + } + measure := func(fn func() *mlx.Array) (int, []float32) { + mlx.DefaultStream().Synchronize() + mlx.ClearCache() + baseline := mlx.ActiveMemory() + mlx.ResetPeakMemory() + var peak int + var values []float32 + mlx.Scoped(func() { + out := mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} })[0] + mlx.DefaultStream().Synchronize() + peak = mlx.PeakMemory() - baseline + values = mlx.Contiguous(out, false).Floats() + }) + return peak, values + } + eagerPeak, want := measure(eager) + fusedPeak, got := measure(actual) + if !slices.Equal(got, want) { + t.Fatal("compilation changed attention output") + } + t.Logf("attention temporary peak: eager=%d compiled=%d gate=%d", eagerPeak, fusedPeak, gate.NumBytes()) + if eagerPeak-fusedPeak < gate.NumBytes() { + t.Fatalf("attention did not eliminate a gate-sized temporary: eager=%d compiled=%d gate=%d", eagerPeak, fusedPeak, gate.NumBytes()) + } + }) +} + +func TestAttentionGateFusionPrecision(t *testing.T) { + for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} { + t.Run(fmt.Sprint(dtype), func(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.EnableCompile() + for _, rows := range []int{1, 8, 3, 1} { + mlx.Scoped(func() { + const width = 256 + outValues, gateValues := make([]float32, rows*width), make([]float32, rows*width) + for i := range gateValues { + outValues[i] = float32((i*11)%257-128) / 16 + gateValues[i] = float32(i%1025-512) / 32 + } + copy(gateValues, []float32{-40, -20, -4, -0.0078125, 0, 0.0078125, 4, 20, 40}) + out := mlx.FromValues(outValues, 1, rows, width).AsType(dtype) + gate := mlx.FromValues(gateValues, 1, rows, width).AsType(dtype) + want := mlx.Mul(out, mlx.Sigmoid(gate.AsType(mlx.DTypeFloat32)).AsType(dtype)) + checkExactArray(t, fmt.Sprintf("rows=%d", rows), gatedAttention(out, gate), want) + }) + } + }) + }) + } +} + +func fixtureWeights(r reference, dtype string) map[string]*mlx.Array { + out := make(map[string]*mlx.Array) + for name, v := range r.Weights[dtype] { + out[name] = v.array() + } + return out +} + +func TestLoadWeights(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + r := readReference(t) + root := testRoot(t, fixtureConfig(t)) + mlxtest.Run(t, func(t *mlxtest.T) { + loaded, err := model.New(root) + if err != nil { + t.Fatal(err) + } + m := loaded.(*Model) + weights := fixtureWeights(r, "bfloat16") + if err := m.LoadWeights(weights); err != nil { + t.Fatal(err) + } + if len(m.Layers) != 3 || m.Tokenizer() == nil || m.MaxContextLength() != 2097152 { + t.Fatal("incomplete model") + } + for name, w := range weights { + delete(weights, name) + if err := m.LoadWeights(weights); err == nil || !strings.Contains(err.Error(), name) { + t.Fatalf("missing %s: %v", name, err) + } + weights[name] = mlx.Zeros(mlx.DTypeBFloat16, 1, 1) + if err := m.LoadWeights(weights); err == nil { + t.Fatalf("accepted bad shape for %s", name) + } + weights[name] = w + } + if err := m.LoadWeights(weights); err != nil { + t.Fatal(err) + } + caches := m.NewCaches() + defer func() { + for _, c := range caches { + c.Free() + } + }() + if _, ok := caches[0].(*cache.RotatingKVCache); !ok { + t.Fatal("local cache type") + } + if _, ok := caches[1].(*cache.KVCache); !ok { + t.Fatal("global cache type") + } + }) +} + +type prequantProjection struct { + name string + rows, cols int + global float32 +} + +var prequantProjections = []prequantProjection{ + {"gate_proj", 64, 32, 0.5}, + {"up_proj", 64, 32, 2}, + {"down_proj", 32, 64, 0.125}, +} + +func prequantWeights() map[string]*mlx.Array { + shapes := map[string][]int{ + "model.embed_tokens.weight": {32, 32}, "lm_head.weight": {32, 32}, "model.norm.weight": {32}, + "model.layers.0.input_layernorm.weight": {32}, "model.layers.0.post_attention_layernorm.weight": {32}, + "model.layers.0.pre_feedforward_layernorm.weight": {32}, "model.layers.0.post_feedforward_layernorm.weight": {32}, + "model.layers.0.self_attn.q_proj.weight": {32, 32}, "model.layers.0.self_attn.k_proj.weight": {16, 32}, + "model.layers.0.self_attn.v_proj.weight": {16, 32}, "model.layers.0.self_attn.o_proj.weight": {32, 32}, + "model.layers.0.self_attn.g_proj.weight": {32, 32}, "model.layers.0.self_attn.q_norm.weight": {16}, + "model.layers.0.self_attn.k_norm.weight": {16}, "model.layers.0.self_attn.value_res_lambda": {1}, + } + weights := make(map[string]*mlx.Array) + for name, shape := range shapes { + size := shape[0] + if len(shape) == 2 { + size *= shape[1] + } + values := make([]float32, size) + for i := range values { + if len(shape) == 1 { + values[i] = 1 + } else if i/shape[1] == i%shape[1] { + values[i] = 0.5 + } + } + weights[name] = mlx.FromValues(values, shape...).AsType(mlx.DTypeBFloat16) + } + for _, p := range prequantProjections { + name := "model.layers.0.mlp." + p.name + ".weight" + packed := make([]uint32, p.rows*p.cols/8) + scales := make([]uint8, p.rows*p.cols/16) + for row := range p.rows { + for col := range p.cols { + packed[row*p.cols/8+col/8] |= uint32((row*3+col)%16) << (4 * (col % 8)) + } + for group := range p.cols / 16 { + // E4M3 encodings for 0.5, 1, 2, and 4. + scales[row*p.cols/16+group] = []uint8{0x30, 0x38, 0x40, 0x48}[(row+group)%4] + } + } + weights[name] = mlx.FromValues(packed, p.rows, p.cols/8) + weights[name+"_scale"] = mlx.FromValues(scales, p.rows, p.cols/16) + weights[name+".global_scale"] = mlx.FromValue(p.global) + } + return weights +} + +func prequantRoot(t *testing.T) *model.Root { + t.Helper() + root := testRoot(t, map[string]any{ + "architectures": []string{"RicursiveForCausalLM"}, "hidden_size": 32, "intermediate_size": 64, + "num_hidden_layers": 1, "num_attention_heads": 2, "num_key_value_heads": 1, "head_dim": 16, + "vocab_size": 32, "max_position_embeddings": 64, "layer_types": []string{"full_attention"}, + "use_valres": true, "ffn_clamp_threshold": 20, + }) + path := filepath.Join(t.TempDir(), "weights.safetensors") + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + weights := prequantWeights() + metadata := make(map[string]string) + for _, p := range prequantProjections { + name := "model.layers.0.mlp." + p.name + ".weight" + weights[name+".scale"] = weights[name+"_scale"] + delete(weights, name+"_scale") + metadata[name+".quant_type"] = "nvfp4" + metadata[name+".group_size"] = "16" + } + if err := mlx.SaveSafetensorsWithMetadata(path, weights, metadata); err != nil { + t.Fatal(err) + } + }) + }) + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + layer, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor) + if err != nil { + t.Fatal(err) + } + root.Manifest.Layers = append(root.Manifest.Layers, layer) + config, err := manifest.NewLayer(strings.NewReader(`{"model_format":"safetensors"}`), manifest.MediaTypeImageConfig) + if err != nil { + t.Fatal(err) + } + const name = "ripple-prequant-test" + if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), config, root.Manifest.Layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil { + t.Fatal(err) + } + root, err = model.Open(name) + if err != nil { + t.Fatal(err) + } + return root +} + +func loadPrequantWeights(t *mlxtest.T, root *model.Root) map[string]*mlx.Array { + t.Helper() + weights := make(map[string]*mlx.Array) + for _, layer := range root.Manifest.TensorLayers() { + path, err := manifest.BlobsPath(layer.Digest) + if err != nil { + t.Fatal(err) + } + for name, weight := range mlx.Load(path) { + if strings.HasSuffix(name, ".scale") { + name = strings.TrimSuffix(name, ".scale") + "_scale" + } + weights[name] = weight + } + } + mlx.Eval(slices.Collect(maps.Values(weights))...) + return weights +} + +func checkExactArray(t *mlxtest.T, name string, got, want *mlx.Array) { + t.Helper() + if got == nil || want == nil { + if got != want { + t.Fatalf("%s: mismatched nil arrays", name) + } + return + } + if got.DType() != want.DType() || !slices.Equal(got.Dims(), want.Dims()) { + t.Fatalf("%s: got %v %v, want %v %v", name, got.DType(), got.Dims(), want.DType(), want.Dims()) + } + // Compare in the original dtype, including every bit of packed uint32 words. + for i, equal := range mlx.Contiguous(got.Equal(want).AsType(mlx.DTypeInt32), false).Ints() { + if equal != 1 { + t.Fatalf("%s: element %d differs", name, i) + } + } +} + +func checkPrequantBasis(t *mlxtest.T, m *Model, withGlobal bool) { + t.Helper() + layers := []nn.LinearLayer{m.Layers[0].MLP.GateProj, m.Layers[0].MLP.UpProj, m.Layers[0].MLP.DownProj} + // Literal E2M1 values, independent of MLX's dequantizer. + fp4 := []float32{0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6} + block := []float32{0.5, 1, 2, 4} + for i, p := range prequantProjections { + basis := make([]float32, p.cols*p.cols) + want := make([]float32, p.cols*p.rows) + global := float32(1) + if withGlobal { + global = p.global + } + for col := range p.cols { + basis[col*p.cols+col] = 1 + for row := range p.rows { + want[col*p.rows+row] = fp4[(row*3+col)%16] * block[(row+col/16)%4] * global + } + } + x := mlx.FromValues(basis, p.cols, p.cols).AsType(mlx.DTypeBFloat16) + checkTensor(t, p.name, layers[i].Forward(x), referenceTensor{Data: want, Shape: []int{p.cols, p.rows}}, 0, 0) + } +} + +func TestLoadWeightsPrequantized(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + root := prequantRoot(t) + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + loaded, err := model.New(root) + if err != nil { + t.Fatal(err) + } + m := loaded.(*Model) + weights := loadPrequantWeights(t, root) + if err := m.LoadWeights(weights); err != nil { + t.Fatal(err) + } + layers := []nn.LinearLayer{m.Layers[0].MLP.GateProj, m.Layers[0].MLP.UpProj, m.Layers[0].MLP.DownProj} + for i, p := range prequantProjections { + name := "model.layers.0.mlp." + p.name + ".weight" + if q := m.TensorQuant[name]; q == nil || q.QuantType != "NVFP4" || q.GroupSize != 16 { + t.Fatalf("%s metadata = %+v", name, q) + } + q, ok := layers[i].(*nn.QuantizedLinear) + if !ok { + t.Fatalf("%s type = %T", name, layers[i]) + } + if q.Mode != "nvfp4" || q.Bits != 4 || q.GroupSize != 16 || q.QBiases != nil || q.Bias != nil { + t.Fatalf("%s lost quantization parameters", name) + } + if q.Weight != weights[name] || q.Scales != weights[name+"_scale"] { + t.Fatalf("%s did not preserve packed weights and scales", name) + } + if q.GlobalScale == nil || q.GlobalScale.DType() != mlx.DTypeFloat32 { + t.Fatalf("%s lost float32 global scale", name) + } + } + a := m.Layers[0].Attention + for name, linear := range map[string]nn.LinearLayer{ + "lm_head.weight": m.LMHead, + "model.layers.0.self_attn.q_proj.weight": a.QProj, "model.layers.0.self_attn.k_proj.weight": a.KProj, + "model.layers.0.self_attn.v_proj.weight": a.VProj, "model.layers.0.self_attn.o_proj.weight": a.OProj, + "model.layers.0.self_attn.g_proj.weight": a.GProj, + } { + dense, ok := linear.(*nn.Linear) + if !ok || dense.Weight != weights[name] || dense.Weight.DType() != mlx.DTypeBFloat16 { + t.Fatalf("%s did not remain dense BF16", name) + } + } + embedding, ok := m.EmbedTokens.(*nn.Embedding) + if !ok || embedding.Weight != weights["model.embed_tokens.weight"] || embedding.Weight.DType() != mlx.DTypeBFloat16 { + t.Fatal("embedding did not remain dense BF16") + } + checkPrequantBasis(t, m, true) + }) + }) +} + +func TestLoadWeightsPrequantizedScaleFormats(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + root := prequantRoot(t) + for _, suffix := range []string{"_scale_2", ""} { + name := "legacy_global" + if suffix == "" { + name = "block_only" + } + t.Run(name, func(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + loaded, err := model.New(root) + if err != nil { + t.Fatal(err) + } + m := loaded.(*Model) + weights := loadPrequantWeights(t, root) + for _, p := range prequantProjections { + name := "model.layers.0.mlp." + p.name + ".weight" + if suffix != "" { + weights[name+suffix] = weights[name+".global_scale"] + } + delete(weights, name+".global_scale") + } + if err := m.LoadWeights(weights); err != nil { + t.Fatal(err) + } + checkPrequantBasis(t, m, suffix != "") + }) + }) + }) + } +} + +func TestLoadWeightsPrequantizedInvalid(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + root := prequantRoot(t) + const name = "model.layers.0.mlp.gate_proj.weight" + for _, tc := range []struct { + name, want string + mutate func(*Model, map[string]*mlx.Array) + }{ + {"quant_type", "unsupported quantization", func(m *Model, _ map[string]*mlx.Array) { + m.TensorQuant[name].QuantType = "unknown" + }}, + {"group_size", "invalid quantization group/bits", func(m *Model, _ map[string]*mlx.Array) { + m.TensorQuant[name].GroupSize = 32 + }}, + {"missing_scale", "missing scale for quantized weight", func(_ *Model, w map[string]*mlx.Array) { + delete(w, name+"_scale") + w[name] = mlx.Zeros(mlx.DTypeBFloat16, 64, 32) + }}, + {"packed_without_scale", "invalid weight", func(_ *Model, w map[string]*mlx.Array) { + delete(w, name+"_scale") + }}, + {"packed_dtype", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) { + w[name] = mlx.Zeros(mlx.DTypeUint8, 64, 4) + }}, + {"packed_rows", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) { + w[name] = mlx.Zeros(mlx.DTypeUint32, 32, 4) + }}, + {"packed_columns", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) { + w[name] = mlx.Zeros(mlx.DTypeUint32, 64, 8) + }}, + {"scale_dtype", "invalid packed scale dtype", func(_ *Model, w map[string]*mlx.Array) { + w[name+"_scale"] = mlx.Zeros(mlx.DTypeFloat32, 64, 2) + }}, + {"scale_rows", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) { + w[name+"_scale"] = mlx.Zeros(mlx.DTypeUint8, 32, 2) + }}, + {"scale_columns", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) { + w[name+"_scale"] = mlx.Zeros(mlx.DTypeUint8, 64, 1) + }}, + {"global_dtype", "invalid " + name + ".global_scale", func(_ *Model, w map[string]*mlx.Array) { + w[name+".global_scale"] = mlx.FromValues([]uint32{1}, 1) + }}, + {"global_shape", "invalid " + name + ".global_scale", func(_ *Model, w map[string]*mlx.Array) { + w[name+".global_scale"] = mlx.FromValues([]float32{1, 2}, 2) + }}, + {"activation_shape", "invalid " + name + ".input_scale", func(_ *Model, w map[string]*mlx.Array) { + w[name+".input_scale"] = mlx.FromValues([]float32{1, 2}, 2) + }}, + {"unexpected_bias", "unexpected weight: " + name + "_qbias", func(_ *Model, w map[string]*mlx.Array) { + w[name+"_qbias"] = mlx.Zeros(mlx.DTypeFloat32, 64, 2) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + loaded, err := model.New(root) + if err != nil { + t.Fatal(err) + } + m := loaded.(*Model) + weights := loadPrequantWeights(t, root) + tc.mutate(m, weights) + err = m.LoadWeights(weights) + if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), name) { + t.Fatalf("LoadWeights error = %v, want %q for %s", err, tc.want, name) + } + }) + }) + }) + } +} + +func TestWeightLifetime(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + root := prequantRoot(t) + mlxtest.Run(t, func(t *mlxtest.T) { + // Warm compiled constants before measuring weight ownership. + mlx.Scoped(func() { + w := mlx.Zeros(mlx.DTypeUint32, 32, 4) + s := mlx.Zeros(mlx.DTypeUint8, 32, 2) + g := mlx.FromValues([]float32{mlx.Nvfp4MaxProduct}, 1) + for _, shape := range [][]int{{1, 32}, {1, 1, 32}} { + x := mlx.Zeros(mlx.DTypeBFloat16, shape...) + mlx.Eval(mlx.QuantizedMatmul(x, w, s, nil, true, 16, 4, "nvfp4", g)) + } + gate := mlx.Zeros(mlx.DTypeBFloat16, 1, 1, 32) + up := mlx.AddScalar(gate, float32(1)) + mlx.Eval(clippedFFNActivation(gate, up, mlx.FromValue(float32(20)))) + lambda := mlx.FromValues([]float32{0.25}, 1).AsType(mlx.DTypeBFloat16) + mlx.Eval(blendValues(gate, up, lambda)) + }) + mlx.DefaultStream().Synchronize() + baseline := mlx.ActiveMemory() + for cycle := range 2 { + held := mlx.NewScope() + defer held.Close() + var m *Model + var loadMemory, temporaryBytes int + retained := mlx.ScopedArrays(func() []*mlx.Array { + loaded, err := model.New(root) + if err != nil { + t.Fatal(err) + } + m = loaded.(*Model) + weights := loadPrequantWeights(t, root) + if err := m.LoadWeights(weights); err != nil { + t.Fatal(err) + } + arrays := mlx.Collect(m) + // Check membership while every source handle is still live. + for name, w := range weights { + if strings.HasSuffix(name, ".global_scale") { + if slices.Contains(arrays, w) { + t.Fatalf("retained load-only source %s", name) + } + } else if !slices.Contains(arrays, w) { + t.Fatalf("model did not retain %s", name) + } + } + for i, linear := range []nn.LinearLayer{m.Layers[0].MLP.GateProj, m.Layers[0].MLP.UpProj, m.Layers[0].MLP.DownProj} { + q, ok := linear.(*nn.QuantizedLinear) + if !ok || q.GlobalScale == nil || !slices.Contains(arrays, q.GlobalScale) { + t.Fatalf("model did not retain converted global scale %d", i) + } + } + temporary := mlx.Zeros(mlx.DTypeFloat32, 256, 256) + mlx.Eval(temporary) + temporaryBytes = temporary.NumBytes() + mlx.DefaultStream().Synchronize() + loadMemory = mlx.ActiveMemory() + return arrays + }) + mlx.DefaultStream().Synchronize() + if got := mlx.ActiveMemory(); got > loadMemory-temporaryBytes { + t.Fatalf("load temporary survived scope: active=%d, load=%d, temporary=%d", got, loadMemory, temporaryBytes) + } + mlx.Eval(retained...) + held.Attach(retained...) + var previous []float32 + for request := range 2 { + mlx.Scoped(func() { + checkPrequantBasis(t, m, true) + hidden, _ := m.Forward(inputBatch([]float32{1, 4, 7}, 0), nil) + logits := mlx.Contiguous(m.Unembed(hidden).AsType(mlx.DTypeFloat32), false) + mlx.Eval(logits) + if !slices.Equal(logits.Dims(), []int{1, 3, 32}) { + t.Fatalf("logits shape = %v", logits.Dims()) + } + values := logits.Floats() + nonzero := false + for _, v := range values { + if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) { + t.Fatal("non-finite logits after load scope ended") + } + nonzero = nonzero || v != 0 + } + if !nonzero { + t.Fatal("zero logits after load scope ended") + } + if request > 0 && !slices.Equal(values, previous) { + t.Fatal("weights changed between request scopes") + } + previous = slices.Clone(values) + }) + } + held.Close() + mlx.DefaultStream().Synchronize() + if got := mlx.ActiveMemory(); got != baseline { + t.Fatalf("cycle %d leaked weight memory: active=%d, baseline=%d", cycle, got, baseline) + } + } + }) +} + +func TestDraftWeightNamespace(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + r := readReference(t) + for _, tc := range []struct { + name, prefix, extra, missing, wantError string + declared bool + }{ + {name: "default", declared: true, extra: "draft.fc.weight"}, + {name: "explicit", declared: true, prefix: "draft.", extra: "draft.fc.weight"}, + {name: "custom", declared: true, prefix: "assistant.", extra: "assistant.fc.weight"}, + {name: "undeclared", extra: "draft.fc.weight", wantError: "unexpected weight: draft.fc.weight"}, + {name: "outside prefix", declared: true, prefix: "assistant.", extra: "draft.fc.weight", wantError: "unexpected weight: draft.fc.weight"}, + {name: "unknown target", declared: true, extra: "model.unknown.weight", wantError: "unexpected weight: model.unknown.weight"}, + {name: "missing target", declared: true, extra: "draft.fc.weight", missing: "lm_head.weight", wantError: "missing weight: lm_head.weight"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := testRoot(t, fixtureConfig(t)) + if tc.declared { + root.Draft = &modeltypes.Draft{TensorPrefix: tc.prefix} + } + m, err := model.New(root) + if err != nil { + t.Fatal(err) + } + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + tensors := fixtureWeights(r, "bfloat16") + tensors[tc.extra] = mlx.FromValues([]float32{2}, 1) + delete(tensors, tc.missing) + before := maps.Clone(tensors) + err := m.LoadWeights(tensors) + if tc.wantError != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantError) { + t.Fatalf("LoadWeights error=%v, want %q", err, tc.wantError) + } + } else if err != nil { + t.Fatalf("declared draft namespace rejected: %v", err) + } + if !maps.Equal(before, tensors) { + t.Fatal("LoadWeights modified the caller's tensor map") + } + }) + }) + }) + } +} + +func TestDFlashTargetCombinedCaches(t *testing.T) { + m, r := dflashTargetFixture(t) + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + if err := m.LoadWeights(fixtureWeights(r, "bfloat16")); err != nil { + t.Fatal(err) + } + plain, combined := m.NewCaches(), m.NewCaches() + draft := []cache.Cache{cache.NewKVCache(), cache.NewRotatingKVCache(3)} + combined = append(combined, draft...) + defer func() { + for _, c := range append(plain, combined...) { + c.Free() + } + }() + b := inputBatch([]float32{1, 4, 7}, 0) + want, _ := m.Forward(b, plain) + mlx.Eval(want) + func() { + defer func() { + if p := recover(); p != nil { + t.Fatalf("target rejected trailing draft caches: %v", p) + } + }() + got, _ := m.Forward(b, combined) + checkExactArray(t, "combined-cache target hidden", got, want) + }() + for _, c := range combined[:len(m.Layers)] { + if c.Offset() != 3 { + t.Fatalf("target cache offset=%d, want 3", c.Offset()) + } + } + for _, c := range draft { + if c.Offset() != 0 { + t.Fatalf("target forward modified draft cache offset: %d", c.Offset()) + } + } + func() { + defer func() { + if p := recover(); p == nil || !strings.Contains(fmt.Sprint(p), "cache count") { + t.Fatalf("short cache slice panic=%v, want cache count rejection", p) + } + }() + m.Forward(b, plain[:len(plain)-1]) + }() + }) + }) +} + +func dflashTargetFixture(t *testing.T) (*Model, reference) { + t.Helper() + t.Setenv("OLLAMA_MODELS", t.TempDir()) + loaded, err := model.New(testRoot(t, fixtureConfig(t))) + if err != nil { + t.Fatal(err) + } + return loaded.(*Model), readReference(t) +} + +func TestDFlashTargetNumLayers(t *testing.T) { + m, _ := dflashTargetFixture(t) + target, ok := any(m).(interface{ NumLayers() int }) + if !ok { + t.Fatal("Ripple target is missing NumLayers") + } + if got := target.NumLayers(); got != 3 { + t.Fatalf("NumLayers() = %d, want 3 fixture blocks", got) + } +} + +func TestDFlashTargetDraftWindow(t *testing.T) { + m, _ := dflashTargetFixture(t) + target, ok := any(m).(interface{ DraftWindow() int }) + if !ok { + t.Fatal("Ripple target is missing DraftWindow") + } + if got := target.DraftWindow(); got != 16384 { + t.Fatalf("DraftWindow() = %d, want 16384", got) + } +} + +func TestDFlashTargetTokenEmbeddings(t *testing.T) { + m, r := dflashTargetFixture(t) + target, ok := any(m).(interface { + TokenEmbeddings(*mlx.Array) *mlx.Array + }) + if !ok { + t.Fatal("Ripple target is missing TokenEmbeddings") + } + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + if err := m.LoadWeights(fixtureWeights(r, "bfloat16")); err != nil { + t.Fatal(err) + } + if m.EmbedScale == nil || *m.EmbedScale == 1 { + t.Fatal("fixture must distinguish raw and scaled embeddings") + } + ids := []float32{1, 4, 7} + weight := r.Weights["bfloat16"]["model.embed_tokens.weight"] + width := weight.Shape[1] + var want []float32 + for _, id := range ids { + start := int(id) * width + want = append(want, weight.Data[start:start+width]...) + } + got := target.TokenEmbeddings(inputBatch(ids, 0).InputIDs) + if got.DType() != mlx.DTypeBFloat16 { + t.Fatalf("embedding dtype = %v, want bfloat16", got.DType()) + } + checkTensor(t, "raw table rows", got, referenceTensor{Data: want, Shape: []int{1, len(ids), width}}, 0, 0) + }) + }) +} + +func TestDFlashTargetRawLogits(t *testing.T) { + m, r := dflashTargetFixture(t) + target, ok := any(m).(interface { + RawLogits(*mlx.Array) *mlx.Array + }) + if !ok { + t.Fatal("Ripple target is missing RawLogits") + } + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + if err := m.LoadWeights(fixtureWeights(r, "bfloat16")); err != nil { + t.Fatal(err) + } + weight := r.Weights["bfloat16"]["lm_head.weight"] + vocab, width := weight.Shape[0], weight.Shape[1] + basis := make([]float32, width*width) + want := make([]float32, width*vocab) + for col := range width { + basis[col*width+col] = 2 + for row := range vocab { + want[col*vocab+row] = 2 * weight.Data[row*width+col] + } + } + x := mlx.FromValues(basis, 1, width, width).AsType(mlx.DTypeBFloat16) + got := target.RawLogits(x) + if got.DType() != mlx.DTypeBFloat16 { + t.Fatalf("logit dtype = %v, want bfloat16", got.DType()) + } + checkTensor(t, "unnormalized head columns", got, referenceTensor{Data: want, Shape: []int{1, width, vocab}}, 0, 0) + }) + }) +} + +// Run the existing blocks directly, independently of Forward's tap selection. +func dflashLayerOutputs(m *Model, b *batch.Batch) []*mlx.Array { + h := scaleEmbedding(m.EmbedTokens.Forward(b.InputIDs), m.EmbedScale) + seed := h + positions := mlx.Add(mlx.Reshape(mlx.FromValues(b.SeqOffsets, len(b.SeqOffsets)), int32(len(b.SeqOffsets)), 1), mlx.Arange(0, float64(b.InputIDs.Dim(1)), 1, mlx.DTypeInt32)) + cos, sin := rotaryTables(positions, m.HeadDim, m.RopeTheta, h.DType()) + states := make([]*mlx.Array, len(m.Layers)) + for i, layer := range m.Layers { + h = layer.Forward(h, seed, b, nil, cos, sin, m.LayerTypes[i] == "sliding_attention", m.Config) + states[i] = h + } + return states +} + +func TestDFlashTargetAuxHiddenLayers(t *testing.T) { + m, r := dflashTargetFixture(t) + target, ok := any(m).(interface{ SetAuxHiddenLayers([]int) }) + if !ok { + t.Fatal("Ripple target is missing SetAuxHiddenLayers") + } + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + if err := m.LoadWeights(fixtureWeights(r, "bfloat16")); err != nil { + t.Fatal(err) + } + b := inputBatch([]float32{1, 4, 7}, 0) + states := dflashLayerOutputs(m, b) + wantHidden := rmsNorm(states[len(states)-1], m.Norm, m.RMSNormEps) + var observed [][]float32 + for i, state := range append(slices.Clone(states), wantHidden) { + values := mlx.Contiguous(state.AsType(mlx.DTypeFloat32), false).Floats() + for j, previous := range observed { + if slices.Equal(values, previous) { + t.Fatalf("fixture states %d and %d are indistinguishable", i, j) + } + } + observed = append(observed, values) + } + + baseline, aux := m.Forward(b, nil) + if baseline != aux { + t.Fatal("without taps, Forward must return h, h") + } + checkExactArray(t, "no-taps hidden", baseline, wantHidden) + baselineLogits := m.Unembed(baseline) + mlx.Eval(baselineLogits) + + for _, tc := range []struct { + name string + ids []int + }{ + {"first_last", []int{0, 2}}, + {"middle", []int{1}}, + {"disabled", nil}, + } { + target.SetAuxHiddenLayers(tc.ids) + hidden, aux := m.Forward(b, nil) + checkExactArray(t, tc.name+" hidden", hidden, wantHidden) + checkExactArray(t, tc.name+" logits", m.Unembed(hidden), baselineLogits) + if len(tc.ids) == 0 { + if hidden != aux { + t.Fatal("clearing taps must restore h, h") + } + continue + } + if aux == nil || !slices.Equal(aux.Dims(), []int{1, 3, len(tc.ids) * int(m.HiddenSize)}) { + t.Fatalf("%s: incorrect auxiliary feature shape", tc.name) + } + for slot, id := range tc.ids { + start := slot * int(m.HiddenSize) + got := aux.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(start, start+int(m.HiddenSize))) + checkExactArray(t, fmt.Sprintf("%s tap %d", tc.name, id), got, states[id]) + } + } + }) + }) +} + +func inputBatch(ids []float32, offset int32) *batch.Batch { + tokens := make([]int32, len(ids)) + for i, v := range ids { + tokens[i] = int32(v) + } + return &batch.Batch{InputIDs: mlx.FromValues(tokens, 1, len(tokens)), SeqOffsets: []int32{offset}, SeqQueryLens: []int32{int32(len(tokens))}} +} + +func TestAttentionWindow(t *testing.T) { + mlxtest.Run(t, func(t *mlxtest.T) { + cfg := &Config{HiddenSize: 4, HeadDim: 4, NumAttentionHeads: 1, NumKeyValueHeads: 1, RMSNormEps: 1e-6, Scale: 0.5, SlidingWindow: 1024} + identity := mlx.FromValues([]float32{1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}, 4, 4) + zero := nn.NewLinear(mlx.Zeros(mlx.DTypeFloat32, 4, 4), nil) + a := &Attention{QProj: zero, KProj: zero, GProj: zero, VProj: nn.NewLinear(identity, nil), OProj: nn.NewLinear(identity, nil), QNorm: mlx.FromValues([]float32{1, 1, 1, 1}, 4), KNorm: mlx.FromValues([]float32{1, 1, 1, 1}, 4), ValueResLambda: mlx.FromValues([]float32{100}, 1)} + for _, local := range []bool{false, true} { + for _, length := range []int{1023, 1024, 1025} { + vals := make([]float32, length*4) + for i := range length { + for j := range 4 { + vals[i*4+j] = float32(i + 1) + } + } + x := mlx.FromValues(vals, 1, length, 4) + b := inputBatch(make([]float32, length), 0) + positions := mlx.Reshape(mlx.Arange(0, float64(length), 1, mlx.DTypeFloat32), 1, int32(length)) + cos, sin := rotaryTables(positions, 4, 10000, mlx.DTypeFloat32) + expected := make([]float32, length*4) + for i := range length { + lo := 0 + if local { + lo = max(0, i-1023) + } + for j := range 4 { + expected[i*4+j] = float32(lo+i+2) / 4 + } + } + want := referenceTensor{Data: expected, Shape: []int{1, length, 4}} + for _, cached := range []bool{false, true} { + var c cache.Cache + if cached { + if local { + c = cache.NewRotatingKVCache(1024) + } else { + c = cache.NewKVCache() + } + } + got := a.Forward(x, x, b, c, cos, sin, local, cfg) + checkTensor(t, fmt.Sprintf("local=%v cached=%v length=%d", local, cached, length), got, want, 2e-3, 1e-5) + if c != nil { + c.Free() + } + } + } + } + }) +} + +func TestCacheContinuation(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + r := readReference(t) + cfg := fixtureConfig(t) + cfg["sliding_window"] = 1024 + root := testRoot(t, cfg) + mlxtest.Run(t, func(t *mlxtest.T) { + loaded, err := model.New(root) + if err != nil { + t.Fatal(err) + } + m := loaded.(*Model) + if err := m.LoadWeights(fixtureWeights(r, "float32")); err != nil { + t.Fatal(err) + } + tokens := make([]float32, 1030) + for i := range tokens { + tokens[i] = float32((i*7 + 2) % 19) + } + all, _ := m.Forward(inputBatch(tokens, 0), nil) + all = mlx.Contiguous(all, false) + mlx.Eval(all) + want := all.Floats() + caches := m.NewCaches() + defer func() { + for _, c := range caches { + c.Free() + } + }() + run := func(start, end int) { + h, _ := m.Forward(inputBatch(tokens[start:end], int32(start)), caches) + checkTensor(t, fmt.Sprintf("continuation %d:%d", start, end), h, referenceTensor{Data: want[start*8 : end*8], Shape: []int{1, end - start, 8}}, 3e-5, 3e-5) + } + run(0, 1023) + run(1023, 1024) + snaps := make([]cache.Snapshot, len(caches)) + for i, c := range caches { + snaps[i] = c.Snapshot(0) + defer snaps[i].Close() + } + run(1024, 1025) + run(1025, 1030) + for i, c := range caches { + if !c.Restore(snaps[i], 1024) { + t.Fatalf("restore layer %d", i) + } + } + run(1024, 1027) + run(1027, 1030) + // A fresh request must not inherit the prior embedding seed. + fresh, _ := m.Forward(inputBatch(tokens[:6], 0), nil) + checkTensor(t, "fresh request", fresh, referenceTensor{Data: want[:48], Shape: []int{1, 6, 8}}, 3e-5, 3e-5) + }) +} + +func TestConfig(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + cfg := fixtureConfig(t) + cfg["is_hybrid_swa"] = false + cfg["hybrid_layer_pattern"] = []int{0, 0, 0} + got, err := loadConfig(testRoot(t, cfg)) + if err != nil { + t.Fatal(err) + } + if got.HeadDim != 4 || got.HiddenSize != 8 || got.LayerTypes[1] != "full_attention" || got.SlidingWindow != 3 { + t.Fatalf("unexpected config: %+v", got) + } + for _, tc := range []struct { + field string + value any + }{ + {"hidden_size", 0}, + {"num_hidden_layers", 0}, + {"intermediate_size", -1}, + {"num_attention_heads", 3}, + {"num_key_value_heads", 0}, + {"head_dim", nil}, + {"head_dim", 3}, + {"vocab_size", 0}, + {"max_position_embeddings", 0}, + {"rms_norm_eps", -1}, + {"rope_theta", 0}, + {"sliding_window", 0}, + {"hidden_act", "gelu"}, + {"layer_types", []string{"full_attention"}}, + {"layer_types", []string{"sliding_attention", "bad", "full_attention"}}, + {"use_valres", false}, + {"use_gated_attention", false}, + {"gate_full_precision", false}, + {"tie_word_embeddings", true}, + {"attention_bias", true}, + {"mlp_bias", true}, + {"rope_scaling", map[string]any{"type": "linear", "factor": 2}}, + {"ffn_clamp_threshold", -1}, + } { + t.Run(tc.field, func(t *testing.T) { + bad := fixtureConfig(t) + bad[tc.field] = tc.value + _, err := loadConfig(testRoot(t, bad)) + if err == nil || !strings.Contains(err.Error(), tc.field) { + t.Fatalf("want %s rejection, got %v", tc.field, err) + } + }) + } +} diff --git a/mlxrunner/ripple_test.go b/mlxrunner/ripple_test.go new file mode 100644 index 000000000..c36a72152 --- /dev/null +++ b/mlxrunner/ripple_test.go @@ -0,0 +1,681 @@ +package mlxrunner + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/llm" + "github.com/ollama/ollama/manifest" + "github.com/ollama/ollama/mlx" + "github.com/ollama/ollama/mlx/mlxtest" + "github.com/ollama/ollama/mlxrunner/batch" + "github.com/ollama/ollama/mlxrunner/cache" + "github.com/ollama/ollama/mlxrunner/model" + "github.com/ollama/ollama/mlxrunner/model/ripple" + "github.com/ollama/ollama/mlxrunner/sample" + modeltypes "github.com/ollama/ollama/types/model" +) + +type rippleRunnerTensor struct { + data []float32 + shape []int +} + +func rippleRunnerFixture() (map[string]any, map[string]rippleRunnerTensor) { + cfg := map[string]any{ + "architectures": []string{"RicursiveForCausalLM"}, + "vocab_size": 19, "hidden_size": 8, "intermediate_size": 12, + "num_hidden_layers": 3, "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 4, + "layer_types": []string{"sliding_attention", "full_attention", "sliding_attention"}, + "sliding_window": 3, "rms_norm_eps": 1e-6, "rope_theta": 10000.0, + "use_valres": true, "use_gated_attention": true, "gate_full_precision": true, + "ffn_clamp_threshold": 20.0, "embed_scale": 71.55417527999327, + "max_position_embeddings": 64, "tie_word_embeddings": false, + } + shapes := map[string][]int{ + "model.embed_tokens.weight": {19, 8}, "lm_head.weight": {19, 8}, "model.norm.weight": {8}, + } + for i := range 3 { + prefix := fmt.Sprintf("model.layers.%d.", i) + for name, shape := range map[string][]int{ + "input_layernorm.weight": {8}, "post_attention_layernorm.weight": {8}, + "pre_feedforward_layernorm.weight": {8}, "post_feedforward_layernorm.weight": {8}, + "self_attn.q_proj.weight": {16, 8}, "self_attn.k_proj.weight": {8, 8}, + "self_attn.v_proj.weight": {8, 8}, "self_attn.o_proj.weight": {8, 16}, + "self_attn.g_proj.weight": {16, 8}, "self_attn.q_norm.weight": {4}, + "self_attn.k_norm.weight": {4}, "self_attn.value_res_lambda": {1}, + "mlp.gate_proj.weight": {12, 8}, "mlp.up_proj.weight": {12, 8}, "mlp.down_proj.weight": {8, 12}, + } { + shapes[prefix+name] = shape + } + } + weights := make(map[string]rippleRunnerTensor) + for name, shape := range shapes { + size := 1 + for _, dim := range shape { + size *= dim + } + values := make([]float32, size) + hash := sha256.Sum256([]byte(name)) + phase := int(binary.BigEndian.Uint32(hash[:4]) % 101) + if strings.HasSuffix(name, "value_res_lambda") { + layer := strings.Split(name, ".")[2] + values[0] = map[string]float32{"0": -1.25, "1": 0.75, "2": 1.5}[layer] + } else { + for i := range values { + v := float32((i*37+phase)%101-50) / 50 + switch { + case strings.Contains(name, "norm.weight"): + scaled := v * 0.375 + v = 1 + scaled + case strings.Contains(name, "embed_tokens"): + v *= 0.045 + case strings.Contains(name, ".mlp.gate_proj") || strings.Contains(name, ".mlp.up_proj"): + v *= 18 + default: + v *= 0.3 + } + values[i] = v + } + } + weights[name] = rippleRunnerTensor{data: values, shape: shape} + } + return cfg, weights +} + +func writeRippleRunnerModel(t *testing.T) string { + t.Helper() + mlxtest.SkipIfUnavailable(t) + dir := t.TempDir() + t.Setenv("OLLAMA_MODELS", dir) + fixtureConfig, fixtureWeights := rippleRunnerFixture() + config, err := json.Marshal(fixtureConfig) + if err != nil { + t.Fatal(err) + } + vocab := make(map[string]int) + for i := range fixtureConfig["vocab_size"].(int) { + vocab[string(rune('a'+i))] = i + } + tokenizer, err := json.Marshal(map[string]any{ + "model": map[string]any{"type": "BPE", "vocab": vocab, "merges": []string{}}, + }) + if err != nil { + t.Fatal(err) + } + name := "ripple-runner-test" + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + weights := make(map[string]*mlx.Array) + for name, tensor := range fixtureWeights { + weights[name] = mlx.FromValues(tensor.data, tensor.shape...) + } + path := filepath.Join(dir, "model.safetensors") + if err := mlx.SaveSafetensors(path, weights); err != nil { + t.Fatal(err) + } + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + tensors, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor) + if err != nil { + t.Fatal(err) + } + tensors.Name = "model.safetensors" + layers := []manifest.Layer{tensors} + for name, data := range map[string][]byte{"config.json": config, "tokenizer.json": tokenizer} { + layer, err := manifest.NewLayer(bytes.NewReader(data), manifest.MediaTypeImageJSON) + if err != nil { + t.Fatal(err) + } + layer.Name = name + layers = append(layers, layer) + } + meta, err := manifest.NewLayer(strings.NewReader(`{"model_format":"safetensors","model_family":"ripple"}`), manifest.MediaTypeImageConfig) + if err != nil { + t.Fatal(err) + } + if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), meta, layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil { + t.Fatal(err) + } + }) + }) + return name +} + +func loadRippleRunner(t *testing.T, name string) *Runner { + t.Helper() + r := &Runner{mlxThread: mlxtest.Worker(t)} + mlxtest.Run(t, func(t *mlxtest.T) { + if err := r.Load(name); err != nil { + t.Fatal(err) + } + if _, ok := r.Model.(*ripple.Model); !ok { + t.Fatalf("loaded %T, want *ripple.Model", r.Model) + } + }) + t.Cleanup(func() { + mlxtest.Run(t, func(t *mlxtest.T) { + r.cache.close() + r.Close() + }) + }) + return r +} + +func rippleRunnerMemory(t *testing.T) (active int) { + t.Helper() + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.DefaultStream().Synchronize() + mlx.ClearCache() + active = mlx.ActiveMemory() + }) + return active +} + +func rippleRunnerRequest(t *testing.T, r *Runner, prompt string, count int) Request { + t.Helper() + request := Request{ + CompletionRequest: CompletionRequest{Prompt: prompt, Options: api.Options{NumPredict: count}}, + Pipeline: r.TextGenerationPipeline, + SamplerOpts: sample.Options{ + Temperature: 0, RepeatLastN: 8, RepeatPenalty: 1.1, + Seed: 1, UseSeed: true, Logprobs: true, TopLogprobs: 3, + }, + Responses: make(chan CompletionResponse, count+1), + } + if err := r.Prepare(&request); err != nil { + t.Fatal(err) + } + if len(request.Tokens) != len(prompt) { + t.Fatalf("tokenized %q into %d tokens, want %d", prompt, len(request.Tokens), len(prompt)) + } + return request +} + +func runRippleRunnerRequest(t *testing.T, r *Runner, prompt string) (string, []llm.Logprob, int) { + t.Helper() + request := rippleRunnerRequest(t, r, prompt, 4) + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + request.Ctx = ctx + if err := r.runRequest(request); err != nil { + t.Fatal(err) + } + close(request.Responses) + var text strings.Builder + var logprobs []llm.Logprob + var final *CompletionResponse + for response := range request.Responses { + if response.Error != nil { + t.Fatal(response.Error) + } + if response.Done { + if final != nil { + t.Fatal("duplicate final response") + } + final = &response + } + text.WriteString(response.Content) + logprobs = append(logprobs, response.Logprobs...) + } + if final == nil || final.EvalCount != 4 || final.DoneReason != 1 || final.PromptEvalCount != len(request.Tokens) || final.PromptEvalCachedCount == nil { + t.Fatalf("invalid final response: %+v", final) + } + if text.Len() != 4 || len(logprobs) != 4 { + t.Fatalf("empty or incomplete model output: text=%q logprobs=%d", text.String(), len(logprobs)) + } + for _, lp := range logprobs { + if math.IsNaN(lp.Logprob) || math.IsInf(lp.Logprob, 0) || lp.Logprob >= 0 || len(lp.TopLogprobs) != 3 { + t.Fatalf("invalid model log probability: %+v", lp) + } + if lp.TopLogprobs[0].Logprob == lp.TopLogprobs[2].Logprob { + t.Fatalf("model produced uniform logits: %+v", lp) + } + } + return text.String(), logprobs, *final.PromptEvalCachedCount +} + +func TestRippleRunnerLoadClose(t *testing.T) { + name := writeRippleRunnerModel(t) + warm := loadRippleRunner(t, name) + mlxtest.Run(t, func(t *mlxtest.T) { warm.Close() }) + baseline := rippleRunnerMemory(t) + for range 3 { + r := loadRippleRunner(t, name) + if loaded := rippleRunnerMemory(t); loaded <= baseline { + t.Fatalf("load allocated no weights: loaded=%d baseline=%d", loaded, baseline) + } + mlxtest.Run(t, func(t *mlxtest.T) { r.Close() }) + if got := rippleRunnerMemory(t); got != baseline { + t.Fatalf("load/close retained %d bytes above warmed baseline %d", got-baseline, baseline) + } + } +} + +func TestRippleRunnerRequests(t *testing.T) { + name := writeRippleRunnerModel(t) + cold, reused := loadRippleRunner(t, name), loadRippleRunner(t, name) + const prompt = "ihgfedcba" + wantText, wantLogprobs, cached := runRippleRunnerRequest(t, cold, prompt) + if cached != 0 { + t.Fatalf("fresh runner reused %d tokens", cached) + } + runRippleRunnerRequest(t, reused, "abcdefghi") + gotText, gotLogprobs, cached := runRippleRunnerRequest(t, reused, prompt) + // These cold forwards have identical chunk schedules; cache-hit forwards do not. + if cached != 0 || gotText != wantText || !reflect.DeepEqual(gotLogprobs, wantLogprobs) { + t.Fatalf("independent request inherited state: cached=%d text=%q want=%q", cached, gotText, wantText) + } + wantText, wantLogprobs, cached = runRippleRunnerRequest(t, reused, prompt) + if cached <= 0 || cached >= len(prompt) { + t.Fatalf("repeat reused %d tokens, want a nonempty prefix with a decode seed", cached) + } + baseline := rippleRunnerMemory(t) + // Settled allocator readings can vary by a few small buffers between requests. + const memorySlack = 256 + for range 16 { + gotText, gotLogprobs, gotCached := runRippleRunnerRequest(t, reused, prompt) + if gotCached != cached || gotText != wantText || !reflect.DeepEqual(gotLogprobs, wantLogprobs) { + t.Fatal("identical cache-hit schedule changed the response") + } + if got := rippleRunnerMemory(t); got > baseline+memorySlack { + t.Fatalf("repeated request retained %d bytes above warmed baseline %d", got-baseline, baseline) + } + } + + request := rippleRunnerRequest(t, reused, prompt, 32) + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + request.Ctx = ctx + request.Responses = make(chan CompletionResponse) + responses := make(chan []CompletionResponse, 1) + go func() { + var received []CompletionResponse + for response := range request.Responses { + received = append(received, response) + cancel() + } + responses <- received + }() + err := reused.runRequest(request) + close(request.Responses) + received := <-responses + if !errors.Is(err, context.Canceled) || len(received) == 0 || received[0].Content == "" { + t.Fatalf("cancel after real output: err=%v responses=%+v", err, received) + } + for _, response := range received { + if response.Done { + t.Fatal("cancelled generation emitted a successful final response") + } + } + // Retry through the same worker, sampler, and prefix-cache owner. + runRippleRunnerRequest(t, reused, prompt) + runRippleRunnerRequest(t, reused, prompt) + baseline = rippleRunnerMemory(t) + runRippleRunnerRequest(t, reused, prompt) + if got := rippleRunnerMemory(t); got > baseline+memorySlack { + t.Fatalf("post-cancellation request retained %d bytes", got-baseline) + } +} + +func writeRippleDFlash2RunnerModel(t *testing.T) string { + t.Helper() + name := writeRippleRunnerModel(t) + m, err := manifest.ParseNamedManifest(modeltypes.ParseName(name)) + if err != nil { + t.Fatal(err) + } + const draftConfig = `{ + "architectures": ["DFlash2DraftModel"], + "hidden_size": 8, "num_hidden_layers": 2, "intermediate_size": 16, + "num_attention_heads": 2, "num_key_value_heads": 1, "head_dim": 4, + "rms_norm_eps": 0.000001, "rope_theta": 10000, + "vocab_size": 19, "num_target_layers": 3, "block_size": 8, + "layer_types": ["sliding_attention", "full_attention"], + "sliding_window": 4, "no_rope_layers": [1], "is_causal": false, + "dflash_config": { + "mask_token_id": 18, "target_layer_ids": [0, 2], + "conv_group_size": 2, "conv_kernel_size": 4, + "selector_rank": 4, "selector_top_k": 4 + } + }` + config, err := manifest.NewLayer(strings.NewReader(draftConfig), manifest.MediaTypeImageJSON) + if err != nil { + t.Fatal(err) + } + config.Name = "draft/config.json" + layers := append(m.Layers, config) + path := filepath.Join(t.TempDir(), "draft.safetensors") + mlxtest.Run(t, func(t *mlxtest.T) { + mlx.Scoped(func() { + weights := make(map[string]*mlx.Array) + add := func(name string, dims ...int) { + n, phase := 1, 0 + for _, dim := range dims { + n *= dim + } + for _, c := range name { + phase += int(c) + } + values := make([]float32, n) + for i := range values { + values[i] = float32((i*17+phase)%31-15) / 128 + if len(dims) == 1 { + values[i] = 1 + } + } + weights["draft."+name] = mlx.FromValues(values, dims...) + } + add("fc.weight", 8, 2*8) + add("hidden_norm.weight", 8) + add("norm.weight", 8) + for i := range 2 { + prefix := fmt.Sprintf("layers.%d.", i) + add(prefix+"input_layernorm.weight", 8) + add(prefix+"post_attention_layernorm.weight", 8) + add(prefix+"self_attn.q_norm.weight", 4) + add(prefix+"self_attn.k_norm.weight", 4) + add(prefix+"self_attn.q_proj.weight", 8, 8) + add(prefix+"self_attn.k_proj.weight", 4, 8) + add(prefix+"self_attn.v_proj.weight", 4, 8) + add(prefix+"self_attn.o_proj.weight", 8, 8) + add(prefix+"mlp.gate_proj.weight", 16, 8) + add(prefix+"mlp.up_proj.weight", 16, 8) + add(prefix+"mlp.down_proj.weight", 8, 16) + for _, conv := range []string{"attention_conv", "mlp_conv"} { + add(prefix+conv+".base_kernel", 2, 4, 8) + add(prefix+conv+".kernel_projection.weight", 2*4*8/2, 8) + } + } + add("candidate_selector.predecessor_codebook", 19, 4) + add("candidate_selector.successor_codebook", 19, 4) + add("candidate_selector.hidden_projection.weight", 4, 8) + if err := mlx.SaveSafetensors(path, weights); err != nil { + t.Fatal(err) + } + }) + }) + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + tensors, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor) + if err != nil { + t.Fatal(err) + } + tensors.Name = "draft/model.safetensors" + layers = append(layers, tensors) + metadata, err := json.Marshal(modeltypes.ConfigV2{ + ModelFormat: "safetensors", ModelFamily: "ripple", + Draft: &modeltypes.Draft{ + ModelFormat: "safetensors", Architecture: "DFlash2DraftModel", + TensorPrefix: "draft.", Config: "draft/config.json", + }, + }) + if err != nil { + t.Fatal(err) + } + meta, err := manifest.NewLayer(bytes.NewReader(metadata), manifest.MediaTypeImageConfig) + if err != nil { + t.Fatal(err) + } + name = "ripple-dflash2-runner-test" + if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), meta, layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil { + t.Fatal(err) + } + return name +} + +type rippleDraftForward struct { + offset, contextRows, featureWidth, blockRows int +} + +// Observe shapes and calls only; every native operand and selector stays real. +type rippleDFlash2Observer struct { + model.BlockSelector + forwards []rippleDraftForward + selections int +} + +func (d *rippleDFlash2Observer) Forward(b *batch.Batch, targets, drafts []cache.Cache) (*mlx.Array, *mlx.Array) { + call := rippleDraftForward{offset: int(b.SeqOffsets[0])} + if b.Hidden != nil { + call.contextRows, call.featureWidth = b.Hidden.Dim(1), b.Hidden.Dim(2) + } + if b.InputIDs != nil { + call.blockRows = b.InputIDs.Dim(1) + } + d.forwards = append(d.forwards, call) + return d.BlockSelector.Forward(b, targets, drafts) +} + +func (d *rippleDFlash2Observer) Select(hidden, logits, anchor *mlx.Array, choose func(ids, scores *mlx.Array) (*mlx.Array, *mlx.Array)) (*mlx.Array, *mlx.Array, *mlx.Array) { + d.selections++ + return d.BlockSelector.Select(hidden, logits, anchor, choose) +} + +func loadRippleDFlash2Runner(t *testing.T, name string) (*Runner, *rippleDFlash2Observer) { + t.Helper() + r := loadRippleRunner(t, name) + if r.spec == nil { + t.Fatal("combined manifest did not attach a draft") + } + draft, ok := r.spec.draft.(model.BlockSelector) + if !ok { + t.Fatalf("loaded draft %T, want DFlash2 BlockSelector", r.spec.draft) + } + if block, mask := draft.BlockParams(); block != 8 || mask != 18 { + t.Fatalf("draft block=%d mask=%d, want 8 and 18", block, mask) + } + if len(r.spec.targets) != 3 || len(r.spec.draftKV) != 2 || len(r.cache.caches) != 5 { + t.Fatalf("cache partitions: target=%d draft=%d combined=%d", len(r.spec.targets), len(r.spec.draftKV), len(r.cache.caches)) + } + if _, ok := r.spec.draftKV[0].(*cache.RotatingKVCache); !ok { + t.Fatalf("local draft cache is %T", r.spec.draftKV[0]) + } + if _, ok := r.spec.draftKV[1].(*cache.RotatingKVCache); !ok { + t.Fatalf("full draft cache is %T", r.spec.draftKV[1]) + } + observer := &rippleDFlash2Observer{BlockSelector: draft} + r.spec.draft = observer + return r, observer +} + +func checkRippleDFlash2Caches(t *testing.T, r *Runner, prompt string) { + t.Helper() + mlxtest.Run(t, func(t *mlxtest.T) { + offset := r.spec.targets[0].Offset() + if offset < len(prompt) { + t.Fatalf("target stopped before prompt completion: offset=%d prompt=%d", offset, len(prompt)) + } + for i, c := range r.cache.caches { + if c.Offset() != offset { + t.Fatalf("cache %d offset=%d, want settled target offset=%d", i, c.Offset(), offset) + } + } + }) +} + +type rippleDFlash2Response struct { + text string + cached int + forwards []rippleDraftForward +} + +func runRippleDFlash2Request(t *testing.T, r *Runner, prompt string, count int) rippleDFlash2Response { + t.Helper() + d := r.spec.draft.(*rippleDFlash2Observer) + d.forwards, d.selections = nil, 0 + request := rippleRunnerRequest(t, r, prompt, count) + // The shared helper requests logprobs, which would silently park drafting. + request.SamplerOpts.Logprobs, request.SamplerOpts.TopLogprobs = false, 0 + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + request.Ctx = ctx + if err := r.runRequest(request); err != nil { + t.Fatal(err) + } + close(request.Responses) + var text strings.Builder + var final *CompletionResponse + for response := range request.Responses { + if response.Error != nil || len(response.Logprobs) != 0 { + t.Fatalf("unexpected active response: %+v", response) + } + text.WriteString(response.Content) + if response.Done { + if final != nil { + t.Fatal("duplicate final response") + } + final = &response + } + } + if final == nil || final.EvalCount != count || final.DoneReason != 1 || final.PromptEvalCount != len(request.Tokens) || final.PromptEvalCachedCount == nil || text.Len() != count { + t.Fatalf("invalid completion: text=%q final=%+v", text.String(), final) + } + proposals := 0 + for _, call := range d.forwards { + if call.contextRows > 0 && call.featureWidth != 16 { + t.Fatalf("draft did not receive both target taps: %+v", call) + } + if call.blockRows > 0 { + proposals++ + if call.blockRows < 2 || call.blockRows > min(8, count) { + t.Fatalf("proposal exceeded block/budget: %+v budget=%d", call, count) + } + } + } + if proposals != d.selections || (count > 1 && proposals == 0) || (count == 1 && proposals != 0) { + t.Fatalf("active budget=%d: draft forwards=%d selector calls=%d", count, proposals, d.selections) + } + checkRippleDFlash2Caches(t, r, prompt) + return rippleDFlash2Response{text: text.String(), cached: *final.PromptEvalCachedCount, forwards: d.forwards} +} + +func TestRippleRunnerDFlash2(t *testing.T) { + name := writeRippleDFlash2RunnerModel(t) + const prompt = "abcdefghi" + + // New wiring: normal combined load, target prefill with trailing draft + // caches, target taps, and a real convolution/selector proposal. + t.Run("combined_manifest_prefill_proposal", func(t *testing.T) { + r, d := loadRippleDFlash2Runner(t, name) + got := runRippleDFlash2Request(t, r, prompt, 9) + if got.cached != 0 || len(d.forwards) < 2 { + t.Fatalf("cold request: cached=%d forwards=%+v", got.cached, d.forwards) + } + if want := (rippleDraftForward{contextRows: len(prompt) - 1, featureWidth: 16}); d.forwards[0] != want { + t.Fatalf("prefill handoff=%+v, want %+v", d.forwards[0], want) + } + if want := (rippleDraftForward{offset: len(prompt) - 1, blockRows: 8}); d.forwards[1] != want { + t.Fatalf("first proposal=%+v, want %+v", d.forwards[1], want) + } + }) + + // Existing runner lifecycle, characterized with the new target/draft pair. + // Compare identities only after matching cache-hit and proposal schedules. + t.Run("prefix_branch_reuse", func(t *testing.T) { + r, _ := loadRippleDFlash2Runner(t, name) + runRippleDFlash2Request(t, r, prompt, 9) + runRippleDFlash2Request(t, r, prompt, 9) // warm the repeat schedule + want := runRippleDFlash2Request(t, r, prompt, 9) + if want.cached <= 0 || want.cached >= len(prompt) { + t.Fatalf("repeat cached=%d, want prefix plus an unforwarded seed", want.cached) + } + if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) { + t.Fatalf("matched repeat changed: got=%+v want=%+v", got, want) + } + const branch = "abcdejklm" + if got := runRippleDFlash2Request(t, r, branch, 9); got.cached != 5 { + t.Fatalf("branch cached=%d, want shared five-token snapshot", got.cached) + } + if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) { + t.Fatalf("returning from branch changed matched schedule: got=%+v want=%+v", got, want) + } + }) + + t.Run("parked_to_enabled", func(t *testing.T) { + r, d := loadRippleDFlash2Runner(t, name) + runRippleRunnerRequest(t, r, prompt) + if len(d.forwards) == 0 || d.selections != 0 { + t.Fatalf("parked maintenance: forwards=%+v selections=%d", d.forwards, d.selections) + } + for _, call := range d.forwards { + if call.blockRows != 0 || call.contextRows == 0 || call.featureWidth != 16 { + t.Fatalf("logprobs must maintain context without proposals: %+v", call) + } + } + checkRippleDFlash2Caches(t, r, prompt) + if got := runRippleDFlash2Request(t, r, prompt, 9); got.cached <= 0 { + t.Fatal("enabled request did not reuse parked context") + } + runRippleDFlash2Request(t, r, prompt, 9) + want := runRippleDFlash2Request(t, r, prompt, 9) + if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) { + t.Fatalf("enabled repeats changed: got=%+v want=%+v", got, want) + } + }) + + t.Run("budget_tail", func(t *testing.T) { + r, d := loadRippleDFlash2Runner(t, name) + runRippleDFlash2Request(t, r, prompt, 3) + if len(d.forwards) < 2 || d.forwards[1].blockRows != 3 { + t.Fatalf("shorter-than-convolution proposal: %+v", d.forwards) + } + runRippleDFlash2Request(t, r, prompt, 1) + }) + + t.Run("cancellation_recovery", func(t *testing.T) { + r, d := loadRippleDFlash2Runner(t, name) + request := rippleRunnerRequest(t, r, prompt, 16) + request.SamplerOpts.Logprobs, request.SamplerOpts.TopLogprobs = false, 0 + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + request.Ctx = ctx + request.Responses = make(chan CompletionResponse) + responses := make(chan []CompletionResponse, 1) + go func() { + var received []CompletionResponse + for response := range request.Responses { + received = append(received, response) + cancel() + } + responses <- received + }() + err := r.runRequest(request) + close(request.Responses) + received := <-responses + if !errors.Is(err, context.Canceled) || len(received) == 0 || received[0].Content == "" || d.selections == 0 { + t.Fatalf("cancel after real speculative output: err=%v responses=%+v selections=%d", err, received, d.selections) + } + for _, response := range received { + if response.Done || response.Error != nil { + t.Fatalf("cancelled generation emitted terminal response: %+v", response) + } + } + checkRippleDFlash2Caches(t, r, prompt) + runRippleDFlash2Request(t, r, prompt, 9) + runRippleDFlash2Request(t, r, prompt, 9) + want := runRippleDFlash2Request(t, r, prompt, 9) + if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) { + t.Fatalf("recovery repeats changed: got=%+v want=%+v", got, want) + } + }) +} diff --git a/mlxrunner/testdata/generate_qwen35_yarn.py b/mlxrunner/testdata/generate_qwen35_yarn.py deleted file mode 100644 index ba21d301e..000000000 --- a/mlxrunner/testdata/generate_qwen35_yarn.py +++ /dev/null @@ -1,194 +0,0 @@ -# /// script -# requires-python = ">=3.12,<3.13" -# dependencies = ["torch==2.6.0", "transformers==5.12.1"] -# /// - -import argparse -import json -from pathlib import Path - -import torch -import transformers -from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig -from transformers.models.qwen3_5.modeling_qwen3_5 import ( - Qwen3_5Attention, - Qwen3_5TextRotaryEmbedding, - apply_rotary_pos_emb, -) - - -def generate(): - torch.set_num_threads(1) - defaults = { - "rope_type": "default", - "rope_theta": 10000000.0, - "partial_rotary_factor": 0.25, - "mrope_section": [11, 11, 10], - } - yarn = { - **defaults, - "rope_type": "yarn", - "factor": 4.0, - "original_max_position_embeddings": 262144, - } - cases = [ - ("default", 262144, defaults, "float32"), - ("no_metadata", 262144, None, "float32"), - ("yarn4", 1048576, yarn, "float32"), - ("unextended_limit", 262144, yarn, "float32"), - ( - "implicit_original", - 262144, - {k: v for k, v in yarn.items() if k != "original_max_position_embeddings"}, - "float32", - ), - ("beta", 1048576, {**yarn, "beta_fast": 16.0, "beta_slow": 2.0}, "float32"), - ("fractional_range", 1048576, {**yarn, "truncate": False}, "float32"), - ("mscale", 1048576, {**yarn, "mscale": 2.0, "mscale_all_dim": 1.0}, "float32"), - ("attention_override", 1048576, {**yarn, "attention_factor": 1.2}, "float32"), - ("zero_attention", 1048576, {**yarn, "attention_factor": 0.0}, "float32"), - ("factor_one", 262144, {**yarn, "factor": 1.0}, "float32"), - ( - "factor_one_attention", - 262144, - {**yarn, "factor": 1.0, "attention_factor": 1.2}, - "float32", - ), - ("theta", 1048576, {**yarn, "rope_theta": 500000.0}, "float32"), - ( - "half_rotary", - 1048576, - {**yarn, "partial_rotary_factor": 0.5, "mrope_section": [22, 21, 21]}, - "float32", - ), - ("sections", 1048576, {**yarn, "mrope_section": [24, 4, 4]}, "float32"), - ("default_bfloat16", 262144, defaults, "bfloat16"), - ("yarn_bfloat16", 1048576, yarn, "bfloat16"), - ] - sample_tokens = [0, 1, 7, 15] - sample_dims = [0, 7, 15, 16, 23, 31, 32, 39, 47, 48, 55, 63, 64, 95, 127, 128, 255] - indices = torch.arange(16) - text_positions = indices.reshape(1, -1) - media_positions = torch.stack( - (32768 + 2 * indices, 7 + indices % 4, 11 + indices // 4) - ).unsqueeze(1) - media_positions[:, 0, -1] = media_positions[:, :, :-1].max() + 1 - mask = torch.full((1, 1, 16, 16), float("-inf")).triu(1) - results = [] - for name, limit, params, dtype_name in cases: - raw_config = { - "model_type": "qwen3_5_text", - "hidden_size": 256, - "num_hidden_layers": 1, - "num_attention_heads": 1, - "num_key_value_heads": 1, - "head_dim": 256, - "intermediate_size": 256, - "rms_norm_eps": 1e-6, - "linear_num_key_heads": 1, - "linear_num_value_heads": 1, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "layer_types": ["full_attention"], - "max_position_embeddings": limit, - "rope_theta": 10000000.0, - "partial_rotary_factor": (params or defaults)["partial_rotary_factor"], - "rope_parameters": params, - } - cfg = Qwen3_5TextConfig( - **{ - k: v - for k, v in raw_config.items() - if k not in ("rope_theta", "rope_parameters") - }, - rope_parameters=dict(params or defaults), - ) - cfg._attn_implementation = "eager" - dtype = getattr(torch, dtype_name) - attention = Qwen3_5Attention(cfg, layer_idx=0).eval().to(dtype) - rope = Qwen3_5TextRotaryEmbedding(cfg) - assert rope.inv_freq.numel() * 2 == int( - cfg.head_dim * (params or defaults)["partial_rotary_factor"] - ) - with torch.no_grad(): - attention.q_proj.weight.zero_() - attention.q_proj.weight[:256].copy_(torch.eye(256)) - for linear in (attention.k_proj, attention.v_proj, attention.o_proj): - linear.weight.copy_(torch.eye(256)) - attention.q_norm.weight.fill_(-0.5) - attention.k_norm.weight.fill_(-0.5) - x = ( - (((torch.arange(16 * 256) % 37) - 18).float() / 16) - .reshape(1, 16, 256) - .to(dtype) - ) - raw_keys = attention.k_norm( - attention.k_proj(x).reshape(1, 16, 1, 256) - ).transpose(1, 2) - entry = { - "name": name, - "dtype": dtype_name, - "config": raw_config, - "inv_freq": rope.inv_freq.tolist(), - "attention_factor": rope.attention_scaling, - } - modes = ( - ("text", text_positions), - ("mrope", media_positions), - ("long_text", text_positions + 32768), - ) - if name == "sections": - entry["modes"] = ["mrope"] - modes = (("mrope", media_positions),) - for mode, positions in modes: - cos, sin = rope(x, positions) - output, _ = attention( - x, position_embeddings=(cos, sin), attention_mask=mask - ) - entry[mode + "_output"] = [ - output[0, token, dim].float().item() - for token in sample_tokens - for dim in sample_dims - ] - if mode != "long_text": - _, keys = apply_rotary_pos_emb(raw_keys, raw_keys, cos, sin) - entry[mode + "_keys"] = [ - keys[0, 0, token, dim].float().item() - for token in sample_tokens - for dim in sample_dims - ] - results.append(entry) - return { - "source": f"transformers {transformers.__version__}; torch {torch.__version__}; CPU; Qwen3_5TextConfig/Qwen3_5TextRotaryEmbedding/Qwen3_5Attention", - "sample_tokens": sample_tokens, - "sample_dims": sample_dims, - "mrope_positions": media_positions.flatten().tolist(), - "cases": results, - } - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("output", type=Path) - args = parser.parse_args() - fixture = generate() - text = json.dumps(fixture, indent=2) - for case in fixture["cases"]: - for key in ( - "inv_freq", - "text_output", - "long_text_output", - "text_keys", - "mrope_output", - "mrope_keys", - ): - if key not in case: - continue - multiline = json.dumps(case[key], indent=2) - body = "\n".join(" " + line for line in multiline.splitlines()[1:-1]) - text = text.replace("[\n" + body + "\n ]", json.dumps(case[key])) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(text + "\n") - print( - f"Generated {len(fixture['cases'])} reference cases: {args.output} ({args.output.stat().st_size} bytes)" - ) diff --git a/mlxrunner/verify_test.go b/mlxrunner/verify_test.go index 35eebd0a6..daa3dd690 100644 --- a/mlxrunner/verify_test.go +++ b/mlxrunner/verify_test.go @@ -6,6 +6,12 @@ import ( "github.com/ollama/ollama/mlx" ) +func TestSupportsRippleArchitecture(t *testing.T) { + if !SupportsArchitecture("RicursiveForCausalLM") { + t.Fatal("missing model architecture") + } +} + func TestCheckRuntimeRequiresLoadedLibrary(t *testing.T) { _, loadErr := mlx.LoadedLibraryPath() err := CheckRuntime() diff --git a/model/parsers/parsers.go b/model/parsers/parsers.go index 69c387f89..fa82ac9d3 100644 --- a/model/parsers/parsers.go +++ b/model/parsers/parsers.go @@ -53,6 +53,8 @@ func ParserForName(name string) Parser { var p Parser switch name { + case "ripple": + p = &RippleParser{} case "qwen3": p = &Qwen3Parser{hasThinkingSupport: false, defaultThinking: false} case "qwen3-thinking": diff --git a/model/parsers/ripple.go b/model/parsers/ripple.go new file mode 100644 index 000000000..2ad3fbe84 --- /dev/null +++ b/model/parsers/ripple.go @@ -0,0 +1,184 @@ +package parsers + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/ollama/ollama/api" +) + +type rippleState int + +const ( + rippleStart rippleState = iota + rippleThinkLead + rippleThinking + rippleThinkEnd + rippleContent + rippleTool +) + +const ( + rippleThinkOpen = "" + rippleThinkClose = "" + rippleToolOpen = "" + rippleToolClose = "" +) + +type RippleParser struct { + state rippleState + buffer string + callIndex int + toolScan int + toolString bool + toolEscape bool +} + +func (p *RippleParser) Init(tools []api.Tool, _ *api.Message, _ *api.ThinkValue) []api.Tool { + *p = RippleParser{} + return tools +} + +func (p *RippleParser) HasToolSupport() bool { return true } +func (p *RippleParser) HasThinkingSupport() bool { return true } + +func (p *RippleParser) ThinkingClose() []string { + if p.state == rippleThinkLead || p.state == rippleThinking { + return []string{rippleThinkClose} + } + return nil +} + +func (p *RippleParser) PreservedTokens() []string { + return []string{rippleThinkOpen, rippleThinkClose, rippleToolOpen, rippleToolClose} +} + +func (p *RippleParser) Add(s string, done bool) (content string, thinking string, calls []api.ToolCall, err error) { + p.buffer += s +parse: + for p.buffer != "" { + switch p.state { + case rippleStart: + if strings.HasPrefix(p.buffer, rippleThinkOpen) { + p.buffer = p.buffer[len(rippleThinkOpen):] + p.state = rippleThinkLead + } else if !done && strings.HasPrefix(rippleThinkOpen, p.buffer) { + return content, thinking, calls, nil + } else { + p.state = rippleContent + } + case rippleThinkLead: + p.buffer = strings.TrimPrefix(p.buffer, "\n") + p.state = rippleThinking + case rippleThinking: + if i := strings.Index(p.buffer, rippleThinkClose); i >= 0 { + thinking += strings.TrimSuffix(p.buffer[:i], "\n") + p.buffer = p.buffer[i+len(rippleThinkClose):] + p.state = rippleThinkEnd + } else { + n := len(p.buffer) + if !done { + // The newline before a closing tag belongs to the template. + n -= max(overlap(p.buffer, rippleThinkClose), overlap(p.buffer, "\n"+rippleThinkClose)) + } + thinking += p.buffer[:n] + p.buffer = p.buffer[n:] + return content, thinking, calls, nil + } + case rippleThinkEnd: + p.buffer = strings.TrimPrefix(p.buffer, "\n") + p.state = rippleContent + case rippleContent: + if i := strings.Index(p.buffer, rippleToolOpen); i >= 0 { + if p.callIndex == 0 { + content += strings.TrimSuffix(p.buffer[:i], "\n") + } + p.buffer = p.buffer[i+len(rippleToolOpen):] + p.state = rippleTool + p.toolScan, p.toolString, p.toolEscape = 0, false, false + } else { + n := len(p.buffer) + if !done { + n -= max(overlap(p.buffer, rippleToolOpen), overlap(p.buffer, "\n"+rippleToolOpen)) + } + if p.callIndex == 0 { + content += p.buffer[:n] + } + p.buffer = p.buffer[n:] + return content, thinking, calls, nil + } + case rippleTool: + if i := p.toolEnd(); i >= 0 { + call, parseErr := parseRippleTool(p.buffer[:i]) + p.buffer = p.buffer[i+len(rippleToolClose):] + p.state = rippleContent + if parseErr != nil { + return content, thinking, calls, parseErr + } + call.Function.Index = p.callIndex + p.callIndex++ + calls = append(calls, call) + } else if !done { + return content, thinking, calls, nil + } else { + break parse + } + } + } + if done && p.state == rippleTool { + p.buffer = "" + p.state = rippleContent + return content, thinking, calls, fmt.Errorf("ripple: unterminated tool call") + } + return content, thinking, calls, nil +} + +// toolEnd ignores closing tags inside JSON strings, including escaped quotes. +func (p *RippleParser) toolEnd() int { + for p.toolScan < len(p.buffer) { + i := p.toolScan + c := p.buffer[i] + if p.toolString { + if p.toolEscape { + p.toolEscape = false + } else if c == '\\' { + p.toolEscape = true + } else if c == '"' { + p.toolString = false + } + } else if c == '"' { + p.toolString = true + } else if c == '<' { + if strings.HasPrefix(p.buffer[i:], rippleToolClose) { + return i + } + if strings.HasPrefix(rippleToolClose, p.buffer[i:]) { + return -1 + } + } + p.toolScan++ + } + return -1 +} + +func parseRippleTool(raw string) (api.ToolCall, error) { + var value struct { + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` + } + if err := json.Unmarshal([]byte(raw), &value); err != nil { + return api.ToolCall{}, fmt.Errorf("ripple: invalid tool call JSON: %w", err) + } + if value.Name == "" { + return api.ToolCall{}, fmt.Errorf("ripple: empty function name") + } + if !strings.HasPrefix(strings.TrimSpace(string(value.Arguments)), "{") { + return api.ToolCall{}, fmt.Errorf("ripple: tool arguments must be a JSON object") + } + var arguments api.ToolCallFunctionArguments + if err := json.Unmarshal(value.Arguments, &arguments); err != nil { + return api.ToolCall{}, fmt.Errorf("ripple: invalid tool arguments: %w", err) + } + return api.ToolCall{Function: api.ToolCallFunction{Name: value.Name, Arguments: arguments}}, nil +} diff --git a/model/parsers/ripple_test.go b/model/parsers/ripple_test.go new file mode 100644 index 000000000..54fc6ca5e --- /dev/null +++ b/model/parsers/ripple_test.go @@ -0,0 +1,228 @@ +package parsers + +import ( + "reflect" + "strings" + "testing" + + "github.com/ollama/ollama/api" +) + +func rippleParseChunks(t *testing.T, chunks []string) (string, string, []api.ToolCall, error) { + t.Helper() + p := ParserForName("ripple") + if p == nil { + t.Fatal("missing ripple parser") + } + p.Init(nil, nil, nil) + var content, thinking strings.Builder + var calls []api.ToolCall + for _, chunk := range append(chunks, "") { + c, r, tc, err := p.Add(chunk, false) + content.WriteString(c) + thinking.WriteString(r) + calls = append(calls, tc...) + if err != nil { + return content.String(), thinking.String(), calls, err + } + } + c, r, tc, err := p.Add("", true) + content.WriteString(c) + thinking.WriteString(r) + calls = append(calls, tc...) + return content.String(), thinking.String(), calls, err +} + +func rippleChunkings(text string) [][]string { + chunks := [][]string{{text}} + for i := 0; i <= len(text); i++ { + chunks = append(chunks, []string{text[:i], text[i:]}) + } + bytes := make([]string, len(text)) + for i := range len(text) { + bytes[i] = text[i : i+1] + } + return append(chunks, bytes) +} + +func TestRippleParserThinking(t *testing.T) { + for _, tt := range []struct{ name, input, content, thinking string }{ + {"plain", " \nhello\n", " \nhello\n", ""}, + {"thinking", "\nreason\n\nanswer", "answer", "reason"}, + {"no structural newlines", "reasonanswer", "answer", "reason"}, + {"one newline only", "\n\n", "", ""}, + {"meaningful whitespace", "\n\n \treason \n\n\n\n answer \n", "\n answer \n", "\n \treason \n"}, + {"CRLF is not a structural newline", "\r\nreason\r\n\r\nanswer", "\r\nanswer", "\r\nreason\r"}, + {"nonleading opener", " \nreason\nanswer", " \nreason\nanswer", ""}, + {"no opener", "reason\n\nanswer", "reason\n\nanswer", ""}, + {"later tags literal", "rcx", "cx", "r"}, + {"tool inside thinking stays literal", "\n{}\n\nanswer", "answer", "{}"}, + {"truncated opener", "\nreason\n\nreason\n", "", "reason\n"}, + {"opener only", "", "", ""}, + {"opener and newline", "\n", "", ""}, + {"close only", "\nreason\n", "", "reason"}, + {"partial tool text", "text\n\n日本語🙂\n\né\n", "é\n", "日本語🙂"}, + } { + t.Run(tt.name, func(t *testing.T) { + for _, chunks := range rippleChunkings(tt.input) { + c, r, calls, err := rippleParseChunks(t, chunks) + if err != nil || c != tt.content || r != tt.thinking || len(calls) != 0 { + t.Fatalf("chunks %q: got (%q, %q, %v, %v), want (%q, %q)", chunks, c, r, calls, err, tt.content, tt.thinking) + } + } + p := ParserForName("ripple") + p.Init(nil, nil, nil) + c, r, calls, err := p.Add(tt.input, true) + if err != nil || c != tt.content || r != tt.thinking || len(calls) != 0 { + t.Fatalf("final chunk: (%q, %q, %v, %v)", c, r, calls, err) + } + c, r, calls, err = p.Add("", true) + if err != nil || c != "" || r != "" || len(calls) != 0 { + t.Fatal("final drain repeated output") + } + }) + } +} + +func TestRippleParserThinkingTransitions(t *testing.T) { + p := ParserForName("ripple") + p.Init(nil, nil, nil) + for _, tt := range []struct { + input, content, thinking string + close bool + }{ + {"", "", "", true}, + {"\nreason\n", "", "reason", true}, + {"", "", "", false}, + {"\nanswer", "answer", "", false}, + } { + c, r, _, err := p.Add(tt.input, false) + if c != tt.content || r != tt.thinking || err != nil { + t.Fatalf("%q: (%q, %q, %v)", tt.input, c, r, err) + } + if close := p.ThinkingClose(); (len(close) != 0) != tt.close || tt.close && close[0] != "" { + t.Fatalf("%q: close %v", tt.input, close) + } + } + p.Init(nil, nil, nil) + c, r, _, err := p.Add("\nfresh\n\nnew", true) + if c != "new" || r != "fresh" || err != nil { + t.Fatalf("reset: (%q, %q, %v)", c, r, err) + } +} + +func TestRippleParserTools(t *testing.T) { + one := "\n" + `{"name": "one", "arguments": {"z": "a\n\"b\"\\<&é", "n": 3, "nested": {"x": true}}}` + "\n" + two := "\n" + `{"name": "two", "arguments": {}}` + "\n" + for _, tt := range []struct { + name, input, content, thinking string + calls []api.ToolCall + }{ + {"one", one, "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "one", Arguments: args(`{"z":"a\n\"b\"\\<&é","n":3,"nested":{"x":true}}`)}}}}, + {"multiple", " \n before \t\n\n" + one + "\n" + two, " \n before \t\n", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "one", Arguments: args(`{"z":"a\n\"b\"\\<&é","n":3,"nested":{"x":true}}`)}}, {Function: api.ToolCallFunction{Index: 1, Name: "two", Arguments: args(`{}`)}}}}, + {"reasoning and tools", "\n R \n\n\n" + two, "", " R ", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}}}, + {"preserve spaces before call", " before \t" + two, " before \t", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}}}, + {"drop post call text", two + "ignored \nalso text\n" + two + "more ignored", "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}, {Function: api.ToolCallFunction{Index: 1, Name: "two", Arguments: args(`{}`)}}}}, + {"delimiter inside string", `{"name":"two","arguments":{"text":" \"quoted\""}}`, "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{"text":" \"quoted\""}`)}}}}, + } { + t.Run(tt.name, func(t *testing.T) { + for _, chunks := range rippleChunkings(tt.input) { + c, r, calls, err := rippleParseChunks(t, chunks) + if err != nil || c != tt.content || r != tt.thinking || len(calls) != len(tt.calls) { + t.Fatalf("chunks %q: (%q, %q, %v, %v)", chunks, c, r, calls, err) + } + for i := range calls { + if !toolCallEqual(calls[i], tt.calls[i]) { + t.Fatalf("call %d = %+v, want %+v", i, calls[i], tt.calls[i]) + } + } + } + }) + } +} + +func TestRippleParserMalformedCalls(t *testing.T) { + for _, tt := range []struct{ name, input, errorText string }{ + {"invalid JSON", "{bad}", "JSON"}, + {"missing name", `{"arguments":{}}`, "name"}, + {"empty name", `{"name":"","arguments":{}}`, "name"}, + {"missing arguments", `{"name":"f"}`, "object"}, + {"string arguments", `{"name":"f","arguments":"{}"}`, "object"}, + {"array arguments", `{"name":"f","arguments":[]}`, "object"}, + {"null arguments", `{"name":"f","arguments":null}`, "object"}, + {"JSON array", `[{"name":"f","arguments":{}}]`, "JSON"}, + {"multiple objects", `{"name":"f","arguments":{}}{}`, "JSON"}, + {"truncated call", `{"name":"f","arguments":{}`, "unterminated"}, + {"truncated closer", `{"name":"f","arguments":{}}{"name":"f","arguments":{}}`, true) + if c != "" || err != nil || len(calls) != 1 { + t.Fatalf("call: %q, %v, %v", c, calls, err) + } + p.Init(nil, nil, nil) + _, _, _, _ = p.Add("before\n", false) + c, _, _, err = p.Add("\nAnswer", " literal", "Text literal", "literal", "é 日本語 🙂"} { + t.Run(text, func(t *testing.T) { + content, thinking, calls, err := rippleParseChunks(t, []string{text}) + if err != nil || content != text || thinking != "" || len(calls) != 0 { + t.Fatalf("got (%q, %q, %v, %v), want %q", content, thinking, calls, err, text) + } + }) + } +} + +func TestRippleParserCapabilities(t *testing.T) { + p := ParserForName("ripple") + if p == nil { + t.Fatal("missing ripple parser") + } + if !p.HasToolSupport() || !p.HasThinkingSupport() { + t.Fatal("missing capabilities") + } + if got := p.PreservedTokens(); !reflect.DeepEqual(got, []string{"", "", "", ""}) { + t.Fatalf("preserved = %v", got) + } + tools := []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "f"}}} + for _, think := range []*api.ThinkValue{nil, {Value: true}, {Value: false}, {Value: "high"}} { + if got := p.Init(tools, &api.Message{Role: "assistant", Thinking: "old", Content: "old"}, think); !reflect.DeepEqual(got, tools) { + t.Fatalf("changed tools: %v", got) + } + if got := p.ThinkingClose(); len(got) != 0 { + t.Fatalf("generation prompt does not open reasoning: %v", got) + } + } +} diff --git a/model/renderers/renderer.go b/model/renderers/renderer.go index c175de20a..c11ca9f8a 100644 --- a/model/renderers/renderer.go +++ b/model/renderers/renderer.go @@ -59,6 +59,8 @@ func rendererForName(name string) Renderer { return constructor() } switch name { + case "ripple": + return &RippleRenderer{} case "qwen3-coder": renderer := &Qwen3CoderRenderer{} return renderer diff --git a/model/renderers/ripple.go b/model/renderers/ripple.go new file mode 100644 index 000000000..d0f88266e --- /dev/null +++ b/model/renderers/ripple.go @@ -0,0 +1,88 @@ +package renderers + +import ( + "fmt" + "strings" + "unicode" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/types/model" +) + +type RippleRenderer struct{} + +func (r *RippleRenderer) LeadingBOS() string { return "" } + +// Ripple can emit reasoning but has no template control to disable or force it. +func (r *RippleRenderer) Thinking() *model.Thinking { return nil } + +func (r *RippleRenderer) Render(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) { + if len(messages) == 0 { + return "", fmt.Errorf("ripple: expected at least one message") + } + for _, message := range messages { + if len(message.Images) > 0 { + return "", fmt.Errorf("ripple: text-only messages required") + } + } + var b strings.Builder + if len(tools) > 0 { + b.WriteString("<|im_start|>system\n") + if messages[0].Role == "system" { + b.WriteString(messages[0].Content + "\n\n") + } + b.WriteString("# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n") + for _, tool := range tools { + encoded, err := PythonJSON(tool, 0) + if err != nil { + return "", err + } + b.WriteString("\n" + encoded) + } + b.WriteString("\n\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n<|im_end|>\n") + } + for i, message := range messages { + if i == 0 && message.Role == "system" && len(tools) > 0 { + continue + } + content := message.Content + if message.Role == "assistant" && message.Thinking != "" && !strings.HasPrefix(strings.TrimLeftFunc(content, func(r rune) bool { + return unicode.IsSpace(r) || r >= '\x1c' && r <= '\x1f' + }), "") { + content = "\n" + message.Thinking + "\n\n" + content + } + switch message.Role { + case "assistant": + b.WriteString("<|im_start|>assistant") + if content != "" || len(message.ToolCalls) == 0 { + b.WriteString("\n" + content) + } + for _, call := range message.ToolCalls { + name, err := PythonJSON(call.Function.Name, 0) + if err != nil { + return "", err + } + arguments, err := PythonJSON(call.Function.Arguments, 0) + if err != nil { + return "", err + } + b.WriteString("\n\n{\"name\": " + name + ", \"arguments\": " + arguments + "}\n") + } + b.WriteString("<|im_end|>\n") + case "tool": + if i == 0 || messages[i-1].Role != "tool" { + b.WriteString("<|im_start|>user") + } + b.WriteString("\n\n" + content + "\n") + if i == len(messages)-1 || messages[i+1].Role != "tool" { + b.WriteString("<|im_end|>\n") + } + case "system", "user": + b.WriteString("<|im_start|>" + message.Role + "\n") + b.WriteString(content) + b.WriteString("<|im_end|>\n") + } + } + b.WriteString("<|im_start|>assistant\n") + return b.String(), nil +} diff --git a/model/renderers/ripple_test.go b/model/renderers/ripple_test.go new file mode 100644 index 000000000..408ddfeaf --- /dev/null +++ b/model/renderers/ripple_test.go @@ -0,0 +1,292 @@ +package renderers + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/model/parsers" +) + +const rippleTemplate = "testdata/ripple_chat_template.jinja" + +func TestRipplePublisherTemplate(t *testing.T) { + verifyLFM3TemplateFixture(t, rippleTemplate, "877c2dbf365243dda14738ca250b31398059b35ffd91d4ed0ec68be45ca6ebd3") +} + +func TestRippleRendererTools(t *testing.T) { + call := api.ToolCall{Function: api.ToolCallFunction{Name: "f\"<&é", Arguments: args(`{"z":"line\n\"quote\"\\<&é","a":[true,null,3]}`)}} + tests := []struct { + name string + messages []api.Message + want string + }{ + { + "call alone", + []api.Message{{Role: "assistant", ToolCalls: []api.ToolCall{call}}}, + "<|im_start|>assistant\n\n" + `{"name": "f\"<&é", "arguments": {"z": "line\n\"quote\"\\<&é", "a": [true, null, 3]}}` + "\n<|im_end|>\n<|im_start|>assistant\n", + }, + { + "multiple calls with reasoning", + []api.Message{{Role: "assistant", Thinking: "R", Content: " C \n", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "one"}}, {Function: api.ToolCallFunction{Name: "two"}}}}}, + "<|im_start|>assistant\n\nR\n\n C \n\n\n{\"name\": \"one\", \"arguments\": {}}\n\n\n{\"name\": \"two\", \"arguments\": {}}\n<|im_end|>\n<|im_start|>assistant\n", + }, + { + "grouped results", + []api.Message{{Role: "tool", Content: " A\n"}, {Role: "tool", Content: "B"}, {Role: "user", Content: "Next"}, {Role: "tool", Content: "C"}}, + "<|im_start|>user\n\n A\n\n\n\nB\n<|im_end|>\n<|im_start|>user\nNext<|im_end|>\n<|im_start|>user\n\nC\n<|im_end|>\n<|im_start|>assistant\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := RenderWithRenderer("ripple", tt.messages, nil, nil) + if err != nil { + t.Fatal(err) + } + if got != tt.want { + t.Fatalf("got %q\nwant %q", got, tt.want) + } + if os.Getenv("VERIFY_JINJA2") != "" { + want := renderQwenJinja(t, rippleTemplate, tt.messages, nil, nil, "") + if got != want { + t.Fatalf("Go %q\nPython %q", got, want) + } + } + }) + } +} + +func TestRippleRendererToolInstructions(t *testing.T) { + tool := api.Tool{Type: "function", Function: api.ToolFunction{Name: "execute_bash", Parameters: api.ToolFunctionParameters{Type: "object"}}} + for _, system := range []string{"", "Be precise."} { + messages := []api.Message{{Role: "system", Content: system}, {Role: "user", Content: "pwd"}} + got, err := RenderWithRenderer("ripple", messages, []api.Tool{tool}, nil) + if err != nil { + t.Fatal(err) + } + want := "<|im_start|>system\n" + system + "\n\n" + `# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{"type": "function", "function": {"name": "execute_bash", "parameters": {"type": "object", "properties": null}}} + + +For each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping: + +{"name": "execute_bash", "arguments": {"command": "pwd"}} +<|im_end|> +<|im_start|>user +pwd<|im_end|> +<|im_start|>assistant +` + if got != want { + t.Fatalf("got %q\nwant %q", got, want) + } + if os.Getenv("VERIFY_JINJA2") != "" { + want := renderQwenJinja(t, rippleTemplate, messages, []api.Tool{tool}, nil, "") + if got != want { + t.Fatalf("Go %q\nPython %q", got, want) + } + } + } +} + +func TestRippleRendererInvalidInput(t *testing.T) { + for _, tt := range []struct { + name string + messages []api.Message + tools []api.Tool + errorText string + }{ + {name: "empty transcript", errorText: "at least one message"}, + {name: "empty transcript with tools", tools: []api.Tool{{Type: "function"}}, errorText: "at least one message"}, + {name: "image", messages: []api.Message{{Role: "user", Content: "describe", Images: []api.ImageData{{1}}}}, errorText: "text-only"}, + {name: "invalid tool schema", messages: []api.Message{{Role: "user"}}, tools: []api.Tool{{Items: make(chan int)}}, errorText: "unsupported type"}, + {name: "invalid tool arguments", messages: []api.Message{{Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "f", Arguments: testArgs(map[string]any{"x": make(chan int)})}}}}}, errorText: "unsupported type"}, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := RenderWithRenderer("ripple", tt.messages, tt.tools, nil) + if got != "" || err == nil || !strings.Contains(err.Error(), tt.errorText) { + t.Fatalf("got (%q, %v), want %s", got, err, tt.errorText) + } + }) + } +} + +func TestRippleRendererToolResultContinuation(t *testing.T) { + messages := []api.Message{ + {Role: "user", Content: "Run the program and report what it prints."}, + {Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "execute_bash", Arguments: args(`{"command":"go run ./cmd/demo"}`)}}}}, + {Role: "tool", Content: `{"exit_code":0,"stdout":"fmt.Println(\"done\")","stderr":""}`}, + {Role: "assistant", Thinking: "The command succeeded; I can report its output.", Content: "The program prints `done`."}, + } + tools := []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "execute_bash"}}} + want := "<|im_start|>system\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"execute_bash\", \"parameters\": {\"type\": \"\", \"properties\": null}}}\n\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n<|im_end|>\n<|im_start|>user\nRun the program and report what it prints.<|im_end|>\n<|im_start|>assistant\n\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"go run ./cmd/demo\"}}\n<|im_end|>\n<|im_start|>user\n\n{\"exit_code\":0,\"stdout\":\"fmt.Println(\\\"done\\\")\",\"stderr\":\"\"}\n<|im_end|>\n<|im_start|>assistant\n\nThe command succeeded; I can report its output.\n\nThe program prints `done`.<|im_end|>\n<|im_start|>assistant\n" + got, err := (&RippleRenderer{}).Render(messages, tools, nil) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("render mismatch:\ngot:\n%s\nwant:\n%s", got, want) + } +} + +func TestRippleRendererJinjaParity(t *testing.T) { + if os.Getenv("VERIFY_JINJA2") == "" { + t.Skip("set VERIFY_JINJA2=1 and JINJA_PYTHON for publisher template parity") + } + values := api.NewToolCallFunctionArguments() + values.Set("z", json.Number("3.0")) + values.Set("a", json.RawMessage(`{"z": 1e-7, "a": ["\u2028\u2029<&é", true, null]}`)) + call := api.ToolCall{Function: api.ToolCallFunction{Name: "f\"\\\n<&é", Arguments: values}} + tool := api.Tool{Type: "function", Function: api.ToolFunction{Name: "f", Description: "Use \"quoted\" <&é\n", Parameters: api.ToolFunctionParameters{Type: "object", Properties: propsMap(`{"z":{"type":"integer"},"a":{"type":"string"}}`), Required: []string{"z"}}}} + for _, tt := range []struct { + name string + messages []api.Message + tools []api.Tool + think *api.ThinkValue + }{ + {name: "plain", messages: []api.Message{{Role: "user", Content: " \nHi\t "}}}, + {name: "empty user", messages: []api.Message{{Role: "user"}}}, + {name: "initial and later systems", messages: []api.Message{{Role: "system", Content: "A"}, {Role: "system", Content: "B"}, {Role: "user", Content: "C"}, {Role: "system", Content: "D"}}}, + {name: "tools without system", messages: []api.Message{{Role: "user", Content: "Call"}}, tools: []api.Tool{tool, tool}}, + {name: "old reasoning retained", messages: []api.Message{{Role: "assistant", Thinking: "old", Content: "first"}, {Role: "user", Content: "next"}, {Role: "assistant", Thinking: "new", Content: "second"}, {Role: "user", Content: "again"}}}, + {name: "no duplicate wrapper", messages: []api.Message{{Role: "assistant", Thinking: "ignored", Content: "\x1c\t\u2003oldanswer"}}}, + {name: "nonleading tag", messages: []api.Message{{Role: "assistant", Thinking: "new", Content: "wordsliteral"}}}, + {name: "thinking false has no invented prefix", messages: []api.Message{{Role: "assistant", Thinking: "retain", Content: "answer"}}, think: &api.ThinkValue{Value: false}}, + {name: "thinking true no forced opening", messages: []api.Message{{Role: "user", Content: "Hi"}}, think: &api.ThinkValue{Value: true}}, + {name: "tools JSON numbers escaping and order", messages: []api.Message{{Role: "assistant", Thinking: "Reason", Content: " \n ", ToolCalls: []api.ToolCall{call}}, {Role: "tool", Content: " result \n"}, {Role: "tool", Content: "other"}}, tools: []api.Tool{tool}}, + {name: "tools reasoning only", messages: []api.Message{{Role: "assistant", Thinking: "Reason", ToolCalls: []api.ToolCall{call}}}}, + {name: "group boundary ignores unknown role", messages: []api.Message{{Role: "tool", Content: "one"}, {Role: "ignored"}, {Role: "tool", Content: "two"}}}, + { + name: "tool result followed by assistant decision", + tools: []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "execute_bash"}}}, + messages: []api.Message{ + {Role: "user", Content: "Run the program and report what it prints."}, + {Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "execute_bash", Arguments: args(`{"command":"go run ./cmd/demo"}`)}}}}, + {Role: "tool", Content: `{"exit_code":0,"stdout":"fmt.Println(\"done\")","stderr":""}`}, + {Role: "assistant", Thinking: "The command succeeded; I can report its output.", Content: "The program prints `done`."}, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := RenderWithRenderer("ripple", tt.messages, tt.tools, tt.think) + if err != nil { + t.Fatal(err) + } + want := renderQwenJinja(t, rippleTemplate, tt.messages, tt.tools, tt.think, "") + if got != want { + t.Fatalf("Go %q\nPython %q", got, want) + } + }) + } + python := os.Getenv("JINJA_PYTHON") + if python == "" { + t.Fatal("set JINJA_PYTHON for negative publisher-template checks") + } + for _, messages := range []any{[]any{}, []any{map[string]any{"role": "user", "content": []any{map[string]any{"type": "image"}}}}} { + output, err := lfm3RunJinja2(t, python, rippleTemplate, map[string]any{"messages": messages, "add_generation_prompt": true}) + if err == nil { + t.Fatalf("publisher accepted unsupported transcript: %q", output) + } + } +} + +func TestRippleRendererParserRoundTrip(t *testing.T) { + for _, content := range []string{"", " ", "\n", "\n\n", " \t\n ", "answer\n", "日本語\n"} { + for _, thinking := range []string{"", "R", " \n", "\n\nR\t\n"} { + for _, calls := range [][]api.ToolCall{nil, {{Function: api.ToolCallFunction{Name: "f", Arguments: args(`{"text":"<&é\n\"quoted\""}`)}}}, {{Function: api.ToolCallFunction{Name: "f"}}, {Function: api.ToolCallFunction{Name: "g"}}}} { + message := api.Message{Role: "assistant", Content: content, Thinking: thinking, ToolCalls: calls} + original, err := RenderWithRenderer("ripple", []api.Message{message}, nil, nil) + if err != nil { + t.Fatal(err) + } + body := strings.TrimSuffix(strings.TrimPrefix(original, "<|im_start|>assistant\n"), "<|im_end|>\n<|im_start|>assistant\n") + p := parsers.ParserForName("ripple") + p.Init(nil, nil, nil) + parsed := api.Message{Role: "assistant"} + for i := 0; i <= len(body); i++ { + chunk := "" + if i < len(body) { + chunk = body[i : i+1] + } + c, r, tc, err := p.Add(chunk, i == len(body)) + if err != nil { + t.Fatal(err) + } + parsed.Content += c + parsed.Thinking += r + parsed.ToolCalls = append(parsed.ToolCalls, tc...) + } + got, err := RenderWithRenderer("ripple", []api.Message{parsed}, nil, nil) + if err != nil { + t.Fatal(err) + } + if got != original { + t.Fatalf("roundtrip got %q\nwant %q", got, original) + } + } + } + } +} + +func TestRippleRendererReasoningHistory(t *testing.T) { + for _, tt := range []struct{ name, content, reasoning, want string }{ + {"separate", " Answer \n", " Reason \n", "\n Reason \n\n\n Answer \n"}, + {"already tagged", "\noriginal\n\nAnswer", "ignored", "\noriginal\n\nAnswer"}, + {"leading whitespace already tagged", "\x1c\t\u2003originalAnswer", "ignored", "\x1c\t\u2003originalAnswer"}, + {"nonleading tag", "Text literal", "Reason", "\nReason\n\nText literal"}, + {"empty reasoning", "Answer", "", "Answer"}, + } { + t.Run(tt.name, func(t *testing.T) { + for _, think := range []*api.ThinkValue{nil, {Value: true}, {Value: false}, {Value: "high"}} { + got, err := RenderWithRenderer("ripple", []api.Message{{Role: "assistant", Content: tt.content, Thinking: tt.reasoning}, {Role: "user", Content: "Next"}}, nil, think) + if err != nil { + t.Fatal(err) + } + want := "<|im_start|>assistant\n" + tt.want + "<|im_end|>\n<|im_start|>user\nNext<|im_end|>\n<|im_start|>assistant\n" + if got != want { + t.Fatalf("got %q\nwant %q", got, want) + } + } + }) + } +} + +func TestRippleRendererMessages(t *testing.T) { + for _, tt := range []struct { + name string + messages []api.Message + want string + }{ + {"plain", []api.Message{{Role: "user", Content: " Hi\n"}}, "<|im_start|>user\n Hi\n<|im_end|>\n<|im_start|>assistant\n"}, + {"system", []api.Message{{Role: "system", Content: "Be precise."}, {Role: "user", Content: "Hi"}}, "<|im_start|>system\nBe precise.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n"}, + {"empty system", []api.Message{{Role: "system"}}, "<|im_start|>system\n<|im_end|>\n<|im_start|>assistant\n"}, + {"later system", []api.Message{{Role: "user", Content: "Hi"}, {Role: "system", Content: "Now"}}, "<|im_start|>user\nHi<|im_end|>\n<|im_start|>system\nNow<|im_end|>\n<|im_start|>assistant\n"}, + {"assistant is closed not prefilled", []api.Message{{Role: "assistant", Content: "Part"}}, "<|im_start|>assistant\nPart<|im_end|>\n<|im_start|>assistant\n"}, + {"unknown role ignored", []api.Message{{Role: "developer", Content: "ignored"}}, "<|im_start|>assistant\n"}, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := RenderWithRenderer("ripple", tt.messages, nil, nil) + if err != nil { + t.Fatal(err) + } + if got != tt.want { + t.Fatalf("got %q\nwant %q", got, tt.want) + } + }) + } + if rendererForName("ripple") == nil { + t.Fatal("missing ripple renderer") + } + if got := LeadingBOSForRenderer("ripple"); got != "" { + t.Fatalf("BOS = %q", got) + } + if got := ThinkingForRenderer("ripple"); got != nil { + t.Fatalf("unexpected thinking control: %+v", got) + } +} diff --git a/model/renderers/testdata/ripple_chat_template.jinja b/model/renderers/testdata/ripple_chat_template.jinja new file mode 100644 index 000000000..e4b0bd675 --- /dev/null +++ b/model/renderers/testdata/ripple_chat_template.jinja @@ -0,0 +1,52 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- set content = message.content or '' %} + {%- if message.role == 'assistant' and message.reasoning_content and not (clear_thinking | default(false)) and not content.lstrip().startswith('') %} + {%- set content = '\n' + message.reasoning_content + '\n\n' + content %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if content %} + {{- '\n' + content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": ' }} + {{- tool_call.name | tojson }} + {{- ', "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' + message.content + '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %}