diff --git a/create/metadata.go b/create/metadata.go
index bce1c7742..84e2d7336 100644
--- a/create/metadata.go
+++ b/create/metadata.go
@@ -282,6 +282,8 @@ func parserNameForConfig(modelDir string, cfg sourceModelConfig, chatTemplate st
func parserNameForIdentifier(modelDir, s, chatTemplate string) (string, error) {
s = strings.ToLower(s)
switch {
+ case s == "ricursiveforcausallm" || s == "ricursive":
+ return "ripple", nil
case strings.Contains(s, "flint"):
return "flint", nil
case strings.HasPrefix(s, "lfm3"):
@@ -322,6 +324,8 @@ func rendererNameForConfig(modelDir string, cfg sourceModelConfig, chatTemplate
func rendererNameForIdentifier(modelDir, s, chatTemplate string) (string, error) {
s = strings.ToLower(s)
switch {
+ case s == "ricursiveforcausallm" || s == "ricursive":
+ return "ripple", nil
case strings.Contains(s, "flint"):
return "flint", nil
case strings.HasPrefix(s, "lfm3"):
diff --git a/create/metadata_test.go b/create/metadata_test.go
index d3872d1af..ccf3d6382 100644
--- a/create/metadata_test.go
+++ b/create/metadata_test.go
@@ -102,6 +102,23 @@ func TestInferSafetensorsConfigFamilies(t *testing.T) {
wantRenderer string
wantCaps []string
}{
+ {
+ name: "ripple architecture",
+ config: `{"architectures":["RicursiveForCausalLM"],"model_type":"ricursive"}`,
+ chatTemplate: `{% for message in messages %}{{ message.content }}{% endfor %}`,
+ wantParser: "ripple",
+ wantRenderer: "ripple",
+ wantCaps: []string{"completion", "tools", "thinking"},
+ },
+ {
+ name: "ripple model type",
+ config: `{"model_type":"ricursive"}`,
+ chatTemplate: `{{ reasoning_content }}{{ content }}`,
+ standalone: true,
+ wantParser: "ripple",
+ wantRenderer: "ripple",
+ wantCaps: []string{"completion", "tools", "thinking"},
+ },
{
name: "qwen3",
config: `{"architectures":["Qwen3ForCausalLM"]}`,
diff --git a/mlxrunner/model/architectures/architectures.go b/mlxrunner/model/architectures/architectures.go
index 3b9d5d9a5..625324b7a 100644
--- a/mlxrunner/model/architectures/architectures.go
+++ b/mlxrunner/model/architectures/architectures.go
@@ -23,5 +23,6 @@ import (
_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5"
_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5_moe"
_ "github.com/ollama/ollama/mlxrunner/model/qwen4_exp"
+ _ "github.com/ollama/ollama/mlxrunner/model/ripple"
_ "github.com/ollama/ollama/mlxrunner/model/strands"
)
diff --git a/mlxrunner/model/dflash/cache_test.go b/mlxrunner/model/dflash/cache_test.go
index b99d15fc8..1d4ff56f8 100644
--- a/mlxrunner/model/dflash/cache_test.go
+++ b/mlxrunner/model/dflash/cache_test.go
@@ -3,12 +3,14 @@ package dflash
import (
"fmt"
"math"
+ "slices"
"testing"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlx/mlxtest"
"github.com/ollama/ollama/mlxrunner/batch"
"github.com/ollama/ollama/mlxrunner/cache"
+ "github.com/ollama/ollama/mlxrunner/model"
"github.com/ollama/ollama/mlxrunner/nn"
)
@@ -120,6 +122,298 @@ func TestRepeatedBlockRollback(t *testing.T) {
}
}
+// The scalar oracle uses nonzero queries and checks both cache K/V and output,
+// so skipping only context K, block K, or block Q cannot satisfy the contract.
+func TestDraftPerLayerNoRoPE(t *testing.T) {
+ for _, policy := range []struct {
+ name, field string
+ noRoPE [2]bool
+ }{
+ {"omitted", "", [2]bool{}},
+ {"empty", `,"no_rope_layers":[]`, [2]bool{}},
+ {"full_nope", `,"no_rope_layers":[1]`, [2]bool{false, true}},
+ {"sliding_nope", `,"no_rope_layers":[0]`, [2]bool{true, false}},
+ } {
+ for _, path := range []struct {
+ name string
+ context, block bool
+ }{
+ {"context_only", true, false},
+ {"block_only", false, true},
+ {"combined", true, true},
+ } {
+ mlxtest.RunSubtest(t, policy.name+"/"+path.name, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ m := newDraftRotaryTestModel(t, policy.field)
+ caches := m.NewCaches()
+ for _, c := range caches {
+ defer c.Free()
+ }
+ var oracle draftRotaryOracle
+ // Seed the real caches without invoking either rotary path.
+ // Block-only must not fail because context maintenance is broken.
+ seed := []float32{0.25, -0.5, 0.75, 0.25, -0.5, 1}
+ for i, c := range caches {
+ c.(cache.Attention).Update(&batch.Batch{SeqOffsets: []int32{0}},
+ mlx.Zeros(mlx.DTypeFloat32, 1, 1, 3, 2), mlx.FromValues(seed, 1, 1, 3, 2))
+ oracle.keys[i] = make([]float64, len(seed))
+ for _, v := range seed {
+ oracle.values[i] = append(oracle.values[i], float64(v))
+ }
+ }
+ for round := range 2 {
+ start := caches[0].Offset()
+ b := &batch.Batch{SeqOffsets: []int32{int32(start)}}
+ var context, block []float64
+ if path.context {
+ context = []float64{1, 2, -3, 1}
+ b.Hidden = mlx.FromValues([]float32{1, 2, -3, 1}, 1, 2, 2)
+ }
+ if path.block {
+ block = []float64{1, 2, -2, 1, 0.5, -1}
+ b.InputIDs = mlx.FromValues([]int32{0, 1, 2}, 1, 3)
+ b.SeqQueryLens = []int32{3}
+ }
+ want := oracle.forward(context, block, start, policy.noRoPE)
+ hidden, aux := m.Forward(b, nil, caches)
+ label := fmt.Sprintf("round %d offset %d", round, start)
+ if !path.block {
+ if hidden != nil || aux != nil {
+ t.Fatal("context-only forward returned block hidden states")
+ }
+ } else {
+ if hidden == nil || aux != hidden {
+ t.Fatal("block forward did not return shared hidden/aux states")
+ }
+ checkDraftRotaryArray(t, label+" hidden", hidden, want)
+ }
+ for i, c := range caches {
+ if wantOffset := len(oracle.keys[i]) / 2; c.Offset() != wantOffset {
+ t.Fatalf("%s layer %d cache offset=%d want=%d", label, i, c.Offset(), wantOffset)
+ }
+ state := c.State()
+ if len(state) != 2 {
+ t.Fatalf("%s layer %d state roots=%d want=2", label, i, len(state))
+ }
+ checkDraftRotaryArray(t, fmt.Sprintf("%s layer %d keys", label, i), state[0], oracle.keys[i])
+ checkDraftRotaryArray(t, fmt.Sprintf("%s layer %d values", label, i), state[1], oracle.values[i])
+ }
+ }
+ })
+ })
+ }
+ }
+}
+
+type windowedDraftTarget struct {
+ *draftRotaryTarget
+ window int
+}
+
+func (t *windowedDraftTarget) DraftWindow() int { return t.window }
+
+func checkDraftWindowValues(t *mlxtest.T, label string, array *mlx.Array, want []float64) {
+ t.Helper()
+ array = mlx.Contiguous(array, false).AsType(mlx.DTypeFloat32)
+ mlx.Eval(array)
+ got := array.Floats()
+ wantFloat := make([]float32, len(want))
+ for i, v := range want {
+ wantFloat[i] = float32(v)
+ }
+ slices.Sort(got)
+ slices.Sort(wantFloat)
+ if !slices.Equal(got, wantFloat) {
+ t.Fatalf("%s values = %v, want %v", label, got, wantFloat)
+ }
+}
+
+func TestDraftWindowCache(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ target draftTarget
+ initial, wrapped [][]float64
+ }{
+ {
+ name: "default", target: &draftRotaryTarget{},
+ initial: [][]float64{{2, 3}, {0, 1, 2, 3}},
+ wrapped: [][]float64{{3, 4}, {0, 1, 2, 3, 4}},
+ },
+ {
+ name: "bounded", target: &windowedDraftTarget{draftRotaryTarget: &draftRotaryTarget{}, window: 3},
+ initial: [][]float64{{2, 3}, {1, 2, 3}},
+ wrapped: [][]float64{{3, 4}, {2, 3, 4}},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ m := &Model{
+ Config: &Config{SlidingWindow: 2},
+ Layers: []*Layer{{IsSliding: true}, {IsSliding: false}},
+ target: tc.target,
+ }
+ caches := m.NewCaches()
+ defer func() {
+ for _, c := range caches {
+ c.Free()
+ }
+ }()
+ initial := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{4}}
+ values := mlx.FromValues([]float32{0, 1, 2, 3}, 1, 1, 4, 1)
+ for i, c := range caches {
+ c.(cache.Attention).Update(initial, values, values)
+ checkDraftWindowValues(t, fmt.Sprintf("initial cache %d", i), c.(cache.Attention).View(initial).V(), tc.initial[i])
+ c.PrepareSnapshots([]int{4})
+ next := &batch.Batch{SeqOffsets: []int32{4}, SeqQueryLens: []int32{1}}
+ value := mlx.FromValue(float32(4))
+ value = mlx.Reshape(value, 1, 1, 1, 1)
+ c.(cache.Attention).Update(next, value, value)
+ checkDraftWindowValues(t, fmt.Sprintf("wrapped cache %d", i), c.(cache.Attention).View(next).V(), tc.wrapped[i])
+ snapshots := c.TakeSnapshots()
+ if len(snapshots) != 1 {
+ t.Fatalf("cache %d snapshots = %d", i, len(snapshots))
+ }
+ if !c.Restore(snapshots[0], 4) {
+ t.Fatalf("cache %d could not restore", i)
+ }
+ if snapshots[0] != nil {
+ snapshots[0].Close()
+ }
+ if c.Offset() != 4 {
+ t.Fatalf("cache %d offset = %d", i, c.Offset())
+ }
+ checkDraftWindowValues(t, fmt.Sprintf("restored cache %d", i), c.(cache.Attention).View(initial).V(), tc.initial[i])
+ }
+ })
+ })
+ }
+}
+
+type draftRotaryTarget struct {
+ model.Model
+ embeddings *mlx.Array
+}
+
+func (m *draftRotaryTarget) TokenEmbeddings(ids *mlx.Array) *mlx.Array {
+ return m.embeddings.TakeAxis(ids, 0)
+}
+
+func (*draftRotaryTarget) RawLogits(x *mlx.Array) *mlx.Array { return x }
+func (*draftRotaryTarget) SetAuxHiddenLayers([]int) {}
+func (*draftRotaryTarget) NumLayers() int { return 2 }
+
+func newDraftRotaryTestModel(t *mlxtest.T, field string) *Model {
+ t.Helper()
+ cfg, err := parseConfig(draftRotaryTestConfig(field))
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := &Model{
+ Config: cfg, Layers: make([]*Layer, cfg.NumHiddenLayers), tensorPrefix: "draft.",
+ target: &draftRotaryTarget{embeddings: mlx.FromValues([]float32{1, 2, -2, 1, 0.5, -1}, 3, 2)},
+ }
+ identity := mlx.FromValues([]float32{1, 0, 0, 1}, 2, 2)
+ ones := mlx.FromValues([]float32{1, 1}, 2)
+ zero := mlx.Zeros(mlx.DTypeFloat32, 2, 2)
+ tensors := map[string]*mlx.Array{
+ "draft.fc.weight": identity, "draft.hidden_norm.weight": ones, "draft.norm.weight": ones,
+ }
+ for i := range m.Layers {
+ prefix := fmt.Sprintf("draft.layers.%d.", i)
+ for _, name := range []string{"q_proj", "k_proj", "v_proj", "o_proj"} {
+ tensors[prefix+"self_attn."+name+".weight"] = identity
+ }
+ for _, name := range []string{"input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm"} {
+ tensors[prefix+name+".weight"] = ones
+ }
+ for _, name := range []string{"gate_proj", "up_proj", "down_proj"} {
+ tensors[prefix+"mlp."+name+".weight"] = zero
+ }
+ }
+ if err := m.LoadWeights(tensors); err != nil {
+ t.Fatal(err)
+ }
+ return m
+}
+
+func checkDraftRotaryArray(t *mlxtest.T, label string, array *mlx.Array, want []float64) {
+ t.Helper()
+ // Cache state can be a strided view; Floats requires contiguous storage.
+ array = mlx.Contiguous(array, false).AsType(mlx.DTypeFloat32)
+ mlx.Eval(array)
+ got := array.Floats()
+ if len(got) != len(want) {
+ t.Fatalf("%s length=%d want=%d", label, len(got), len(want))
+ }
+ for i, v := range got {
+ if math.IsNaN(float64(v)) || math.Abs(float64(v)-want[i]) > 2e-3*math.Max(1, math.Abs(want[i])) {
+ t.Errorf("%s[%d]: got %g want %g", label, i, v, want[i])
+ return
+ }
+ }
+}
+
+// This two-wide scalar model has identity projections, unit norms, zero MLPs,
+// and a window larger than its history. It does not call production RoPE/SDPA.
+type draftRotaryOracle struct {
+ keys, values [2][]float64
+}
+
+func draftRotaryNorm(x []float64) []float64 {
+ out := make([]float64, len(x))
+ for r := 0; r < len(x); r += 2 {
+ inv := 1 / math.Sqrt((x[r]*x[r]+x[r+1]*x[r+1])/2+1e-6)
+ out[r], out[r+1] = x[r]*inv, x[r+1]*inv
+ }
+ return out
+}
+
+func draftRotaryRotate(x []float64, start int) []float64 {
+ out := slices.Clone(x)
+ for r := 0; r < len(x); r += 2 {
+ s, c := math.Sincos(float64(start + r/2))
+ out[r], out[r+1] = x[r]*c-x[r+1]*s, x[r]*s+x[r+1]*c
+ }
+ return out
+}
+
+func (o *draftRotaryOracle) forward(context, block []float64, start int, noRoPE [2]bool) []float64 {
+ contextValues := draftRotaryNorm(context)
+ x := slices.Clone(block)
+ for layer := range o.keys {
+ contextKeys := draftRotaryNorm(contextValues)
+ values := draftRotaryNorm(x)
+ q := draftRotaryNorm(values)
+ k := slices.Clone(q)
+ if !noRoPE[layer] {
+ contextKeys = draftRotaryRotate(contextKeys, start)
+ q = draftRotaryRotate(q, start+len(context)/2)
+ k = draftRotaryRotate(k, start+len(context)/2)
+ }
+ o.keys[layer] = append(o.keys[layer], contextKeys...)
+ o.keys[layer] = append(o.keys[layer], k...)
+ o.values[layer] = append(o.values[layer], contextValues...)
+ o.values[layer] = append(o.values[layer], values...)
+ for r := 0; r < len(x); r += 2 {
+ scores := make([]float64, len(o.keys[layer])/2)
+ for j := range scores {
+ scores[j] = (q[r]*o.keys[layer][2*j] + q[r+1]*o.keys[layer][2*j+1]) / math.Sqrt(2)
+ }
+ maxScore := slices.Max(scores)
+ total := 0.0
+ for j, score := range scores {
+ scores[j] = math.Exp(score - maxScore)
+ total += scores[j]
+ }
+ for j, score := range scores {
+ x[r] += score / total * o.values[layer][2*j]
+ x[r+1] += score / total * o.values[layer][2*j+1]
+ }
+ }
+ }
+ return draftRotaryNorm(x)
+}
+
// meanAttention is one head of zero queries over width-wide rows: every output
// row is the mean of the value rows it may see, and values copy their input.
func meanAttention(width int, window int32) (*Config, *Attention) {
diff --git a/mlxrunner/model/dflash/dflash.go b/mlxrunner/model/dflash/dflash.go
index 5056844ae..04744f98a 100644
--- a/mlxrunner/model/dflash/dflash.go
+++ b/mlxrunner/model/dflash/dflash.go
@@ -7,6 +7,7 @@ import (
"encoding/json"
"fmt"
"math"
+ "slices"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlxrunner/batch"
@@ -49,6 +50,7 @@ type Config struct {
Scale float32
SlidingWindow int32
LayerTypes []string
+ NoRoPELayers []int
BlockSize int
MaskTokenID int32
@@ -143,6 +145,7 @@ type Attention struct {
OProj nn.LinearLayer
QNorm *nn.RMSNorm
KNorm *nn.RMSNorm
+ NoRoPE bool
// ctxInputNorm applies the layer's input norm to context rows (laguna).
ctxInputNorm *nn.RMSNorm
@@ -183,6 +186,7 @@ func parseConfig(data []byte) (*Config, error) {
VocabSize int32 `json:"vocab_size"`
HasOwnLMHead bool `json:"has_own_lm_head"`
LayerTypes []string `json:"layer_types"`
+ NoRoPELayers []int `json:"no_rope_layers"`
SlidingWindow int32 `json:"sliding_window"`
RopeInterleaved *bool `json:"rope_interleaved"`
}
@@ -200,6 +204,7 @@ func parseConfig(data []byte) (*Config, error) {
RopeTheta: raw.RopeTheta,
SlidingWindow: raw.SlidingWindow,
LayerTypes: raw.LayerTypes,
+ NoRoPELayers: raw.NoRoPELayers,
BlockSize: raw.DFlashConfig.BlockSize,
VocabSize: raw.VocabSize,
HasOwnLMHead: raw.HasOwnLMHead,
@@ -217,6 +222,11 @@ func parseConfig(data []byte) (*Config, error) {
if cfg.NumHiddenLayers > maxDraftLayers {
return nil, fmt.Errorf("dflash num_hidden_layers %d exceeds supported maximum %d", cfg.NumHiddenLayers, maxDraftLayers)
}
+ for _, id := range cfg.NoRoPELayers {
+ if id < 0 || id >= int(cfg.NumHiddenLayers) {
+ return nil, fmt.Errorf("dflash no_rope_layers index %d out of range for %d layers", id, cfg.NumHiddenLayers)
+ }
+ }
// The fused projection holds the query, key and value rows.
if (int64(cfg.NumAttentionHeads)+2*int64(cfg.NumKeyValueHeads))*int64(cfg.HeadDim) > math.MaxInt32 {
return nil, fmt.Errorf("dflash attention projection dimensions exceed int32")
@@ -391,8 +401,9 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {
layer := &Layer{
IsSliding: m.LayerTypes[i] == "sliding_attention",
Attention: &Attention{
- GProj: linears.Make(layerPrefix + ".self_attn.g_proj"),
- OProj: linears.Make(layerPrefix + ".self_attn.o_proj"),
+ NoRoPE: slices.Contains(m.NoRoPELayers, i),
+ GProj: linears.Make(layerPrefix + ".self_attn.g_proj"),
+ OProj: linears.Make(layerPrefix + ".self_attn.o_proj"),
},
MLP: &MLP{
DownProj: linears.Make(layerPrefix + ".mlp.down_proj"),
@@ -583,9 +594,19 @@ func (m *Model) BlockParams() (int, int32) { return m.BlockSize, m.MaskTokenID }
// NewCaches builds the per-layer context caches.
func (m *Model) NewCaches() []cache.Cache {
caches := make([]cache.Cache, len(m.Layers))
+ draftWindow := 0
+ if target, ok := m.target.(interface{ DraftWindow() int }); ok {
+ draftWindow = target.DraftWindow()
+ }
for i, layer := range m.Layers {
if layer.IsSliding {
- caches[i] = cache.NewRotatingKVCache(int(m.SlidingWindow))
+ window := int(m.SlidingWindow)
+ if draftWindow > 0 {
+ window = min(window, draftWindow)
+ }
+ caches[i] = cache.NewRotatingKVCache(window)
+ } else if draftWindow > 0 {
+ caches[i] = cache.NewRotatingKVCache(draftWindow)
} else {
caches[i] = cache.NewKVCache()
}
@@ -681,7 +702,9 @@ func (a *Attention) contextKV(hctx *mlx.Array, positions *mlx.Array, cfg *Config
k = a.KNorm.Forward(k, cfg.RMSNormEps)
k = mlx.Transpose(k, 0, 2, 1, 3)
v = mlx.Transpose(v, 0, 2, 1, 3)
- k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
+ if !a.NoRoPE {
+ k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
+ }
return k, v
}
@@ -703,8 +726,10 @@ func (a *Attention) Forward(x, ctxK, ctxV *mlx.Array, c cache.Cache, bb *batch.B
k = mlx.Transpose(k, 0, 2, 1, 3)
v = mlx.Transpose(v, 0, 2, 1, 3)
- q = mlx.RoPEWithBase(q, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
- k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
+ if !a.NoRoPE {
+ q = mlx.RoPEWithBase(q, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
+ k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
+ }
if ctxK != nil {
k = ctxK.Concatenate(2, k)
diff --git a/mlxrunner/model/dflash/dflash2_test.go b/mlxrunner/model/dflash/dflash2_test.go
index 13296a2a7..07840a59d 100644
--- a/mlxrunner/model/dflash/dflash2_test.go
+++ b/mlxrunner/model/dflash/dflash2_test.go
@@ -241,6 +241,89 @@ func TestSelectorConditionalProbabilities(t *testing.T) {
}
}
+func TestFourTapConvolutionOracle(t *testing.T) {
+ for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} {
+ for _, length := range []int{1, 2, 3, 4, 8, 16} {
+ mlxtest.RunSubtest(t, fmt.Sprintf("%v length %d", dtype, length), func(t *mlxtest.T) {
+ const width, group, taps = 32, 16, 4
+ values := func(n int) []float32 {
+ v := make([]float32, n)
+ for i := range v {
+ v[i] = float32(i%7-3) / 8
+ }
+ return v
+ }
+ h, d, s := values(length*width), values(length*taps*width/group), values(taps*width)
+ want := convolutionOracle(h, d, s, 1, length, width, taps, width/group, func(v float32) float32 {
+ return rounded(v, dtype == mlx.DTypeBFloat16)
+ })
+ inputs := []*mlx.Array{mlx.FromValues(h, 1, length, width).AsType(dtype), mlx.FromValues(d, 1, length, taps, width/group).AsType(dtype), mlx.FromValues(s, taps, width).AsType(dtype)}
+ for _, got := range []*mlx.Array{convolveGraph(inputs[0], inputs[1], inputs[2]), convolve(inputs[0], inputs[1], inputs[2])} {
+ got = mlx.Contiguous(got.AsType(mlx.DTypeFloat32), false)
+ mlx.Eval(got)
+ if !slices.Equal(got.Floats(), want) {
+ t.Fatal("four-tap convolution differs from scalar oracle")
+ }
+ }
+ })
+ }
+ }
+}
+
+func TestSelectorTop32Probabilities(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ const vocab, rank, k, length = 40, 256, 32, 3
+ pred, succ, projection := make([]float32, vocab*rank), make([]float32, vocab*rank), make([]float32, rank)
+ for id := range vocab {
+ for j := range rank {
+ pred[id*rank+j] = float32(id%3-1) / 32
+ succ[id*rank+j] = float32(id%5-2) / 64
+ }
+ }
+ for j := range projection {
+ projection[j] = 0.5
+ }
+ logits := make([]float32, length*vocab)
+ for i := range logits {
+ logits[i] = -float32(i%vocab) / 8
+ }
+ m := &model2{Selector: &candidateSelector{
+ Projection: nn.NewLinear(mlx.FromValues(projection, rank, 1).AsType(mlx.DTypeBFloat16), nil),
+ Predecessor: mlx.FromValues(pred, vocab, rank).AsType(mlx.DTypeBFloat16),
+ Successor: mlx.FromValues(succ, vocab, rank).AsType(mlx.DTypeBFloat16), TopK: k,
+ }}
+ sampler := sample.New(16)
+ defer sampler.Free()
+ sampler.Add(0, sample.Options{Temperature: 1, UseSeed: true, Seed: 42}, nil)
+ tokens, ids, probs := m.Select(mlx.FromValues([]float32{1, 1, 1}, 1, length, 1).AsType(mlx.DTypeBFloat16), mlx.FromValues(logits, 1, length, vocab).AsType(mlx.DTypeBFloat16), mlx.FromValues([]int32{0}, 1), func(ids, scores *mlx.Array) (*mlx.Array, *mlx.Array) {
+ tokens, distribution := sampler.SampleCandidates(0, ids, scores)
+ return tokens, distribution.Probs
+ })
+ mlx.Eval(tokens, ids, probs)
+ path, support, q := tokens.Ints(), ids.Ints(), mlx.Contiguous(probs, false).Floats()
+ previous := 0
+ for row := range length {
+ weights, total := make([]float64, k), float64(0)
+ for id := range k {
+ weights[id] = math.Exp(-float64(id)/8 + float64((previous%3-1)*(id%5-2))/16)
+ total += weights[id]
+ }
+ for j, id := range support[row*k : (row+1)*k] {
+ if id < 0 || id >= k {
+ t.Fatalf("unexpected candidate %d", id)
+ }
+ if want := weights[id] / total; math.Abs(float64(q[row*k+j])-want) > 1e-6 {
+ t.Fatalf("row %d candidate %d probability=%g want=%g", row, id, q[row*k+j], want)
+ }
+ }
+ if !slices.Contains(support[row*k:(row+1)*k], path[row]) {
+ t.Fatal("selected token outside proposal support")
+ }
+ previous = int(path[row])
+ }
+ })
+}
+
func TestDFlash2ConfigValidation(t *testing.T) {
valid := Config{HiddenSize: 5120, ConvGroupSize: 16, ConvKernelSize: 2, BlockSize: 8, SelectorRank: 256, SelectorTopK: 16, VocabSize: 248320}
for _, tt := range []struct {
diff --git a/mlxrunner/model/dflash/dflash_test.go b/mlxrunner/model/dflash/dflash_test.go
index 42654a485..0cc503f43 100644
--- a/mlxrunner/model/dflash/dflash_test.go
+++ b/mlxrunner/model/dflash/dflash_test.go
@@ -2,9 +2,11 @@ package dflash
import (
"encoding/json"
+ "fmt"
"maps"
"math"
"slices"
+ "strings"
"testing"
)
@@ -50,6 +52,50 @@ func TestDraftConfigBoundsAndCausality(t *testing.T) {
}
}
+// Keep the policy in JSON so these tests exercise parsing and loader wiring.
+func draftRotaryTestConfig(noRoPEField string) []byte {
+ return []byte(fmt.Sprintf(`{
+ "hidden_size": 2, "num_hidden_layers": 2,
+ "num_attention_heads": 1, "num_key_value_heads": 1, "head_dim": 2,
+ "rms_norm_eps": 0.000001, "rope_theta": 10000,
+ "layer_types": ["sliding_attention", "full_attention"], "sliding_window": 64,
+ "dflash_config": {"block_size": 4, "mask_token_id": 0, "target_layer_ids": [0], "causal": false}
+ %s
+ }`, noRoPEField))
+}
+
+func TestDraftConfigNoRoPELayers(t *testing.T) {
+ for _, tt := range []struct {
+ name, field string
+ valid bool
+ }{
+ {"omitted", "", true},
+ {"empty", `,"no_rope_layers":[]`, true},
+ {"first", `,"no_rope_layers":[0]`, true},
+ {"last", `,"no_rope_layers":[1]`, true},
+ {"unordered", `,"no_rope_layers":[1,0]`, true},
+ {"negative", `,"no_rope_layers":[-1]`, false},
+ {"past_last", `,"no_rope_layers":[2]`, false},
+ {"invalid_after_valid", `,"no_rope_layers":[0,2]`, false},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := parseConfig(draftRotaryTestConfig(tt.field))
+ if tt.valid {
+ if err != nil {
+ t.Fatal(err)
+ }
+ return
+ }
+ if err == nil {
+ t.Fatalf("parseConfig accepted invalid %s", tt.field)
+ }
+ if !strings.Contains(err.Error(), "no_rope_layers") {
+ t.Fatalf("error %q does not identify no_rope_layers", err)
+ }
+ })
+ }
+}
+
func FuzzDraftConfig(f *testing.F) {
f.Add([]byte(`{"hidden_size":32,"num_hidden_layers":2,"num_attention_heads":4,"num_key_value_heads":2,"head_dim":8,"is_causal":false,"vocab_size":64,"dflash_config":{"block_size":8,"mask_token_id":7,"target_layer_ids":[1,3],"conv_group_size":16,"conv_kernel_size":2,"selector_rank":4,"selector_top_k":2}}`))
f.Add([]byte(`{"num_hidden_layers":-1}`))
diff --git a/mlxrunner/model/ripple/forward.go b/mlxrunner/model/ripple/forward.go
new file mode 100644
index 000000000..957446331
--- /dev/null
+++ b/mlxrunner/model/ripple/forward.go
@@ -0,0 +1,156 @@
+package ripple
+
+import (
+ "math"
+
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlxrunner/batch"
+ "github.com/ollama/ollama/mlxrunner/cache"
+ "github.com/ollama/ollama/mlxrunner/nn"
+)
+
+func (m *Model) Forward(b *batch.Batch, caches []cache.Cache) (hidden, auxHidden *mlx.Array) {
+ if caches != nil && len(caches) < len(m.Layers) {
+ panic("ripple: cache count must cover target layers")
+ }
+ if caches != nil && b.InputIDs.Dim(0) != 1 {
+ panic("ripple: cached batches require one sequence")
+ }
+ h := scaleEmbedding(m.EmbedTokens.Forward(b.InputIDs), m.EmbedScale)
+ seed := h
+ positions := mlx.Add(mlx.Reshape(mlx.FromValues(b.SeqOffsets, len(b.SeqOffsets)), int32(len(b.SeqOffsets)), 1), mlx.Arange(0, float64(b.InputIDs.Dim(1)), 1, mlx.DTypeInt32))
+ cos, sin := rotaryTables(positions, m.HeadDim, m.RopeTheta, h.DType())
+ var features []*mlx.Array
+ for i, layer := range m.Layers {
+ var c cache.Cache
+ if caches != nil {
+ c = caches[i]
+ }
+ h = layer.Forward(h, seed, b, c, cos, sin, m.LayerTypes[i] == "sliding_attention", m.Config)
+ if next := len(features); next < len(m.auxHiddenLayers) && i == m.auxHiddenLayers[next] {
+ features = append(features, h)
+ }
+ }
+ h = rmsNorm(h, m.Norm, m.RMSNormEps)
+ if len(features) > 0 {
+ return h, mlx.Concatenate(features, -1)
+ }
+ return h, h
+}
+
+func (l *Layer) Forward(x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array {
+ u := rmsNorm(x, l.InputNorm, cfg.RMSNormEps)
+ a := l.Attention.Forward(u, seed, b, c, cos, sin, local, cfg)
+ h := mlx.Add(x, rmsNorm(a, l.PostAttentionNorm, cfg.RMSNormEps))
+ f := l.MLP.Forward(rmsNorm(h, l.PreFFNNorm, cfg.RMSNormEps), cfg.FFNClampThreshold)
+ return mlx.Add(h, rmsNorm(f, l.PostFFNNorm, cfg.RMSNormEps))
+}
+
+func (a *Attention) Forward(x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array {
+ B, L := int32(x.Dim(0)), int32(x.Dim(1))
+ q := mlx.Reshape(a.QProj.Forward(x), B, L, cfg.NumAttentionHeads, cfg.HeadDim)
+ k := mlx.Reshape(a.KProj.Forward(x), B, L, cfg.NumKeyValueHeads, cfg.HeadDim)
+ v := mlx.Reshape(a.VProj.Forward(blendValues(seed, x, a.ValueResLambda)), B, L, cfg.NumKeyValueHeads, cfg.HeadDim)
+ q = mlx.Transpose(rmsNorm(q, a.QNorm, cfg.RMSNormEps), 0, 2, 1, 3)
+ k = mlx.Transpose(rmsNorm(k, a.KNorm, cfg.RMSNormEps), 0, 2, 1, 3)
+ v = mlx.Transpose(v, 0, 2, 1, 3)
+ if local {
+ q, k = applyRotary(q, cos, sin), applyRotary(k, cos, sin)
+ }
+ mask := nn.CausalMask()
+ var kv nn.SDPAOption
+ if c != nil {
+ kv = nn.WithKVHistory(c.(cache.Attention).Update(b, k, v))
+ } else {
+ kv = nn.WithKV(k, v, b.SeqQueryLens)
+ if local {
+ mask = mask.Intersect(nn.SlidingWindowMask(b, int(L), cfg.SlidingWindow, k.DType()))
+ }
+ }
+ out := nn.ScaledDotProductAttention(b, q, cfg.Scale, kv, nn.WithMask(mask))
+ out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim)
+ return a.OProj.Forward(gatedAttention(out, a.GProj.Forward(x)))
+}
+
+// MLP is the clipped SwiGLU feed-forward block.
+type MLP struct {
+ GateProj, UpProj, DownProj nn.LinearLayer
+}
+
+var ffnActivation = mlx.Compile2("SwiGLUF32", func(gate, up *mlx.Array) *mlx.Array {
+ f := gate.AsType(mlx.DTypeFloat32)
+ silu := mlx.Mul(f, mlx.Sigmoid(f)).AsType(gate.DType())
+ return mlx.Mul(silu, up)
+}, mlx.Shapeless())
+
+var clippedFFNActivation = mlx.Compile3("ClippedSwiGLUF32", func(gate, up, threshold *mlx.Array) *mlx.Array {
+ gate = mlx.Minimum(mlx.Maximum(gate, mlx.FromValue(float32(math.Inf(-1)))), threshold).AsType(gate.DType())
+ up = mlx.Minimum(mlx.Maximum(up, threshold.Negative()), threshold).AsType(up.DType())
+ return ffnActivation(gate, up)
+}, mlx.Shapeless())
+
+func (m *MLP) Forward(x *mlx.Array, threshold *float32) *mlx.Array {
+ gate, up := m.GateProj.Forward(x), m.UpProj.Forward(x)
+ var activated *mlx.Array
+ if threshold != nil && *threshold > 0 {
+ activated = clippedFFNActivation(gate, up, mlx.FromValue(*threshold))
+ } else {
+ activated = ffnActivation(gate, up)
+ }
+ return m.DownProj.Forward(activated)
+}
+
+func rotaryTables(positions *mlx.Array, dim int32, theta float32, dtype mlx.DType) (*mlx.Array, *mlx.Array) {
+ inv := make([]float32, dim/2)
+ for i := range inv {
+ inv[i] = 1 / float32(math.Pow(float64(theta), float64(float32(2*i)/float32(dim))))
+ }
+ freq := mlx.Mul(mlx.ExpandDims(positions.AsType(mlx.DTypeFloat32), -1), mlx.FromValues(inv, len(inv)))
+ freq = mlx.Concatenate([]*mlx.Array{freq, freq}, -1)
+ return mlx.Cos(freq).AsType(dtype), mlx.Sin(freq).AsType(dtype)
+}
+
+func applyRotary(x, cos, sin *mlx.Array) *mlx.Array {
+ f := x.AsType(mlx.DTypeFloat32)
+ half := x.Dim(-1) / 2
+ first := f.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(0, half))
+ second := f.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(half, x.Dim(-1)))
+ rotated := mlx.Concatenate([]*mlx.Array{second.Negative(), first}, -1)
+ c := mlx.ExpandDims(cos.AsType(mlx.DTypeFloat32), 1)
+ s := mlx.ExpandDims(sin.AsType(mlx.DTypeFloat32), 1)
+ return mlx.Add(mlx.Mul(f, c), mlx.Mul(rotated, s)).AsType(x.DType())
+}
+
+func rmsNorm(x, weight *mlx.Array, eps float32) *mlx.Array {
+ // Round the normalized activation before multiplying the learned weight.
+ return mlx.Mul(mlx.RMSNormFn(x, nil, eps), weight)
+}
+
+func scaleEmbedding(x *mlx.Array, scale *float64) *mlx.Array {
+ if scale == nil {
+ return x
+ }
+ return mlx.MulScalar(x.AsType(mlx.DTypeFloat32), float32(*scale)).AsType(x.DType())
+}
+
+func attentionGate(x *mlx.Array) *mlx.Array {
+ return mlx.Sigmoid(x.AsType(mlx.DTypeFloat32)).AsType(x.DType())
+}
+
+var gatedAttention = mlx.Compile2("AttentionOutputGate", func(out, gate *mlx.Array) *mlx.Array {
+ return mlx.Mul(out, attentionGate(gate))
+}, mlx.Shapeless())
+
+func blendValues(seed, x, lambda *mlx.Array) *mlx.Array {
+ // Sigmoid rounds to the parameter dtype; lerp has one activation rounding.
+ a := mlx.Sigmoid(lambda.AsType(mlx.DTypeFloat32)).AsType(lambda.DType()).AsType(x.DType()).AsType(mlx.DTypeFloat32)
+ return interpolateValues(seed, x, a)
+}
+
+var interpolateValues = mlx.Compile3("ValueBlend", func(seed, x, a *mlx.Array) *mlx.Array {
+ start, end := seed.AsType(mlx.DTypeFloat32), x.AsType(mlx.DTypeFloat32)
+ delta := mlx.Sub(end, start)
+ low := mlx.Add(start, mlx.Mul(a, delta))
+ high := mlx.Sub(end, mlx.Mul(mlx.Sub(mlx.FromValue(float32(1)), a), delta))
+ return mlx.Where(a.Less(mlx.FromValue(float32(0.5))), low, high).AsType(x.DType())
+}, mlx.Shapeless())
diff --git a/mlxrunner/model/ripple/ripple.go b/mlxrunner/model/ripple/ripple.go
new file mode 100644
index 000000000..feb99527e
--- /dev/null
+++ b/mlxrunner/model/ripple/ripple.go
@@ -0,0 +1,318 @@
+// Package ripple implements the Ricursive transformer.
+package ripple
+
+import (
+ "encoding/json"
+ "fmt"
+ "math"
+ "slices"
+ "strings"
+
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlx/quant"
+ "github.com/ollama/ollama/mlxrunner/cache"
+ "github.com/ollama/ollama/mlxrunner/nn"
+ "github.com/ollama/ollama/mlxrunner/tokenizer"
+
+ "github.com/ollama/ollama/mlxrunner/model"
+)
+
+func init() { model.Register("RicursiveForCausalLM", newModel) }
+
+// Model retains all loaded arrays; value-residual seeds belong to Forward.
+type Model struct {
+ *Config
+ EmbedTokens nn.EmbeddingLayer
+ Layers []*Layer
+ Norm *mlx.Array
+ LMHead nn.LinearLayer
+ tok *tokenizer.Tokenizer
+ auxHiddenLayers []int
+ draftTensorPrefix string
+}
+
+type Layer struct {
+ Attention *Attention
+ MLP *MLP
+ InputNorm, PostAttentionNorm, PreFFNNorm, PostFFNNorm *mlx.Array
+}
+
+type Attention struct {
+ QProj, KProj, VProj, OProj, GProj nn.LinearLayer
+ QNorm, KNorm, ValueResLambda *mlx.Array
+}
+
+func newModel(root *model.Root) (model.Model, error) {
+ cfg, err := loadConfig(root)
+ if err != nil {
+ return nil, err
+ }
+ configData, err := root.Manifest.ReadConfig("config.json")
+ if err != nil {
+ return nil, err
+ }
+ tokData, err := root.Manifest.ReadConfig("tokenizer.json")
+ if err != nil {
+ return nil, fmt.Errorf("load tokenizer: %w", err)
+ }
+ tc := &tokenizer.TokenizerConfig{ConfigJSON: configData}
+ tc.GenerationConfigJSON, _ = root.Manifest.ReadConfig("generation_config.json")
+ tc.TokenizerConfigJSON, _ = root.Manifest.ReadConfig("tokenizer_config.json")
+ tok, err := tokenizer.LoadFromBytesWithConfig(tokData, tc)
+ if err != nil {
+ return nil, fmt.Errorf("parse tokenizer: %w", err)
+ }
+ m := &Model{Config: cfg, Layers: make([]*Layer, cfg.NumHiddenLayers), tok: tok}
+ if root.Draft != nil {
+ m.draftTensorPrefix = root.Draft.TensorPrefix
+ if m.draftTensorPrefix == "" {
+ m.draftTensorPrefix = "draft."
+ }
+ }
+ return m, nil
+}
+
+func (m *Model) Tokenizer() *tokenizer.Tokenizer { return m.tok }
+func (m *Model) MaxContextLength() int { return int(m.MaxPositionEmbeddings) }
+func (m *Model) Unembed(x *mlx.Array) *mlx.Array { return m.LMHead.Forward(x) }
+func (m *Model) NumLayers() int { return len(m.Layers) }
+func (m *Model) SetAuxHiddenLayers(layers []int) { m.auxHiddenLayers = slices.Clone(layers) }
+func (m *Model) TokenEmbeddings(ids *mlx.Array) *mlx.Array { return m.EmbedTokens.Forward(ids) }
+func (m *Model) RawLogits(hidden *mlx.Array) *mlx.Array { return m.LMHead.Forward(hidden) }
+
+// DraftWindow bounds draft attention to its training context.
+func (*Model) DraftWindow() int { return 16384 }
+
+func (m *Model) NewCaches() []cache.Cache {
+ out := make([]cache.Cache, len(m.LayerTypes))
+ for i, kind := range m.LayerTypes {
+ if kind == "sliding_attention" {
+ out[i] = cache.NewRotatingKVCache(m.SlidingWindow)
+ } else {
+ out[i] = cache.NewKVCache()
+ }
+ }
+ return out
+}
+
+func (m *Model) weightShapes() map[string][]int {
+ h, d, f, v := int(m.HiddenSize), int(m.HeadDim), int(m.IntermediateSize), int(m.VocabSize)
+ q, k := int(m.NumAttentionHeads)*d, int(m.NumKeyValueHeads)*d
+ shapes := map[string][]int{"model.embed_tokens.weight": {v, h}, "lm_head.weight": {v, h}, "model.norm.weight": {h}}
+ for i := range m.NumHiddenLayers {
+ p := fmt.Sprintf("model.layers.%d.", i)
+ for name, shape := range map[string][]int{
+ "input_layernorm.weight": {h}, "post_attention_layernorm.weight": {h}, "pre_feedforward_layernorm.weight": {h}, "post_feedforward_layernorm.weight": {h},
+ "self_attn.q_proj.weight": {q, h}, "self_attn.k_proj.weight": {k, h}, "self_attn.v_proj.weight": {k, h}, "self_attn.o_proj.weight": {h, q}, "self_attn.g_proj.weight": {q, h},
+ "self_attn.q_norm.weight": {d}, "self_attn.k_norm.weight": {d}, "self_attn.value_res_lambda": {1},
+ "mlp.gate_proj.weight": {f, h}, "mlp.up_proj.weight": {f, h}, "mlp.down_proj.weight": {h, f},
+ } {
+ shapes[p+name] = shape
+ }
+ }
+ return shapes
+}
+
+func floatTensor(a *mlx.Array) bool {
+ return a.DType() == mlx.DTypeFloat32 || a.DType() == mlx.DTypeBFloat16 || a.DType() == mlx.DTypeFloat16
+}
+
+func (m *Model) validateWeights(tensors map[string]*mlx.Array) error {
+ allowed := make(map[string]bool)
+ for name, shape := range m.weightShapes() {
+ w := tensors[name]
+ if w == nil {
+ return fmt.Errorf("missing weight: %s", name)
+ }
+ allowed[name] = true
+ scales := tensors[name+"_scale"]
+ if scales == nil {
+ if !floatTensor(w) || !slices.Equal(w.Dims(), shape) {
+ return fmt.Errorf("invalid weight %s: dtype %v shape %v, want float %v", name, w.DType(), w.Dims(), shape)
+ }
+ if m.TensorQuant[name] != nil {
+ return fmt.Errorf("missing scale for quantized weight %s", name)
+ }
+ continue
+ }
+ if len(shape) != 2 {
+ return fmt.Errorf("cannot quantize %s", name)
+ }
+ if tq := m.TensorQuant[name]; tq != nil && quant.Canonical(tq.QuantType) == "" {
+ return fmt.Errorf("unsupported quantization for %s: %q", name, tq.QuantType)
+ }
+ gs, bits, mode := model.ResolveLinearQuantParams(m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant, name, w, scales)
+ if gs <= 0 || (bits != 4 && bits != 8) || (mode != "affine" && mode != "nvfp4" && mode != "mxfp4" && mode != "mxfp8") {
+ return fmt.Errorf("invalid quantization for %s", name)
+ }
+ if (mode == "nvfp4" && (gs != 16 || bits != 4)) || (mode == "mxfp4" && (gs != 32 || bits != 4)) || (mode == "mxfp8" && (gs != 32 || bits != 8)) {
+ return fmt.Errorf("invalid quantization group/bits for %s", name)
+ }
+ packed, ss := []int{shape[0], shape[1] * bits / 32}, []int{shape[0], shape[1] / gs}
+ if shape[1]%gs != 0 || shape[1]*bits%32 != 0 || w.DType() != mlx.DTypeUint32 || !slices.Equal(w.Dims(), packed) || !slices.Equal(scales.Dims(), ss) {
+ return fmt.Errorf("invalid quantized shape for %s: weight %v scale %v, want %v %v", name, w.Dims(), scales.Dims(), packed, ss)
+ }
+ allowed[name+"_scale"] = true
+ if mode == "affine" {
+ qb := tensors[name+"_qbias"]
+ if !floatTensor(scales) || qb == nil || !floatTensor(qb) || !slices.Equal(qb.Dims(), ss) {
+ return fmt.Errorf("invalid affine scale/bias for %s", name)
+ }
+ allowed[name+"_qbias"] = true
+ } else if scales.DType() != mlx.DTypeUint8 {
+ return fmt.Errorf("invalid packed scale dtype for %s", name)
+ }
+ for _, suffix := range []string{".global_scale", "_scale_2", ".input_scale", ".input_global_scale"} {
+ if a := tensors[name+suffix]; a != nil {
+ if mode != "nvfp4" || !floatTensor(a) || a.Size() != 1 {
+ return fmt.Errorf("invalid %s%s", name, suffix)
+ }
+ allowed[name+suffix] = true
+ }
+ }
+ }
+ for name := range tensors {
+ if !allowed[name] && !(m.draftTensorPrefix != "" && strings.HasPrefix(name, m.draftTensorPrefix)) {
+ return fmt.Errorf("unexpected weight: %s", name)
+ }
+ }
+ return nil
+}
+
+func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {
+ if err := m.validateWeights(tensors); err != nil {
+ return err
+ }
+ linears := model.NewLinearFactory(tensors, m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant)
+ m.EmbedTokens = model.MakeEmbeddingLayer(tensors, "model.embed_tokens", m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant)
+ m.LMHead = linears.Make("lm_head")
+ m.Norm = tensors["model.norm.weight"]
+ for i := range m.Layers {
+ p := fmt.Sprintf("model.layers.%d.", i)
+ m.Layers[i] = &Layer{
+ InputNorm: tensors[p+"input_layernorm.weight"], PostAttentionNorm: tensors[p+"post_attention_layernorm.weight"], PreFFNNorm: tensors[p+"pre_feedforward_layernorm.weight"], PostFFNNorm: tensors[p+"post_feedforward_layernorm.weight"],
+ Attention: &Attention{QProj: linears.Make(p + "self_attn.q_proj"), KProj: linears.Make(p + "self_attn.k_proj"), VProj: linears.Make(p + "self_attn.v_proj"), OProj: linears.Make(p + "self_attn.o_proj"), GProj: linears.Make(p + "self_attn.g_proj"), QNorm: tensors[p+"self_attn.q_norm.weight"], KNorm: tensors[p+"self_attn.k_norm.weight"], ValueResLambda: tensors[p+"self_attn.value_res_lambda"]},
+ MLP: &MLP{GateProj: linears.Make(p + "mlp.gate_proj"), UpProj: linears.Make(p + "mlp.up_proj"), DownProj: linears.Make(p + "mlp.down_proj")},
+ }
+ }
+ return nil
+}
+
+type Config struct {
+ HiddenSize int32 `json:"hidden_size"`
+ IntermediateSize int32 `json:"intermediate_size"`
+ NumHiddenLayers int32 `json:"num_hidden_layers"`
+ NumAttentionHeads int32 `json:"num_attention_heads"`
+ NumKeyValueHeads int32 `json:"num_key_value_heads"`
+ HeadDim int32 `json:"head_dim"`
+ VocabSize int32 `json:"vocab_size"`
+ MaxPositionEmbeddings int32 `json:"max_position_embeddings"`
+ RMSNormEps float32 `json:"rms_norm_eps"`
+ RopeTheta float32 `json:"rope_theta"`
+ SlidingWindow int `json:"sliding_window"`
+ LayerTypes []string `json:"layer_types"`
+ HiddenAct string `json:"hidden_act"`
+ UseGatedAttention bool `json:"use_gated_attention"`
+ GateFullPrecision bool `json:"gate_full_precision"`
+ UseValRes bool `json:"use_valres"`
+ TieWordEmbeddings bool `json:"tie_word_embeddings"`
+ AttentionBias bool `json:"attention_bias"`
+ MLPBias bool `json:"mlp_bias"`
+ FFNClampThreshold *float32 `json:"ffn_clamp_threshold"`
+ EmbedScale *float64 `json:"embed_scale"`
+ RopeScaling json.RawMessage `json:"rope_scaling"`
+ RopeParameters json.RawMessage `json:"rope_parameters"`
+
+ QuantGroupSize int `json:"-"`
+ QuantBits int `json:"-"`
+ QuantMode string `json:"-"`
+ TensorQuant map[string]*model.TensorQuantInfo `json:"-"`
+ Scale float32 `json:"-"`
+}
+
+func loadConfig(root *model.Root) (*Config, error) {
+ data, err := root.Manifest.ReadConfig("config.json")
+ if err != nil {
+ return nil, fmt.Errorf("load config: %w", err)
+ }
+ cfg := &Config{HiddenAct: "silu", RMSNormEps: 1e-6, RopeTheta: 10000, SlidingWindow: 1024, UseGatedAttention: true, GateFullPrecision: true}
+ if err := json.Unmarshal(data, cfg); err != nil {
+ return nil, fmt.Errorf("parse config: %w", err)
+ }
+ for _, v := range []struct {
+ name string
+ value int32
+ }{
+ {"hidden_size", cfg.HiddenSize},
+ {"intermediate_size", cfg.IntermediateSize},
+ {"num_hidden_layers", cfg.NumHiddenLayers},
+ {"num_attention_heads", cfg.NumAttentionHeads},
+ {"num_key_value_heads", cfg.NumKeyValueHeads},
+ {"head_dim", cfg.HeadDim},
+ {"vocab_size", cfg.VocabSize},
+ {"max_position_embeddings", cfg.MaxPositionEmbeddings},
+ } {
+ if v.value <= 0 {
+ return nil, fmt.Errorf("invalid %s: %d", v.name, v.value)
+ }
+ }
+ if cfg.HeadDim%2 != 0 {
+ return nil, fmt.Errorf("head_dim must be even")
+ }
+ if cfg.NumAttentionHeads%cfg.NumKeyValueHeads != 0 {
+ return nil, fmt.Errorf("num_attention_heads must be divisible by num_key_value_heads")
+ }
+ if int64(cfg.NumAttentionHeads)*int64(cfg.HeadDim) > math.MaxInt32 {
+ return nil, fmt.Errorf("num_attention_heads * head_dim overflows")
+ }
+ if cfg.RMSNormEps <= 0 {
+ return nil, fmt.Errorf("rms_norm_eps must be positive")
+ }
+ if cfg.RopeTheta <= 0 {
+ return nil, fmt.Errorf("rope_theta must be positive")
+ }
+ if cfg.SlidingWindow <= 0 {
+ return nil, fmt.Errorf("sliding_window must be positive")
+ }
+ if len(cfg.LayerTypes) != int(cfg.NumHiddenLayers) {
+ return nil, fmt.Errorf("layer_types must have num_hidden_layers entries")
+ }
+ for i, kind := range cfg.LayerTypes {
+ if kind != "sliding_attention" && kind != "full_attention" {
+ return nil, fmt.Errorf("unsupported layer_types[%d]: %q", i, kind)
+ }
+ }
+ if cfg.HiddenAct != "silu" {
+ return nil, fmt.Errorf("unsupported hidden_act: %q", cfg.HiddenAct)
+ }
+ for _, v := range []struct {
+ name string
+ unsupported bool
+ }{
+ {"use_gated_attention", !cfg.UseGatedAttention},
+ {"gate_full_precision", !cfg.GateFullPrecision},
+ {"use_valres", !cfg.UseValRes},
+ {"tie_word_embeddings", cfg.TieWordEmbeddings},
+ {"attention_bias", cfg.AttentionBias},
+ {"mlp_bias", cfg.MLPBias},
+ {"rope_scaling", len(cfg.RopeScaling) > 0 && string(cfg.RopeScaling) != "null"},
+ {"rope_parameters", len(cfg.RopeParameters) > 0 && string(cfg.RopeParameters) != "null"},
+ } {
+ if v.unsupported {
+ return nil, fmt.Errorf("unsupported %s", v.name)
+ }
+ }
+ if cfg.FFNClampThreshold != nil && *cfg.FFNClampThreshold < 0 {
+ return nil, fmt.Errorf("ffn_clamp_threshold must be nonnegative")
+ }
+ if cfg.EmbedScale != nil && (math.IsInf(*cfg.EmbedScale, 0) || math.IsNaN(*cfg.EmbedScale) || math.Abs(*cfg.EmbedScale) > math.MaxFloat32) {
+ return nil, fmt.Errorf("invalid embed_scale")
+ }
+ cfg.Scale = float32(1 / math.Sqrt(float64(cfg.HeadDim)))
+ cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode = model.QuantizationParams(root.QuantType())
+ if gs := root.GroupSize(); gs > 0 {
+ cfg.QuantGroupSize = gs
+ }
+ cfg.TensorQuant = root.AllTensorQuant()
+ return cfg, nil
+}
diff --git a/mlxrunner/model/ripple/ripple_test.go b/mlxrunner/model/ripple/ripple_test.go
new file mode 100644
index 000000000..fa13f73ce
--- /dev/null
+++ b/mlxrunner/model/ripple/ripple_test.go
@@ -0,0 +1,1393 @@
+package ripple
+
+import (
+ "crypto/sha256"
+ "encoding/binary"
+ "encoding/json"
+ "fmt"
+ "maps"
+ "math"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/ollama/ollama/manifest"
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlx/mlxtest"
+ "github.com/ollama/ollama/mlxrunner/batch"
+ "github.com/ollama/ollama/mlxrunner/cache"
+ "github.com/ollama/ollama/mlxrunner/model"
+ "github.com/ollama/ollama/mlxrunner/nn"
+ modeltypes "github.com/ollama/ollama/types/model"
+)
+
+func TestMain(m *testing.M) {
+ if err := os.Setenv("MLX_ENABLE_TF32", "0"); err != nil {
+ panic(err)
+ }
+ os.Exit(m.Run())
+}
+
+func fixtureConfig(t *testing.T) map[string]any {
+ t.Helper()
+ return map[string]any{
+ "architectures": []string{"RicursiveForCausalLM"},
+ "vocab_size": 19, "hidden_size": 8, "intermediate_size": 12,
+ "num_hidden_layers": 3, "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 4,
+ "layer_types": []string{"sliding_attention", "full_attention", "sliding_attention"},
+ "sliding_window": 3, "rms_norm_eps": 1e-6, "rope_theta": 10000.0,
+ "use_valres": true, "use_gated_attention": true, "gate_full_precision": true,
+ "ffn_clamp_threshold": 20.0, "embed_scale": 71.55417527999327,
+ "max_position_embeddings": 2097152, "tie_word_embeddings": false,
+ }
+}
+
+func testRoot(t *testing.T, cfg map[string]any) *model.Root {
+ t.Helper()
+ data, err := json.Marshal(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ root := &model.Root{Manifest: &manifest.Manifest{}}
+ for name, data := range map[string]string{
+ "config.json": string(data),
+ "tokenizer.json": `{"model":{"type":"BPE","vocab":{"a":0},"merges":[]}}`,
+ } {
+ layer, err := manifest.NewLayer(strings.NewReader(data), manifest.MediaTypeImageJSON)
+ if err != nil {
+ t.Fatal(err)
+ }
+ layer.Name = name
+ root.Manifest.Layers = append(root.Manifest.Layers, layer)
+ }
+ return root
+}
+
+type referenceTensor struct {
+ Data []float32 `json:"data"`
+ Shape []int `json:"shape"`
+ DType string `json:"dtype"`
+}
+
+func (r referenceTensor) array() *mlx.Array {
+ var a *mlx.Array
+ if len(r.Shape) == 0 {
+ a = mlx.FromValue(r.Data[0])
+ } else {
+ a = mlx.FromValues(r.Data, r.Shape...)
+ }
+ if r.DType == "bfloat16" {
+ a = a.AsType(mlx.DTypeBFloat16)
+ }
+ return a
+}
+
+type reference struct {
+ Weights map[string]map[string]referenceTensor
+}
+
+func readReference(t *testing.T) reference {
+ t.Helper()
+ m := &Model{Config: &Config{
+ HiddenSize: 8, IntermediateSize: 12, NumHiddenLayers: 3,
+ NumAttentionHeads: 4, NumKeyValueHeads: 2, HeadDim: 4, VocabSize: 19,
+ }}
+ r := reference{Weights: make(map[string]map[string]referenceTensor)}
+ for _, dtype := range []string{"float32", "bfloat16"} {
+ weights := make(map[string]referenceTensor)
+ for name, shape := range m.weightShapes() {
+ size := 1
+ for _, dim := range shape {
+ size *= dim
+ }
+ values := make([]float32, size)
+ hash := sha256.Sum256([]byte(name))
+ phase := int(binary.BigEndian.Uint32(hash[:4]) % 101)
+ if strings.HasSuffix(name, "value_res_lambda") {
+ layer := strings.Split(name, ".")[2]
+ values[0] = map[string]float32{"0": -1.25, "1": 0.75, "2": 1.5}[layer]
+ } else {
+ for i := range values {
+ v := float32((i*37+phase)%101-50) / 50
+ switch {
+ case strings.Contains(name, "norm.weight"):
+ scaled := v * 0.375
+ v = 1 + scaled
+ case strings.Contains(name, "embed_tokens"):
+ v *= 0.045
+ case strings.Contains(name, ".mlp.gate_proj") || strings.Contains(name, ".mlp.up_proj"):
+ v *= 18
+ default:
+ v *= 0.3
+ }
+ values[i] = v
+ }
+ }
+ if dtype == "bfloat16" {
+ for i, v := range values {
+ bits := math.Float32bits(v)
+ values[i] = math.Float32frombits((bits + 0x7fff + ((bits >> 16) & 1)) & 0xffff0000)
+ }
+ }
+ weights[name] = referenceTensor{Data: values, Shape: shape, DType: dtype}
+ }
+ r.Weights[dtype] = weights
+ }
+ return r
+}
+
+func checkTensor(t *mlxtest.T, label string, got *mlx.Array, want referenceTensor, atol, rtol float64) {
+ t.Helper()
+ if !slices.Equal(got.Dims(), want.Shape) {
+ t.Fatalf("%s shape=%v want %v", label, got.Dims(), want.Shape)
+ }
+ a := mlx.Contiguous(got.AsType(mlx.DTypeFloat32), false)
+ mlx.Eval(a)
+ values := a.Floats()
+ for i, v := range values {
+ d := math.Abs(float64(v - want.Data[i]))
+ bound := atol + rtol*math.Abs(float64(want.Data[i]))
+ if math.IsNaN(float64(v)) || d > bound {
+ t.Fatalf("%s[%d]=%.9g want %.9g (abs error %.9g > %.9g)", label, i, v, want.Data[i], d, bound)
+ }
+ }
+}
+
+type activationLinear struct {
+ value *mlx.Array
+ width int32
+}
+
+func (l activationLinear) Forward(x *mlx.Array) *mlx.Array {
+ if l.value != nil {
+ return l.value
+ }
+ return x
+}
+
+func (l activationLinear) OutputDim() int32 { return l.width }
+
+func clipFFN(gate, up *mlx.Array, threshold *float32) (*mlx.Array, *mlx.Array) {
+ if threshold != nil && *threshold > 0 {
+ gate = mlx.Clamp(gate, float32(math.Inf(-1)), *threshold).AsType(gate.DType())
+ up = mlx.Clamp(up, -*threshold, *threshold).AsType(up.DType())
+ }
+ return gate, up
+}
+
+func eagerFFN(gate, up *mlx.Array, threshold *float32) *mlx.Array {
+ gate, up = clipFFN(gate, up, threshold)
+ f := gate.AsType(mlx.DTypeFloat32)
+ silu := mlx.Mul(f, mlx.Sigmoid(f)).AsType(gate.DType())
+ return mlx.Mul(silu, up)
+}
+
+func TestMLPActivationFusionMemory(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.EnableCompile()
+ const width, rows = 1024, 256
+ gateValues, upValues := make([]float32, rows*width), make([]float32, rows*width)
+ for i := range gateValues {
+ gateValues[i] = float32(i%513-256) / 8
+ upValues[i] = float32((i*7)%513-256) / 8
+ }
+ gate := mlx.FromValues(gateValues, 1, rows, width)
+ up := mlx.FromValues(upValues, 1, rows, width)
+ mlx.Eval(gate, up)
+ m := MLP{GateProj: activationLinear{gate, width}, UpProj: activationLinear{up, width}, DownProj: activationLinear{width: width}}
+ threshold := float32(20)
+ eager := func() *mlx.Array { return eagerFFN(gate, up, &threshold) }
+ actual := func() *mlx.Array { return m.Forward(gate, &threshold) }
+ for _, fn := range []func() *mlx.Array{eager, actual} {
+ mlx.Scoped(func() { mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} }) })
+ }
+ measure := func(fn func() *mlx.Array) int {
+ mlx.DefaultStream().Synchronize()
+ mlx.ClearCache()
+ baseline := mlx.ActiveMemory()
+ mlx.ResetPeakMemory()
+ var peak int
+ mlx.Scoped(func() {
+ mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} })
+ mlx.DefaultStream().Synchronize()
+ peak = mlx.PeakMemory() - baseline
+ })
+ return peak
+ }
+ want, got := measure(eager), measure(actual)
+ t.Logf("activation temporary peak: eager=%d actual=%d", want, got)
+ if got >= want {
+ t.Fatalf("MLP activation peak %d did not improve on eager %d", got, want)
+ }
+ })
+}
+
+func TestMLPActivationPrecision(t *testing.T) {
+ for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} {
+ t.Run(fmt.Sprint(dtype), func(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.EnableCompile()
+ zero, small, large := float32(0), float32(2), float32(20)
+ for _, rows := range []int{1, 8, 3, 1} {
+ mlx.Scoped(func() {
+ const width = 256
+ gateValues, upValues := make([]float32, rows*width), make([]float32, rows*width)
+ for i := range gateValues {
+ gateValues[i] = float32(i%1025-512) / 16
+ upValues[i] = float32((i*29)%701-350) / 8
+ }
+ copy(gateValues, []float32{-40, -20.125, -20, -19.875, -2.015625, -2, -1.984375, -0.125, 0, 0.125, 1.984375, 2, 2.015625, 19.875, 20, 20.125, 40})
+ gate := mlx.FromValues(gateValues, 1, rows, width).AsType(dtype)
+ up := mlx.FromValues(upValues, 1, rows, width).AsType(dtype)
+ m := MLP{GateProj: activationLinear{gate, width}, UpProj: activationLinear{up, width}, DownProj: activationLinear{width: width}}
+ for _, threshold := range []*float32{nil, &zero, &small, &large, &small} {
+ want := eagerFFN(gate, up, threshold)
+ got := m.Forward(gate, threshold)
+ label := fmt.Sprintf("rows=%d unclipped", rows)
+ if threshold != nil {
+ label = fmt.Sprintf("rows=%d threshold=%g", rows, *threshold)
+ }
+ checkExactArray(t, label, got, want)
+ }
+ })
+ }
+ })
+ })
+ }
+}
+
+func eagerValueBlend(seed, x, lambda *mlx.Array) *mlx.Array {
+ a := mlx.Sigmoid(lambda.AsType(mlx.DTypeFloat32)).AsType(lambda.DType()).AsType(x.DType()).AsType(mlx.DTypeFloat32)
+ start, end := seed.AsType(mlx.DTypeFloat32), x.AsType(mlx.DTypeFloat32)
+ delta := mlx.Sub(end, start)
+ low := mlx.Add(start, mlx.Mul(a, delta))
+ high := mlx.Sub(end, mlx.Mul(mlx.Sub(mlx.FromValue(float32(1)), a), delta))
+ return mlx.Where(a.Less(mlx.FromValue(float32(0.5))), low, high).AsType(x.DType())
+}
+
+func TestValueBlendFusionMemory(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.EnableCompile()
+ const width, rows = 1024, 256
+ seedValues, xValues := make([]float32, rows*width), make([]float32, rows*width)
+ for i := range seedValues {
+ seedValues[i] = float32(i%257-128) / 64
+ xValues[i] = float32((i*7)%257-128) / 64
+ }
+ seed := mlx.FromValues(seedValues, 1, rows, width)
+ x := mlx.FromValues(xValues, 1, rows, width)
+ lambda := mlx.FromValues([]float32{-0.5}, 1)
+ mlx.Eval(seed, x, lambda)
+ eager := func() *mlx.Array { return eagerValueBlend(seed, x, lambda) }
+ actual := func() *mlx.Array { return blendValues(seed, x, lambda) }
+ for _, fn := range []func() *mlx.Array{eager, actual} {
+ mlx.Scoped(func() { mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} }) })
+ }
+ measure := func(fn func() *mlx.Array) int {
+ mlx.DefaultStream().Synchronize()
+ mlx.ClearCache()
+ baseline := mlx.ActiveMemory()
+ mlx.ResetPeakMemory()
+ var peak int
+ mlx.Scoped(func() {
+ mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} })
+ mlx.DefaultStream().Synchronize()
+ peak = mlx.PeakMemory() - baseline
+ })
+ return peak
+ }
+ want, got := measure(eager), measure(actual)
+ t.Logf("value-blend temporary peak: eager=%d actual=%d", want, got)
+ if got >= want {
+ t.Fatalf("value-blend peak %d did not improve on eager %d", got, want)
+ }
+ })
+}
+
+func TestValueBlendFusionPrecision(t *testing.T) {
+ for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} {
+ for _, parameterType := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} {
+ t.Run(fmt.Sprintf("%v/parameter_%v", dtype, parameterType), func(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.EnableCompile()
+ var maxError float64
+ for _, rows := range []int{1, 8, 3, 1} {
+ mlx.Scoped(func() {
+ const width = 256
+ seedValues, xValues := make([]float32, rows*width), make([]float32, rows*width)
+ for i := range seedValues {
+ seedValues[i] = float32(i%257-128) / 64
+ xValues[i] = float32((i*11)%257-128) / 64
+ }
+ seed := mlx.FromValues(seedValues, 1, rows, width).AsType(dtype)
+ x := mlx.FromValues(xValues, 1, rows, width).AsType(dtype)
+ for _, value := range []float32{-20, -4, -0.0078125, 0, 0.0078125, 4, 20} {
+ lambda := mlx.FromValues([]float32{value}, 1).AsType(parameterType)
+ want, got := eagerValueBlend(seed, x, lambda), blendValues(seed, x, lambda)
+ if got.DType() != dtype || !slices.Equal(got.Dims(), want.Dims()) {
+ t.Fatal("value blend changed dtype or shape")
+ }
+ wantValues := mlx.Contiguous(want.AsType(mlx.DTypeFloat32), false).Floats()
+ gotValues := mlx.Contiguous(got.AsType(mlx.DTypeFloat32), false).Floats()
+ atol := 1e-6
+ if dtype == mlx.DTypeBFloat16 {
+ atol = 0
+ }
+ for i, v := range gotValues {
+ delta := math.Abs(float64(v - wantValues[i]))
+ maxError = math.Max(maxError, delta)
+ if math.IsNaN(delta) || delta > atol {
+ t.Fatalf("rows=%d lambda=%g element=%d: error=%g exceeds %g", rows, value, i, delta, atol)
+ }
+ }
+ }
+ })
+ }
+ t.Logf("maximum absolute difference=%g", maxError)
+ })
+ })
+ }
+ }
+}
+
+// Pre-fusion attention path for the allocation comparison.
+func eagerAttentionForGateTest(a *Attention, x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array {
+ B, L := int32(x.Dim(0)), int32(x.Dim(1))
+ q := mlx.Reshape(a.QProj.Forward(x), B, L, cfg.NumAttentionHeads, cfg.HeadDim)
+ k := mlx.Reshape(a.KProj.Forward(x), B, L, cfg.NumKeyValueHeads, cfg.HeadDim)
+ v := mlx.Reshape(a.VProj.Forward(blendValues(seed, x, a.ValueResLambda)), B, L, cfg.NumKeyValueHeads, cfg.HeadDim)
+ q = mlx.Transpose(rmsNorm(q, a.QNorm, cfg.RMSNormEps), 0, 2, 1, 3)
+ k = mlx.Transpose(rmsNorm(k, a.KNorm, cfg.RMSNormEps), 0, 2, 1, 3)
+ v = mlx.Transpose(v, 0, 2, 1, 3)
+ if local {
+ q, k = applyRotary(q, cos, sin), applyRotary(k, cos, sin)
+ }
+ mask := nn.CausalMask()
+ var kv nn.SDPAOption
+ if c != nil {
+ kv = nn.WithKVHistory(c.(cache.Attention).Update(b, k, v))
+ } else {
+ kv = nn.WithKV(k, v, b.SeqQueryLens)
+ if local {
+ mask = mask.Intersect(nn.SlidingWindowMask(b, int(L), cfg.SlidingWindow, k.DType()))
+ }
+ }
+ out := nn.ScaledDotProductAttention(b, q, cfg.Scale, kv, nn.WithMask(mask))
+ out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim)
+ return a.OProj.Forward(mlx.Mul(out, attentionGate(a.GProj.Forward(x))))
+}
+
+func TestAttentionGateFusionMemory(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.ClearCache()
+ const heads, kvHeads, headDim = 80, 8, 128
+ const width, kvWidth = heads * headDim, kvHeads * headDim
+ qValues, gateValues := make([]float32, width), make([]float32, width)
+ vValues, normValues := make([]float32, kvWidth), make([]float32, headDim)
+ for i := range qValues {
+ qValues[i] = float32(i%17-8) / 32
+ gateValues[i] = float32(i%65-32) / 4
+ }
+ for i := range vValues {
+ vValues[i] = float32(i%13-6) / 8
+ }
+ for i := range normValues {
+ normValues[i] = 1
+ }
+ q := mlx.FromValues(qValues, 1, 1, width)
+ k := mlx.Zeros(mlx.DTypeFloat32, 1, 1, kvWidth)
+ v := mlx.FromValues(vValues, 1, 1, kvWidth)
+ gate := mlx.FromValues(gateValues, 1, 1, width)
+ norm := mlx.FromValues(normValues, headDim)
+ // Keep the unrelated value blend scalar in this allocation probe.
+ x := mlx.FromValues([]float32{1}, 1, 1, 1)
+ lambda := mlx.FromValues([]float32{0}, 1)
+ mlx.Eval(q, k, v, gate, norm, x, lambda)
+ a := Attention{
+ QProj: activationLinear{q, width}, KProj: activationLinear{k, kvWidth}, VProj: activationLinear{v, kvWidth},
+ GProj: activationLinear{gate, width}, OProj: activationLinear{width: width},
+ QNorm: norm, KNorm: norm, ValueResLambda: lambda,
+ }
+ cfg := Config{NumAttentionHeads: heads, NumKeyValueHeads: kvHeads, HeadDim: headDim, RMSNormEps: 1e-6, Scale: 1 / float32(math.Sqrt(headDim))}
+ b := inputBatch([]float32{1}, 0)
+ eager := func() *mlx.Array { return eagerAttentionForGateTest(&a, x, x, b, nil, nil, nil, false, &cfg) }
+ actual := func() *mlx.Array { return a.Forward(x, x, b, nil, nil, nil, false, &cfg) }
+ mlx.EnableCompile()
+ for _, fn := range []func() *mlx.Array{eager, actual} {
+ mlx.Scoped(func() { mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} }) })
+ }
+ measure := func(fn func() *mlx.Array) (int, []float32) {
+ mlx.DefaultStream().Synchronize()
+ mlx.ClearCache()
+ baseline := mlx.ActiveMemory()
+ mlx.ResetPeakMemory()
+ var peak int
+ var values []float32
+ mlx.Scoped(func() {
+ out := mlx.ScopedEval(func() []*mlx.Array { return []*mlx.Array{fn()} })[0]
+ mlx.DefaultStream().Synchronize()
+ peak = mlx.PeakMemory() - baseline
+ values = mlx.Contiguous(out, false).Floats()
+ })
+ return peak, values
+ }
+ eagerPeak, want := measure(eager)
+ fusedPeak, got := measure(actual)
+ if !slices.Equal(got, want) {
+ t.Fatal("compilation changed attention output")
+ }
+ t.Logf("attention temporary peak: eager=%d compiled=%d gate=%d", eagerPeak, fusedPeak, gate.NumBytes())
+ if eagerPeak-fusedPeak < gate.NumBytes() {
+ t.Fatalf("attention did not eliminate a gate-sized temporary: eager=%d compiled=%d gate=%d", eagerPeak, fusedPeak, gate.NumBytes())
+ }
+ })
+}
+
+func TestAttentionGateFusionPrecision(t *testing.T) {
+ for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} {
+ t.Run(fmt.Sprint(dtype), func(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.EnableCompile()
+ for _, rows := range []int{1, 8, 3, 1} {
+ mlx.Scoped(func() {
+ const width = 256
+ outValues, gateValues := make([]float32, rows*width), make([]float32, rows*width)
+ for i := range gateValues {
+ outValues[i] = float32((i*11)%257-128) / 16
+ gateValues[i] = float32(i%1025-512) / 32
+ }
+ copy(gateValues, []float32{-40, -20, -4, -0.0078125, 0, 0.0078125, 4, 20, 40})
+ out := mlx.FromValues(outValues, 1, rows, width).AsType(dtype)
+ gate := mlx.FromValues(gateValues, 1, rows, width).AsType(dtype)
+ want := mlx.Mul(out, mlx.Sigmoid(gate.AsType(mlx.DTypeFloat32)).AsType(dtype))
+ checkExactArray(t, fmt.Sprintf("rows=%d", rows), gatedAttention(out, gate), want)
+ })
+ }
+ })
+ })
+ }
+}
+
+func fixtureWeights(r reference, dtype string) map[string]*mlx.Array {
+ out := make(map[string]*mlx.Array)
+ for name, v := range r.Weights[dtype] {
+ out[name] = v.array()
+ }
+ return out
+}
+
+func TestLoadWeights(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ r := readReference(t)
+ root := testRoot(t, fixtureConfig(t))
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ loaded, err := model.New(root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := loaded.(*Model)
+ weights := fixtureWeights(r, "bfloat16")
+ if err := m.LoadWeights(weights); err != nil {
+ t.Fatal(err)
+ }
+ if len(m.Layers) != 3 || m.Tokenizer() == nil || m.MaxContextLength() != 2097152 {
+ t.Fatal("incomplete model")
+ }
+ for name, w := range weights {
+ delete(weights, name)
+ if err := m.LoadWeights(weights); err == nil || !strings.Contains(err.Error(), name) {
+ t.Fatalf("missing %s: %v", name, err)
+ }
+ weights[name] = mlx.Zeros(mlx.DTypeBFloat16, 1, 1)
+ if err := m.LoadWeights(weights); err == nil {
+ t.Fatalf("accepted bad shape for %s", name)
+ }
+ weights[name] = w
+ }
+ if err := m.LoadWeights(weights); err != nil {
+ t.Fatal(err)
+ }
+ caches := m.NewCaches()
+ defer func() {
+ for _, c := range caches {
+ c.Free()
+ }
+ }()
+ if _, ok := caches[0].(*cache.RotatingKVCache); !ok {
+ t.Fatal("local cache type")
+ }
+ if _, ok := caches[1].(*cache.KVCache); !ok {
+ t.Fatal("global cache type")
+ }
+ })
+}
+
+type prequantProjection struct {
+ name string
+ rows, cols int
+ global float32
+}
+
+var prequantProjections = []prequantProjection{
+ {"gate_proj", 64, 32, 0.5},
+ {"up_proj", 64, 32, 2},
+ {"down_proj", 32, 64, 0.125},
+}
+
+func prequantWeights() map[string]*mlx.Array {
+ shapes := map[string][]int{
+ "model.embed_tokens.weight": {32, 32}, "lm_head.weight": {32, 32}, "model.norm.weight": {32},
+ "model.layers.0.input_layernorm.weight": {32}, "model.layers.0.post_attention_layernorm.weight": {32},
+ "model.layers.0.pre_feedforward_layernorm.weight": {32}, "model.layers.0.post_feedforward_layernorm.weight": {32},
+ "model.layers.0.self_attn.q_proj.weight": {32, 32}, "model.layers.0.self_attn.k_proj.weight": {16, 32},
+ "model.layers.0.self_attn.v_proj.weight": {16, 32}, "model.layers.0.self_attn.o_proj.weight": {32, 32},
+ "model.layers.0.self_attn.g_proj.weight": {32, 32}, "model.layers.0.self_attn.q_norm.weight": {16},
+ "model.layers.0.self_attn.k_norm.weight": {16}, "model.layers.0.self_attn.value_res_lambda": {1},
+ }
+ weights := make(map[string]*mlx.Array)
+ for name, shape := range shapes {
+ size := shape[0]
+ if len(shape) == 2 {
+ size *= shape[1]
+ }
+ values := make([]float32, size)
+ for i := range values {
+ if len(shape) == 1 {
+ values[i] = 1
+ } else if i/shape[1] == i%shape[1] {
+ values[i] = 0.5
+ }
+ }
+ weights[name] = mlx.FromValues(values, shape...).AsType(mlx.DTypeBFloat16)
+ }
+ for _, p := range prequantProjections {
+ name := "model.layers.0.mlp." + p.name + ".weight"
+ packed := make([]uint32, p.rows*p.cols/8)
+ scales := make([]uint8, p.rows*p.cols/16)
+ for row := range p.rows {
+ for col := range p.cols {
+ packed[row*p.cols/8+col/8] |= uint32((row*3+col)%16) << (4 * (col % 8))
+ }
+ for group := range p.cols / 16 {
+ // E4M3 encodings for 0.5, 1, 2, and 4.
+ scales[row*p.cols/16+group] = []uint8{0x30, 0x38, 0x40, 0x48}[(row+group)%4]
+ }
+ }
+ weights[name] = mlx.FromValues(packed, p.rows, p.cols/8)
+ weights[name+"_scale"] = mlx.FromValues(scales, p.rows, p.cols/16)
+ weights[name+".global_scale"] = mlx.FromValue(p.global)
+ }
+ return weights
+}
+
+func prequantRoot(t *testing.T) *model.Root {
+ t.Helper()
+ root := testRoot(t, map[string]any{
+ "architectures": []string{"RicursiveForCausalLM"}, "hidden_size": 32, "intermediate_size": 64,
+ "num_hidden_layers": 1, "num_attention_heads": 2, "num_key_value_heads": 1, "head_dim": 16,
+ "vocab_size": 32, "max_position_embeddings": 64, "layer_types": []string{"full_attention"},
+ "use_valres": true, "ffn_clamp_threshold": 20,
+ })
+ path := filepath.Join(t.TempDir(), "weights.safetensors")
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ weights := prequantWeights()
+ metadata := make(map[string]string)
+ for _, p := range prequantProjections {
+ name := "model.layers.0.mlp." + p.name + ".weight"
+ weights[name+".scale"] = weights[name+"_scale"]
+ delete(weights, name+"_scale")
+ metadata[name+".quant_type"] = "nvfp4"
+ metadata[name+".group_size"] = "16"
+ }
+ if err := mlx.SaveSafetensorsWithMetadata(path, weights, metadata); err != nil {
+ t.Fatal(err)
+ }
+ })
+ })
+ f, err := os.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ layer, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor)
+ if err != nil {
+ t.Fatal(err)
+ }
+ root.Manifest.Layers = append(root.Manifest.Layers, layer)
+ config, err := manifest.NewLayer(strings.NewReader(`{"model_format":"safetensors"}`), manifest.MediaTypeImageConfig)
+ if err != nil {
+ t.Fatal(err)
+ }
+ const name = "ripple-prequant-test"
+ if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), config, root.Manifest.Layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil {
+ t.Fatal(err)
+ }
+ root, err = model.Open(name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return root
+}
+
+func loadPrequantWeights(t *mlxtest.T, root *model.Root) map[string]*mlx.Array {
+ t.Helper()
+ weights := make(map[string]*mlx.Array)
+ for _, layer := range root.Manifest.TensorLayers() {
+ path, err := manifest.BlobsPath(layer.Digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for name, weight := range mlx.Load(path) {
+ if strings.HasSuffix(name, ".scale") {
+ name = strings.TrimSuffix(name, ".scale") + "_scale"
+ }
+ weights[name] = weight
+ }
+ }
+ mlx.Eval(slices.Collect(maps.Values(weights))...)
+ return weights
+}
+
+func checkExactArray(t *mlxtest.T, name string, got, want *mlx.Array) {
+ t.Helper()
+ if got == nil || want == nil {
+ if got != want {
+ t.Fatalf("%s: mismatched nil arrays", name)
+ }
+ return
+ }
+ if got.DType() != want.DType() || !slices.Equal(got.Dims(), want.Dims()) {
+ t.Fatalf("%s: got %v %v, want %v %v", name, got.DType(), got.Dims(), want.DType(), want.Dims())
+ }
+ // Compare in the original dtype, including every bit of packed uint32 words.
+ for i, equal := range mlx.Contiguous(got.Equal(want).AsType(mlx.DTypeInt32), false).Ints() {
+ if equal != 1 {
+ t.Fatalf("%s: element %d differs", name, i)
+ }
+ }
+}
+
+func checkPrequantBasis(t *mlxtest.T, m *Model, withGlobal bool) {
+ t.Helper()
+ layers := []nn.LinearLayer{m.Layers[0].MLP.GateProj, m.Layers[0].MLP.UpProj, m.Layers[0].MLP.DownProj}
+ // Literal E2M1 values, independent of MLX's dequantizer.
+ fp4 := []float32{0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6}
+ block := []float32{0.5, 1, 2, 4}
+ for i, p := range prequantProjections {
+ basis := make([]float32, p.cols*p.cols)
+ want := make([]float32, p.cols*p.rows)
+ global := float32(1)
+ if withGlobal {
+ global = p.global
+ }
+ for col := range p.cols {
+ basis[col*p.cols+col] = 1
+ for row := range p.rows {
+ want[col*p.rows+row] = fp4[(row*3+col)%16] * block[(row+col/16)%4] * global
+ }
+ }
+ x := mlx.FromValues(basis, p.cols, p.cols).AsType(mlx.DTypeBFloat16)
+ checkTensor(t, p.name, layers[i].Forward(x), referenceTensor{Data: want, Shape: []int{p.cols, p.rows}}, 0, 0)
+ }
+}
+
+func TestLoadWeightsPrequantized(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ root := prequantRoot(t)
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ loaded, err := model.New(root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := loaded.(*Model)
+ weights := loadPrequantWeights(t, root)
+ if err := m.LoadWeights(weights); err != nil {
+ t.Fatal(err)
+ }
+ layers := []nn.LinearLayer{m.Layers[0].MLP.GateProj, m.Layers[0].MLP.UpProj, m.Layers[0].MLP.DownProj}
+ for i, p := range prequantProjections {
+ name := "model.layers.0.mlp." + p.name + ".weight"
+ if q := m.TensorQuant[name]; q == nil || q.QuantType != "NVFP4" || q.GroupSize != 16 {
+ t.Fatalf("%s metadata = %+v", name, q)
+ }
+ q, ok := layers[i].(*nn.QuantizedLinear)
+ if !ok {
+ t.Fatalf("%s type = %T", name, layers[i])
+ }
+ if q.Mode != "nvfp4" || q.Bits != 4 || q.GroupSize != 16 || q.QBiases != nil || q.Bias != nil {
+ t.Fatalf("%s lost quantization parameters", name)
+ }
+ if q.Weight != weights[name] || q.Scales != weights[name+"_scale"] {
+ t.Fatalf("%s did not preserve packed weights and scales", name)
+ }
+ if q.GlobalScale == nil || q.GlobalScale.DType() != mlx.DTypeFloat32 {
+ t.Fatalf("%s lost float32 global scale", name)
+ }
+ }
+ a := m.Layers[0].Attention
+ for name, linear := range map[string]nn.LinearLayer{
+ "lm_head.weight": m.LMHead,
+ "model.layers.0.self_attn.q_proj.weight": a.QProj, "model.layers.0.self_attn.k_proj.weight": a.KProj,
+ "model.layers.0.self_attn.v_proj.weight": a.VProj, "model.layers.0.self_attn.o_proj.weight": a.OProj,
+ "model.layers.0.self_attn.g_proj.weight": a.GProj,
+ } {
+ dense, ok := linear.(*nn.Linear)
+ if !ok || dense.Weight != weights[name] || dense.Weight.DType() != mlx.DTypeBFloat16 {
+ t.Fatalf("%s did not remain dense BF16", name)
+ }
+ }
+ embedding, ok := m.EmbedTokens.(*nn.Embedding)
+ if !ok || embedding.Weight != weights["model.embed_tokens.weight"] || embedding.Weight.DType() != mlx.DTypeBFloat16 {
+ t.Fatal("embedding did not remain dense BF16")
+ }
+ checkPrequantBasis(t, m, true)
+ })
+ })
+}
+
+func TestLoadWeightsPrequantizedScaleFormats(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ root := prequantRoot(t)
+ for _, suffix := range []string{"_scale_2", ""} {
+ name := "legacy_global"
+ if suffix == "" {
+ name = "block_only"
+ }
+ t.Run(name, func(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ loaded, err := model.New(root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := loaded.(*Model)
+ weights := loadPrequantWeights(t, root)
+ for _, p := range prequantProjections {
+ name := "model.layers.0.mlp." + p.name + ".weight"
+ if suffix != "" {
+ weights[name+suffix] = weights[name+".global_scale"]
+ }
+ delete(weights, name+".global_scale")
+ }
+ if err := m.LoadWeights(weights); err != nil {
+ t.Fatal(err)
+ }
+ checkPrequantBasis(t, m, suffix != "")
+ })
+ })
+ })
+ }
+}
+
+func TestLoadWeightsPrequantizedInvalid(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ root := prequantRoot(t)
+ const name = "model.layers.0.mlp.gate_proj.weight"
+ for _, tc := range []struct {
+ name, want string
+ mutate func(*Model, map[string]*mlx.Array)
+ }{
+ {"quant_type", "unsupported quantization", func(m *Model, _ map[string]*mlx.Array) {
+ m.TensorQuant[name].QuantType = "unknown"
+ }},
+ {"group_size", "invalid quantization group/bits", func(m *Model, _ map[string]*mlx.Array) {
+ m.TensorQuant[name].GroupSize = 32
+ }},
+ {"missing_scale", "missing scale for quantized weight", func(_ *Model, w map[string]*mlx.Array) {
+ delete(w, name+"_scale")
+ w[name] = mlx.Zeros(mlx.DTypeBFloat16, 64, 32)
+ }},
+ {"packed_without_scale", "invalid weight", func(_ *Model, w map[string]*mlx.Array) {
+ delete(w, name+"_scale")
+ }},
+ {"packed_dtype", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) {
+ w[name] = mlx.Zeros(mlx.DTypeUint8, 64, 4)
+ }},
+ {"packed_rows", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) {
+ w[name] = mlx.Zeros(mlx.DTypeUint32, 32, 4)
+ }},
+ {"packed_columns", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) {
+ w[name] = mlx.Zeros(mlx.DTypeUint32, 64, 8)
+ }},
+ {"scale_dtype", "invalid packed scale dtype", func(_ *Model, w map[string]*mlx.Array) {
+ w[name+"_scale"] = mlx.Zeros(mlx.DTypeFloat32, 64, 2)
+ }},
+ {"scale_rows", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) {
+ w[name+"_scale"] = mlx.Zeros(mlx.DTypeUint8, 32, 2)
+ }},
+ {"scale_columns", "invalid quantized shape", func(_ *Model, w map[string]*mlx.Array) {
+ w[name+"_scale"] = mlx.Zeros(mlx.DTypeUint8, 64, 1)
+ }},
+ {"global_dtype", "invalid " + name + ".global_scale", func(_ *Model, w map[string]*mlx.Array) {
+ w[name+".global_scale"] = mlx.FromValues([]uint32{1}, 1)
+ }},
+ {"global_shape", "invalid " + name + ".global_scale", func(_ *Model, w map[string]*mlx.Array) {
+ w[name+".global_scale"] = mlx.FromValues([]float32{1, 2}, 2)
+ }},
+ {"activation_shape", "invalid " + name + ".input_scale", func(_ *Model, w map[string]*mlx.Array) {
+ w[name+".input_scale"] = mlx.FromValues([]float32{1, 2}, 2)
+ }},
+ {"unexpected_bias", "unexpected weight: " + name + "_qbias", func(_ *Model, w map[string]*mlx.Array) {
+ w[name+"_qbias"] = mlx.Zeros(mlx.DTypeFloat32, 64, 2)
+ }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ loaded, err := model.New(root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := loaded.(*Model)
+ weights := loadPrequantWeights(t, root)
+ tc.mutate(m, weights)
+ err = m.LoadWeights(weights)
+ if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), name) {
+ t.Fatalf("LoadWeights error = %v, want %q for %s", err, tc.want, name)
+ }
+ })
+ })
+ })
+ }
+}
+
+func TestWeightLifetime(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ root := prequantRoot(t)
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ // Warm compiled constants before measuring weight ownership.
+ mlx.Scoped(func() {
+ w := mlx.Zeros(mlx.DTypeUint32, 32, 4)
+ s := mlx.Zeros(mlx.DTypeUint8, 32, 2)
+ g := mlx.FromValues([]float32{mlx.Nvfp4MaxProduct}, 1)
+ for _, shape := range [][]int{{1, 32}, {1, 1, 32}} {
+ x := mlx.Zeros(mlx.DTypeBFloat16, shape...)
+ mlx.Eval(mlx.QuantizedMatmul(x, w, s, nil, true, 16, 4, "nvfp4", g))
+ }
+ gate := mlx.Zeros(mlx.DTypeBFloat16, 1, 1, 32)
+ up := mlx.AddScalar(gate, float32(1))
+ mlx.Eval(clippedFFNActivation(gate, up, mlx.FromValue(float32(20))))
+ lambda := mlx.FromValues([]float32{0.25}, 1).AsType(mlx.DTypeBFloat16)
+ mlx.Eval(blendValues(gate, up, lambda))
+ })
+ mlx.DefaultStream().Synchronize()
+ baseline := mlx.ActiveMemory()
+ for cycle := range 2 {
+ held := mlx.NewScope()
+ defer held.Close()
+ var m *Model
+ var loadMemory, temporaryBytes int
+ retained := mlx.ScopedArrays(func() []*mlx.Array {
+ loaded, err := model.New(root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ m = loaded.(*Model)
+ weights := loadPrequantWeights(t, root)
+ if err := m.LoadWeights(weights); err != nil {
+ t.Fatal(err)
+ }
+ arrays := mlx.Collect(m)
+ // Check membership while every source handle is still live.
+ for name, w := range weights {
+ if strings.HasSuffix(name, ".global_scale") {
+ if slices.Contains(arrays, w) {
+ t.Fatalf("retained load-only source %s", name)
+ }
+ } else if !slices.Contains(arrays, w) {
+ t.Fatalf("model did not retain %s", name)
+ }
+ }
+ for i, linear := range []nn.LinearLayer{m.Layers[0].MLP.GateProj, m.Layers[0].MLP.UpProj, m.Layers[0].MLP.DownProj} {
+ q, ok := linear.(*nn.QuantizedLinear)
+ if !ok || q.GlobalScale == nil || !slices.Contains(arrays, q.GlobalScale) {
+ t.Fatalf("model did not retain converted global scale %d", i)
+ }
+ }
+ temporary := mlx.Zeros(mlx.DTypeFloat32, 256, 256)
+ mlx.Eval(temporary)
+ temporaryBytes = temporary.NumBytes()
+ mlx.DefaultStream().Synchronize()
+ loadMemory = mlx.ActiveMemory()
+ return arrays
+ })
+ mlx.DefaultStream().Synchronize()
+ if got := mlx.ActiveMemory(); got > loadMemory-temporaryBytes {
+ t.Fatalf("load temporary survived scope: active=%d, load=%d, temporary=%d", got, loadMemory, temporaryBytes)
+ }
+ mlx.Eval(retained...)
+ held.Attach(retained...)
+ var previous []float32
+ for request := range 2 {
+ mlx.Scoped(func() {
+ checkPrequantBasis(t, m, true)
+ hidden, _ := m.Forward(inputBatch([]float32{1, 4, 7}, 0), nil)
+ logits := mlx.Contiguous(m.Unembed(hidden).AsType(mlx.DTypeFloat32), false)
+ mlx.Eval(logits)
+ if !slices.Equal(logits.Dims(), []int{1, 3, 32}) {
+ t.Fatalf("logits shape = %v", logits.Dims())
+ }
+ values := logits.Floats()
+ nonzero := false
+ for _, v := range values {
+ if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {
+ t.Fatal("non-finite logits after load scope ended")
+ }
+ nonzero = nonzero || v != 0
+ }
+ if !nonzero {
+ t.Fatal("zero logits after load scope ended")
+ }
+ if request > 0 && !slices.Equal(values, previous) {
+ t.Fatal("weights changed between request scopes")
+ }
+ previous = slices.Clone(values)
+ })
+ }
+ held.Close()
+ mlx.DefaultStream().Synchronize()
+ if got := mlx.ActiveMemory(); got != baseline {
+ t.Fatalf("cycle %d leaked weight memory: active=%d, baseline=%d", cycle, got, baseline)
+ }
+ }
+ })
+}
+
+func TestDraftWeightNamespace(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ r := readReference(t)
+ for _, tc := range []struct {
+ name, prefix, extra, missing, wantError string
+ declared bool
+ }{
+ {name: "default", declared: true, extra: "draft.fc.weight"},
+ {name: "explicit", declared: true, prefix: "draft.", extra: "draft.fc.weight"},
+ {name: "custom", declared: true, prefix: "assistant.", extra: "assistant.fc.weight"},
+ {name: "undeclared", extra: "draft.fc.weight", wantError: "unexpected weight: draft.fc.weight"},
+ {name: "outside prefix", declared: true, prefix: "assistant.", extra: "draft.fc.weight", wantError: "unexpected weight: draft.fc.weight"},
+ {name: "unknown target", declared: true, extra: "model.unknown.weight", wantError: "unexpected weight: model.unknown.weight"},
+ {name: "missing target", declared: true, extra: "draft.fc.weight", missing: "lm_head.weight", wantError: "missing weight: lm_head.weight"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ root := testRoot(t, fixtureConfig(t))
+ if tc.declared {
+ root.Draft = &modeltypes.Draft{TensorPrefix: tc.prefix}
+ }
+ m, err := model.New(root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ tensors := fixtureWeights(r, "bfloat16")
+ tensors[tc.extra] = mlx.FromValues([]float32{2}, 1)
+ delete(tensors, tc.missing)
+ before := maps.Clone(tensors)
+ err := m.LoadWeights(tensors)
+ if tc.wantError != "" {
+ if err == nil || !strings.Contains(err.Error(), tc.wantError) {
+ t.Fatalf("LoadWeights error=%v, want %q", err, tc.wantError)
+ }
+ } else if err != nil {
+ t.Fatalf("declared draft namespace rejected: %v", err)
+ }
+ if !maps.Equal(before, tensors) {
+ t.Fatal("LoadWeights modified the caller's tensor map")
+ }
+ })
+ })
+ })
+ }
+}
+
+func TestDFlashTargetCombinedCaches(t *testing.T) {
+ m, r := dflashTargetFixture(t)
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ if err := m.LoadWeights(fixtureWeights(r, "bfloat16")); err != nil {
+ t.Fatal(err)
+ }
+ plain, combined := m.NewCaches(), m.NewCaches()
+ draft := []cache.Cache{cache.NewKVCache(), cache.NewRotatingKVCache(3)}
+ combined = append(combined, draft...)
+ defer func() {
+ for _, c := range append(plain, combined...) {
+ c.Free()
+ }
+ }()
+ b := inputBatch([]float32{1, 4, 7}, 0)
+ want, _ := m.Forward(b, plain)
+ mlx.Eval(want)
+ func() {
+ defer func() {
+ if p := recover(); p != nil {
+ t.Fatalf("target rejected trailing draft caches: %v", p)
+ }
+ }()
+ got, _ := m.Forward(b, combined)
+ checkExactArray(t, "combined-cache target hidden", got, want)
+ }()
+ for _, c := range combined[:len(m.Layers)] {
+ if c.Offset() != 3 {
+ t.Fatalf("target cache offset=%d, want 3", c.Offset())
+ }
+ }
+ for _, c := range draft {
+ if c.Offset() != 0 {
+ t.Fatalf("target forward modified draft cache offset: %d", c.Offset())
+ }
+ }
+ func() {
+ defer func() {
+ if p := recover(); p == nil || !strings.Contains(fmt.Sprint(p), "cache count") {
+ t.Fatalf("short cache slice panic=%v, want cache count rejection", p)
+ }
+ }()
+ m.Forward(b, plain[:len(plain)-1])
+ }()
+ })
+ })
+}
+
+func dflashTargetFixture(t *testing.T) (*Model, reference) {
+ t.Helper()
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ loaded, err := model.New(testRoot(t, fixtureConfig(t)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return loaded.(*Model), readReference(t)
+}
+
+func TestDFlashTargetNumLayers(t *testing.T) {
+ m, _ := dflashTargetFixture(t)
+ target, ok := any(m).(interface{ NumLayers() int })
+ if !ok {
+ t.Fatal("Ripple target is missing NumLayers")
+ }
+ if got := target.NumLayers(); got != 3 {
+ t.Fatalf("NumLayers() = %d, want 3 fixture blocks", got)
+ }
+}
+
+func TestDFlashTargetDraftWindow(t *testing.T) {
+ m, _ := dflashTargetFixture(t)
+ target, ok := any(m).(interface{ DraftWindow() int })
+ if !ok {
+ t.Fatal("Ripple target is missing DraftWindow")
+ }
+ if got := target.DraftWindow(); got != 16384 {
+ t.Fatalf("DraftWindow() = %d, want 16384", got)
+ }
+}
+
+func TestDFlashTargetTokenEmbeddings(t *testing.T) {
+ m, r := dflashTargetFixture(t)
+ target, ok := any(m).(interface {
+ TokenEmbeddings(*mlx.Array) *mlx.Array
+ })
+ if !ok {
+ t.Fatal("Ripple target is missing TokenEmbeddings")
+ }
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ if err := m.LoadWeights(fixtureWeights(r, "bfloat16")); err != nil {
+ t.Fatal(err)
+ }
+ if m.EmbedScale == nil || *m.EmbedScale == 1 {
+ t.Fatal("fixture must distinguish raw and scaled embeddings")
+ }
+ ids := []float32{1, 4, 7}
+ weight := r.Weights["bfloat16"]["model.embed_tokens.weight"]
+ width := weight.Shape[1]
+ var want []float32
+ for _, id := range ids {
+ start := int(id) * width
+ want = append(want, weight.Data[start:start+width]...)
+ }
+ got := target.TokenEmbeddings(inputBatch(ids, 0).InputIDs)
+ if got.DType() != mlx.DTypeBFloat16 {
+ t.Fatalf("embedding dtype = %v, want bfloat16", got.DType())
+ }
+ checkTensor(t, "raw table rows", got, referenceTensor{Data: want, Shape: []int{1, len(ids), width}}, 0, 0)
+ })
+ })
+}
+
+func TestDFlashTargetRawLogits(t *testing.T) {
+ m, r := dflashTargetFixture(t)
+ target, ok := any(m).(interface {
+ RawLogits(*mlx.Array) *mlx.Array
+ })
+ if !ok {
+ t.Fatal("Ripple target is missing RawLogits")
+ }
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ if err := m.LoadWeights(fixtureWeights(r, "bfloat16")); err != nil {
+ t.Fatal(err)
+ }
+ weight := r.Weights["bfloat16"]["lm_head.weight"]
+ vocab, width := weight.Shape[0], weight.Shape[1]
+ basis := make([]float32, width*width)
+ want := make([]float32, width*vocab)
+ for col := range width {
+ basis[col*width+col] = 2
+ for row := range vocab {
+ want[col*vocab+row] = 2 * weight.Data[row*width+col]
+ }
+ }
+ x := mlx.FromValues(basis, 1, width, width).AsType(mlx.DTypeBFloat16)
+ got := target.RawLogits(x)
+ if got.DType() != mlx.DTypeBFloat16 {
+ t.Fatalf("logit dtype = %v, want bfloat16", got.DType())
+ }
+ checkTensor(t, "unnormalized head columns", got, referenceTensor{Data: want, Shape: []int{1, width, vocab}}, 0, 0)
+ })
+ })
+}
+
+// Run the existing blocks directly, independently of Forward's tap selection.
+func dflashLayerOutputs(m *Model, b *batch.Batch) []*mlx.Array {
+ h := scaleEmbedding(m.EmbedTokens.Forward(b.InputIDs), m.EmbedScale)
+ seed := h
+ positions := mlx.Add(mlx.Reshape(mlx.FromValues(b.SeqOffsets, len(b.SeqOffsets)), int32(len(b.SeqOffsets)), 1), mlx.Arange(0, float64(b.InputIDs.Dim(1)), 1, mlx.DTypeInt32))
+ cos, sin := rotaryTables(positions, m.HeadDim, m.RopeTheta, h.DType())
+ states := make([]*mlx.Array, len(m.Layers))
+ for i, layer := range m.Layers {
+ h = layer.Forward(h, seed, b, nil, cos, sin, m.LayerTypes[i] == "sliding_attention", m.Config)
+ states[i] = h
+ }
+ return states
+}
+
+func TestDFlashTargetAuxHiddenLayers(t *testing.T) {
+ m, r := dflashTargetFixture(t)
+ target, ok := any(m).(interface{ SetAuxHiddenLayers([]int) })
+ if !ok {
+ t.Fatal("Ripple target is missing SetAuxHiddenLayers")
+ }
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ if err := m.LoadWeights(fixtureWeights(r, "bfloat16")); err != nil {
+ t.Fatal(err)
+ }
+ b := inputBatch([]float32{1, 4, 7}, 0)
+ states := dflashLayerOutputs(m, b)
+ wantHidden := rmsNorm(states[len(states)-1], m.Norm, m.RMSNormEps)
+ var observed [][]float32
+ for i, state := range append(slices.Clone(states), wantHidden) {
+ values := mlx.Contiguous(state.AsType(mlx.DTypeFloat32), false).Floats()
+ for j, previous := range observed {
+ if slices.Equal(values, previous) {
+ t.Fatalf("fixture states %d and %d are indistinguishable", i, j)
+ }
+ }
+ observed = append(observed, values)
+ }
+
+ baseline, aux := m.Forward(b, nil)
+ if baseline != aux {
+ t.Fatal("without taps, Forward must return h, h")
+ }
+ checkExactArray(t, "no-taps hidden", baseline, wantHidden)
+ baselineLogits := m.Unembed(baseline)
+ mlx.Eval(baselineLogits)
+
+ for _, tc := range []struct {
+ name string
+ ids []int
+ }{
+ {"first_last", []int{0, 2}},
+ {"middle", []int{1}},
+ {"disabled", nil},
+ } {
+ target.SetAuxHiddenLayers(tc.ids)
+ hidden, aux := m.Forward(b, nil)
+ checkExactArray(t, tc.name+" hidden", hidden, wantHidden)
+ checkExactArray(t, tc.name+" logits", m.Unembed(hidden), baselineLogits)
+ if len(tc.ids) == 0 {
+ if hidden != aux {
+ t.Fatal("clearing taps must restore h, h")
+ }
+ continue
+ }
+ if aux == nil || !slices.Equal(aux.Dims(), []int{1, 3, len(tc.ids) * int(m.HiddenSize)}) {
+ t.Fatalf("%s: incorrect auxiliary feature shape", tc.name)
+ }
+ for slot, id := range tc.ids {
+ start := slot * int(m.HiddenSize)
+ got := aux.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(start, start+int(m.HiddenSize)))
+ checkExactArray(t, fmt.Sprintf("%s tap %d", tc.name, id), got, states[id])
+ }
+ }
+ })
+ })
+}
+
+func inputBatch(ids []float32, offset int32) *batch.Batch {
+ tokens := make([]int32, len(ids))
+ for i, v := range ids {
+ tokens[i] = int32(v)
+ }
+ return &batch.Batch{InputIDs: mlx.FromValues(tokens, 1, len(tokens)), SeqOffsets: []int32{offset}, SeqQueryLens: []int32{int32(len(tokens))}}
+}
+
+func TestAttentionWindow(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ cfg := &Config{HiddenSize: 4, HeadDim: 4, NumAttentionHeads: 1, NumKeyValueHeads: 1, RMSNormEps: 1e-6, Scale: 0.5, SlidingWindow: 1024}
+ identity := mlx.FromValues([]float32{1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}, 4, 4)
+ zero := nn.NewLinear(mlx.Zeros(mlx.DTypeFloat32, 4, 4), nil)
+ a := &Attention{QProj: zero, KProj: zero, GProj: zero, VProj: nn.NewLinear(identity, nil), OProj: nn.NewLinear(identity, nil), QNorm: mlx.FromValues([]float32{1, 1, 1, 1}, 4), KNorm: mlx.FromValues([]float32{1, 1, 1, 1}, 4), ValueResLambda: mlx.FromValues([]float32{100}, 1)}
+ for _, local := range []bool{false, true} {
+ for _, length := range []int{1023, 1024, 1025} {
+ vals := make([]float32, length*4)
+ for i := range length {
+ for j := range 4 {
+ vals[i*4+j] = float32(i + 1)
+ }
+ }
+ x := mlx.FromValues(vals, 1, length, 4)
+ b := inputBatch(make([]float32, length), 0)
+ positions := mlx.Reshape(mlx.Arange(0, float64(length), 1, mlx.DTypeFloat32), 1, int32(length))
+ cos, sin := rotaryTables(positions, 4, 10000, mlx.DTypeFloat32)
+ expected := make([]float32, length*4)
+ for i := range length {
+ lo := 0
+ if local {
+ lo = max(0, i-1023)
+ }
+ for j := range 4 {
+ expected[i*4+j] = float32(lo+i+2) / 4
+ }
+ }
+ want := referenceTensor{Data: expected, Shape: []int{1, length, 4}}
+ for _, cached := range []bool{false, true} {
+ var c cache.Cache
+ if cached {
+ if local {
+ c = cache.NewRotatingKVCache(1024)
+ } else {
+ c = cache.NewKVCache()
+ }
+ }
+ got := a.Forward(x, x, b, c, cos, sin, local, cfg)
+ checkTensor(t, fmt.Sprintf("local=%v cached=%v length=%d", local, cached, length), got, want, 2e-3, 1e-5)
+ if c != nil {
+ c.Free()
+ }
+ }
+ }
+ }
+ })
+}
+
+func TestCacheContinuation(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ r := readReference(t)
+ cfg := fixtureConfig(t)
+ cfg["sliding_window"] = 1024
+ root := testRoot(t, cfg)
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ loaded, err := model.New(root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := loaded.(*Model)
+ if err := m.LoadWeights(fixtureWeights(r, "float32")); err != nil {
+ t.Fatal(err)
+ }
+ tokens := make([]float32, 1030)
+ for i := range tokens {
+ tokens[i] = float32((i*7 + 2) % 19)
+ }
+ all, _ := m.Forward(inputBatch(tokens, 0), nil)
+ all = mlx.Contiguous(all, false)
+ mlx.Eval(all)
+ want := all.Floats()
+ caches := m.NewCaches()
+ defer func() {
+ for _, c := range caches {
+ c.Free()
+ }
+ }()
+ run := func(start, end int) {
+ h, _ := m.Forward(inputBatch(tokens[start:end], int32(start)), caches)
+ checkTensor(t, fmt.Sprintf("continuation %d:%d", start, end), h, referenceTensor{Data: want[start*8 : end*8], Shape: []int{1, end - start, 8}}, 3e-5, 3e-5)
+ }
+ run(0, 1023)
+ run(1023, 1024)
+ snaps := make([]cache.Snapshot, len(caches))
+ for i, c := range caches {
+ snaps[i] = c.Snapshot(0)
+ defer snaps[i].Close()
+ }
+ run(1024, 1025)
+ run(1025, 1030)
+ for i, c := range caches {
+ if !c.Restore(snaps[i], 1024) {
+ t.Fatalf("restore layer %d", i)
+ }
+ }
+ run(1024, 1027)
+ run(1027, 1030)
+ // A fresh request must not inherit the prior embedding seed.
+ fresh, _ := m.Forward(inputBatch(tokens[:6], 0), nil)
+ checkTensor(t, "fresh request", fresh, referenceTensor{Data: want[:48], Shape: []int{1, 6, 8}}, 3e-5, 3e-5)
+ })
+}
+
+func TestConfig(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ cfg := fixtureConfig(t)
+ cfg["is_hybrid_swa"] = false
+ cfg["hybrid_layer_pattern"] = []int{0, 0, 0}
+ got, err := loadConfig(testRoot(t, cfg))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.HeadDim != 4 || got.HiddenSize != 8 || got.LayerTypes[1] != "full_attention" || got.SlidingWindow != 3 {
+ t.Fatalf("unexpected config: %+v", got)
+ }
+ for _, tc := range []struct {
+ field string
+ value any
+ }{
+ {"hidden_size", 0},
+ {"num_hidden_layers", 0},
+ {"intermediate_size", -1},
+ {"num_attention_heads", 3},
+ {"num_key_value_heads", 0},
+ {"head_dim", nil},
+ {"head_dim", 3},
+ {"vocab_size", 0},
+ {"max_position_embeddings", 0},
+ {"rms_norm_eps", -1},
+ {"rope_theta", 0},
+ {"sliding_window", 0},
+ {"hidden_act", "gelu"},
+ {"layer_types", []string{"full_attention"}},
+ {"layer_types", []string{"sliding_attention", "bad", "full_attention"}},
+ {"use_valres", false},
+ {"use_gated_attention", false},
+ {"gate_full_precision", false},
+ {"tie_word_embeddings", true},
+ {"attention_bias", true},
+ {"mlp_bias", true},
+ {"rope_scaling", map[string]any{"type": "linear", "factor": 2}},
+ {"ffn_clamp_threshold", -1},
+ } {
+ t.Run(tc.field, func(t *testing.T) {
+ bad := fixtureConfig(t)
+ bad[tc.field] = tc.value
+ _, err := loadConfig(testRoot(t, bad))
+ if err == nil || !strings.Contains(err.Error(), tc.field) {
+ t.Fatalf("want %s rejection, got %v", tc.field, err)
+ }
+ })
+ }
+}
diff --git a/mlxrunner/ripple_test.go b/mlxrunner/ripple_test.go
new file mode 100644
index 000000000..c36a72152
--- /dev/null
+++ b/mlxrunner/ripple_test.go
@@ -0,0 +1,681 @@
+package mlxrunner
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "math"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/manifest"
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlx/mlxtest"
+ "github.com/ollama/ollama/mlxrunner/batch"
+ "github.com/ollama/ollama/mlxrunner/cache"
+ "github.com/ollama/ollama/mlxrunner/model"
+ "github.com/ollama/ollama/mlxrunner/model/ripple"
+ "github.com/ollama/ollama/mlxrunner/sample"
+ modeltypes "github.com/ollama/ollama/types/model"
+)
+
+type rippleRunnerTensor struct {
+ data []float32
+ shape []int
+}
+
+func rippleRunnerFixture() (map[string]any, map[string]rippleRunnerTensor) {
+ cfg := map[string]any{
+ "architectures": []string{"RicursiveForCausalLM"},
+ "vocab_size": 19, "hidden_size": 8, "intermediate_size": 12,
+ "num_hidden_layers": 3, "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 4,
+ "layer_types": []string{"sliding_attention", "full_attention", "sliding_attention"},
+ "sliding_window": 3, "rms_norm_eps": 1e-6, "rope_theta": 10000.0,
+ "use_valres": true, "use_gated_attention": true, "gate_full_precision": true,
+ "ffn_clamp_threshold": 20.0, "embed_scale": 71.55417527999327,
+ "max_position_embeddings": 64, "tie_word_embeddings": false,
+ }
+ shapes := map[string][]int{
+ "model.embed_tokens.weight": {19, 8}, "lm_head.weight": {19, 8}, "model.norm.weight": {8},
+ }
+ for i := range 3 {
+ prefix := fmt.Sprintf("model.layers.%d.", i)
+ for name, shape := range map[string][]int{
+ "input_layernorm.weight": {8}, "post_attention_layernorm.weight": {8},
+ "pre_feedforward_layernorm.weight": {8}, "post_feedforward_layernorm.weight": {8},
+ "self_attn.q_proj.weight": {16, 8}, "self_attn.k_proj.weight": {8, 8},
+ "self_attn.v_proj.weight": {8, 8}, "self_attn.o_proj.weight": {8, 16},
+ "self_attn.g_proj.weight": {16, 8}, "self_attn.q_norm.weight": {4},
+ "self_attn.k_norm.weight": {4}, "self_attn.value_res_lambda": {1},
+ "mlp.gate_proj.weight": {12, 8}, "mlp.up_proj.weight": {12, 8}, "mlp.down_proj.weight": {8, 12},
+ } {
+ shapes[prefix+name] = shape
+ }
+ }
+ weights := make(map[string]rippleRunnerTensor)
+ for name, shape := range shapes {
+ size := 1
+ for _, dim := range shape {
+ size *= dim
+ }
+ values := make([]float32, size)
+ hash := sha256.Sum256([]byte(name))
+ phase := int(binary.BigEndian.Uint32(hash[:4]) % 101)
+ if strings.HasSuffix(name, "value_res_lambda") {
+ layer := strings.Split(name, ".")[2]
+ values[0] = map[string]float32{"0": -1.25, "1": 0.75, "2": 1.5}[layer]
+ } else {
+ for i := range values {
+ v := float32((i*37+phase)%101-50) / 50
+ switch {
+ case strings.Contains(name, "norm.weight"):
+ scaled := v * 0.375
+ v = 1 + scaled
+ case strings.Contains(name, "embed_tokens"):
+ v *= 0.045
+ case strings.Contains(name, ".mlp.gate_proj") || strings.Contains(name, ".mlp.up_proj"):
+ v *= 18
+ default:
+ v *= 0.3
+ }
+ values[i] = v
+ }
+ }
+ weights[name] = rippleRunnerTensor{data: values, shape: shape}
+ }
+ return cfg, weights
+}
+
+func writeRippleRunnerModel(t *testing.T) string {
+ t.Helper()
+ mlxtest.SkipIfUnavailable(t)
+ dir := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", dir)
+ fixtureConfig, fixtureWeights := rippleRunnerFixture()
+ config, err := json.Marshal(fixtureConfig)
+ if err != nil {
+ t.Fatal(err)
+ }
+ vocab := make(map[string]int)
+ for i := range fixtureConfig["vocab_size"].(int) {
+ vocab[string(rune('a'+i))] = i
+ }
+ tokenizer, err := json.Marshal(map[string]any{
+ "model": map[string]any{"type": "BPE", "vocab": vocab, "merges": []string{}},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ name := "ripple-runner-test"
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ weights := make(map[string]*mlx.Array)
+ for name, tensor := range fixtureWeights {
+ weights[name] = mlx.FromValues(tensor.data, tensor.shape...)
+ }
+ path := filepath.Join(dir, "model.safetensors")
+ if err := mlx.SaveSafetensors(path, weights); err != nil {
+ t.Fatal(err)
+ }
+ f, err := os.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ tensors, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tensors.Name = "model.safetensors"
+ layers := []manifest.Layer{tensors}
+ for name, data := range map[string][]byte{"config.json": config, "tokenizer.json": tokenizer} {
+ layer, err := manifest.NewLayer(bytes.NewReader(data), manifest.MediaTypeImageJSON)
+ if err != nil {
+ t.Fatal(err)
+ }
+ layer.Name = name
+ layers = append(layers, layer)
+ }
+ meta, err := manifest.NewLayer(strings.NewReader(`{"model_format":"safetensors","model_family":"ripple"}`), manifest.MediaTypeImageConfig)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), meta, layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil {
+ t.Fatal(err)
+ }
+ })
+ })
+ return name
+}
+
+func loadRippleRunner(t *testing.T, name string) *Runner {
+ t.Helper()
+ r := &Runner{mlxThread: mlxtest.Worker(t)}
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ if err := r.Load(name); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := r.Model.(*ripple.Model); !ok {
+ t.Fatalf("loaded %T, want *ripple.Model", r.Model)
+ }
+ })
+ t.Cleanup(func() {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ r.cache.close()
+ r.Close()
+ })
+ })
+ return r
+}
+
+func rippleRunnerMemory(t *testing.T) (active int) {
+ t.Helper()
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.DefaultStream().Synchronize()
+ mlx.ClearCache()
+ active = mlx.ActiveMemory()
+ })
+ return active
+}
+
+func rippleRunnerRequest(t *testing.T, r *Runner, prompt string, count int) Request {
+ t.Helper()
+ request := Request{
+ CompletionRequest: CompletionRequest{Prompt: prompt, Options: api.Options{NumPredict: count}},
+ Pipeline: r.TextGenerationPipeline,
+ SamplerOpts: sample.Options{
+ Temperature: 0, RepeatLastN: 8, RepeatPenalty: 1.1,
+ Seed: 1, UseSeed: true, Logprobs: true, TopLogprobs: 3,
+ },
+ Responses: make(chan CompletionResponse, count+1),
+ }
+ if err := r.Prepare(&request); err != nil {
+ t.Fatal(err)
+ }
+ if len(request.Tokens) != len(prompt) {
+ t.Fatalf("tokenized %q into %d tokens, want %d", prompt, len(request.Tokens), len(prompt))
+ }
+ return request
+}
+
+func runRippleRunnerRequest(t *testing.T, r *Runner, prompt string) (string, []llm.Logprob, int) {
+ t.Helper()
+ request := rippleRunnerRequest(t, r, prompt, 4)
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+ defer cancel()
+ request.Ctx = ctx
+ if err := r.runRequest(request); err != nil {
+ t.Fatal(err)
+ }
+ close(request.Responses)
+ var text strings.Builder
+ var logprobs []llm.Logprob
+ var final *CompletionResponse
+ for response := range request.Responses {
+ if response.Error != nil {
+ t.Fatal(response.Error)
+ }
+ if response.Done {
+ if final != nil {
+ t.Fatal("duplicate final response")
+ }
+ final = &response
+ }
+ text.WriteString(response.Content)
+ logprobs = append(logprobs, response.Logprobs...)
+ }
+ if final == nil || final.EvalCount != 4 || final.DoneReason != 1 || final.PromptEvalCount != len(request.Tokens) || final.PromptEvalCachedCount == nil {
+ t.Fatalf("invalid final response: %+v", final)
+ }
+ if text.Len() != 4 || len(logprobs) != 4 {
+ t.Fatalf("empty or incomplete model output: text=%q logprobs=%d", text.String(), len(logprobs))
+ }
+ for _, lp := range logprobs {
+ if math.IsNaN(lp.Logprob) || math.IsInf(lp.Logprob, 0) || lp.Logprob >= 0 || len(lp.TopLogprobs) != 3 {
+ t.Fatalf("invalid model log probability: %+v", lp)
+ }
+ if lp.TopLogprobs[0].Logprob == lp.TopLogprobs[2].Logprob {
+ t.Fatalf("model produced uniform logits: %+v", lp)
+ }
+ }
+ return text.String(), logprobs, *final.PromptEvalCachedCount
+}
+
+func TestRippleRunnerLoadClose(t *testing.T) {
+ name := writeRippleRunnerModel(t)
+ warm := loadRippleRunner(t, name)
+ mlxtest.Run(t, func(t *mlxtest.T) { warm.Close() })
+ baseline := rippleRunnerMemory(t)
+ for range 3 {
+ r := loadRippleRunner(t, name)
+ if loaded := rippleRunnerMemory(t); loaded <= baseline {
+ t.Fatalf("load allocated no weights: loaded=%d baseline=%d", loaded, baseline)
+ }
+ mlxtest.Run(t, func(t *mlxtest.T) { r.Close() })
+ if got := rippleRunnerMemory(t); got != baseline {
+ t.Fatalf("load/close retained %d bytes above warmed baseline %d", got-baseline, baseline)
+ }
+ }
+}
+
+func TestRippleRunnerRequests(t *testing.T) {
+ name := writeRippleRunnerModel(t)
+ cold, reused := loadRippleRunner(t, name), loadRippleRunner(t, name)
+ const prompt = "ihgfedcba"
+ wantText, wantLogprobs, cached := runRippleRunnerRequest(t, cold, prompt)
+ if cached != 0 {
+ t.Fatalf("fresh runner reused %d tokens", cached)
+ }
+ runRippleRunnerRequest(t, reused, "abcdefghi")
+ gotText, gotLogprobs, cached := runRippleRunnerRequest(t, reused, prompt)
+ // These cold forwards have identical chunk schedules; cache-hit forwards do not.
+ if cached != 0 || gotText != wantText || !reflect.DeepEqual(gotLogprobs, wantLogprobs) {
+ t.Fatalf("independent request inherited state: cached=%d text=%q want=%q", cached, gotText, wantText)
+ }
+ wantText, wantLogprobs, cached = runRippleRunnerRequest(t, reused, prompt)
+ if cached <= 0 || cached >= len(prompt) {
+ t.Fatalf("repeat reused %d tokens, want a nonempty prefix with a decode seed", cached)
+ }
+ baseline := rippleRunnerMemory(t)
+ // Settled allocator readings can vary by a few small buffers between requests.
+ const memorySlack = 256
+ for range 16 {
+ gotText, gotLogprobs, gotCached := runRippleRunnerRequest(t, reused, prompt)
+ if gotCached != cached || gotText != wantText || !reflect.DeepEqual(gotLogprobs, wantLogprobs) {
+ t.Fatal("identical cache-hit schedule changed the response")
+ }
+ if got := rippleRunnerMemory(t); got > baseline+memorySlack {
+ t.Fatalf("repeated request retained %d bytes above warmed baseline %d", got-baseline, baseline)
+ }
+ }
+
+ request := rippleRunnerRequest(t, reused, prompt, 32)
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+ defer cancel()
+ request.Ctx = ctx
+ request.Responses = make(chan CompletionResponse)
+ responses := make(chan []CompletionResponse, 1)
+ go func() {
+ var received []CompletionResponse
+ for response := range request.Responses {
+ received = append(received, response)
+ cancel()
+ }
+ responses <- received
+ }()
+ err := reused.runRequest(request)
+ close(request.Responses)
+ received := <-responses
+ if !errors.Is(err, context.Canceled) || len(received) == 0 || received[0].Content == "" {
+ t.Fatalf("cancel after real output: err=%v responses=%+v", err, received)
+ }
+ for _, response := range received {
+ if response.Done {
+ t.Fatal("cancelled generation emitted a successful final response")
+ }
+ }
+ // Retry through the same worker, sampler, and prefix-cache owner.
+ runRippleRunnerRequest(t, reused, prompt)
+ runRippleRunnerRequest(t, reused, prompt)
+ baseline = rippleRunnerMemory(t)
+ runRippleRunnerRequest(t, reused, prompt)
+ if got := rippleRunnerMemory(t); got > baseline+memorySlack {
+ t.Fatalf("post-cancellation request retained %d bytes", got-baseline)
+ }
+}
+
+func writeRippleDFlash2RunnerModel(t *testing.T) string {
+ t.Helper()
+ name := writeRippleRunnerModel(t)
+ m, err := manifest.ParseNamedManifest(modeltypes.ParseName(name))
+ if err != nil {
+ t.Fatal(err)
+ }
+ const draftConfig = `{
+ "architectures": ["DFlash2DraftModel"],
+ "hidden_size": 8, "num_hidden_layers": 2, "intermediate_size": 16,
+ "num_attention_heads": 2, "num_key_value_heads": 1, "head_dim": 4,
+ "rms_norm_eps": 0.000001, "rope_theta": 10000,
+ "vocab_size": 19, "num_target_layers": 3, "block_size": 8,
+ "layer_types": ["sliding_attention", "full_attention"],
+ "sliding_window": 4, "no_rope_layers": [1], "is_causal": false,
+ "dflash_config": {
+ "mask_token_id": 18, "target_layer_ids": [0, 2],
+ "conv_group_size": 2, "conv_kernel_size": 4,
+ "selector_rank": 4, "selector_top_k": 4
+ }
+ }`
+ config, err := manifest.NewLayer(strings.NewReader(draftConfig), manifest.MediaTypeImageJSON)
+ if err != nil {
+ t.Fatal(err)
+ }
+ config.Name = "draft/config.json"
+ layers := append(m.Layers, config)
+ path := filepath.Join(t.TempDir(), "draft.safetensors")
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ mlx.Scoped(func() {
+ weights := make(map[string]*mlx.Array)
+ add := func(name string, dims ...int) {
+ n, phase := 1, 0
+ for _, dim := range dims {
+ n *= dim
+ }
+ for _, c := range name {
+ phase += int(c)
+ }
+ values := make([]float32, n)
+ for i := range values {
+ values[i] = float32((i*17+phase)%31-15) / 128
+ if len(dims) == 1 {
+ values[i] = 1
+ }
+ }
+ weights["draft."+name] = mlx.FromValues(values, dims...)
+ }
+ add("fc.weight", 8, 2*8)
+ add("hidden_norm.weight", 8)
+ add("norm.weight", 8)
+ for i := range 2 {
+ prefix := fmt.Sprintf("layers.%d.", i)
+ add(prefix+"input_layernorm.weight", 8)
+ add(prefix+"post_attention_layernorm.weight", 8)
+ add(prefix+"self_attn.q_norm.weight", 4)
+ add(prefix+"self_attn.k_norm.weight", 4)
+ add(prefix+"self_attn.q_proj.weight", 8, 8)
+ add(prefix+"self_attn.k_proj.weight", 4, 8)
+ add(prefix+"self_attn.v_proj.weight", 4, 8)
+ add(prefix+"self_attn.o_proj.weight", 8, 8)
+ add(prefix+"mlp.gate_proj.weight", 16, 8)
+ add(prefix+"mlp.up_proj.weight", 16, 8)
+ add(prefix+"mlp.down_proj.weight", 8, 16)
+ for _, conv := range []string{"attention_conv", "mlp_conv"} {
+ add(prefix+conv+".base_kernel", 2, 4, 8)
+ add(prefix+conv+".kernel_projection.weight", 2*4*8/2, 8)
+ }
+ }
+ add("candidate_selector.predecessor_codebook", 19, 4)
+ add("candidate_selector.successor_codebook", 19, 4)
+ add("candidate_selector.hidden_projection.weight", 4, 8)
+ if err := mlx.SaveSafetensors(path, weights); err != nil {
+ t.Fatal(err)
+ }
+ })
+ })
+ f, err := os.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ tensors, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tensors.Name = "draft/model.safetensors"
+ layers = append(layers, tensors)
+ metadata, err := json.Marshal(modeltypes.ConfigV2{
+ ModelFormat: "safetensors", ModelFamily: "ripple",
+ Draft: &modeltypes.Draft{
+ ModelFormat: "safetensors", Architecture: "DFlash2DraftModel",
+ TensorPrefix: "draft.", Config: "draft/config.json",
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ meta, err := manifest.NewLayer(bytes.NewReader(metadata), manifest.MediaTypeImageConfig)
+ if err != nil {
+ t.Fatal(err)
+ }
+ name = "ripple-dflash2-runner-test"
+ if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), meta, layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil {
+ t.Fatal(err)
+ }
+ return name
+}
+
+type rippleDraftForward struct {
+ offset, contextRows, featureWidth, blockRows int
+}
+
+// Observe shapes and calls only; every native operand and selector stays real.
+type rippleDFlash2Observer struct {
+ model.BlockSelector
+ forwards []rippleDraftForward
+ selections int
+}
+
+func (d *rippleDFlash2Observer) Forward(b *batch.Batch, targets, drafts []cache.Cache) (*mlx.Array, *mlx.Array) {
+ call := rippleDraftForward{offset: int(b.SeqOffsets[0])}
+ if b.Hidden != nil {
+ call.contextRows, call.featureWidth = b.Hidden.Dim(1), b.Hidden.Dim(2)
+ }
+ if b.InputIDs != nil {
+ call.blockRows = b.InputIDs.Dim(1)
+ }
+ d.forwards = append(d.forwards, call)
+ return d.BlockSelector.Forward(b, targets, drafts)
+}
+
+func (d *rippleDFlash2Observer) Select(hidden, logits, anchor *mlx.Array, choose func(ids, scores *mlx.Array) (*mlx.Array, *mlx.Array)) (*mlx.Array, *mlx.Array, *mlx.Array) {
+ d.selections++
+ return d.BlockSelector.Select(hidden, logits, anchor, choose)
+}
+
+func loadRippleDFlash2Runner(t *testing.T, name string) (*Runner, *rippleDFlash2Observer) {
+ t.Helper()
+ r := loadRippleRunner(t, name)
+ if r.spec == nil {
+ t.Fatal("combined manifest did not attach a draft")
+ }
+ draft, ok := r.spec.draft.(model.BlockSelector)
+ if !ok {
+ t.Fatalf("loaded draft %T, want DFlash2 BlockSelector", r.spec.draft)
+ }
+ if block, mask := draft.BlockParams(); block != 8 || mask != 18 {
+ t.Fatalf("draft block=%d mask=%d, want 8 and 18", block, mask)
+ }
+ if len(r.spec.targets) != 3 || len(r.spec.draftKV) != 2 || len(r.cache.caches) != 5 {
+ t.Fatalf("cache partitions: target=%d draft=%d combined=%d", len(r.spec.targets), len(r.spec.draftKV), len(r.cache.caches))
+ }
+ if _, ok := r.spec.draftKV[0].(*cache.RotatingKVCache); !ok {
+ t.Fatalf("local draft cache is %T", r.spec.draftKV[0])
+ }
+ if _, ok := r.spec.draftKV[1].(*cache.RotatingKVCache); !ok {
+ t.Fatalf("full draft cache is %T", r.spec.draftKV[1])
+ }
+ observer := &rippleDFlash2Observer{BlockSelector: draft}
+ r.spec.draft = observer
+ return r, observer
+}
+
+func checkRippleDFlash2Caches(t *testing.T, r *Runner, prompt string) {
+ t.Helper()
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ offset := r.spec.targets[0].Offset()
+ if offset < len(prompt) {
+ t.Fatalf("target stopped before prompt completion: offset=%d prompt=%d", offset, len(prompt))
+ }
+ for i, c := range r.cache.caches {
+ if c.Offset() != offset {
+ t.Fatalf("cache %d offset=%d, want settled target offset=%d", i, c.Offset(), offset)
+ }
+ }
+ })
+}
+
+type rippleDFlash2Response struct {
+ text string
+ cached int
+ forwards []rippleDraftForward
+}
+
+func runRippleDFlash2Request(t *testing.T, r *Runner, prompt string, count int) rippleDFlash2Response {
+ t.Helper()
+ d := r.spec.draft.(*rippleDFlash2Observer)
+ d.forwards, d.selections = nil, 0
+ request := rippleRunnerRequest(t, r, prompt, count)
+ // The shared helper requests logprobs, which would silently park drafting.
+ request.SamplerOpts.Logprobs, request.SamplerOpts.TopLogprobs = false, 0
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+ defer cancel()
+ request.Ctx = ctx
+ if err := r.runRequest(request); err != nil {
+ t.Fatal(err)
+ }
+ close(request.Responses)
+ var text strings.Builder
+ var final *CompletionResponse
+ for response := range request.Responses {
+ if response.Error != nil || len(response.Logprobs) != 0 {
+ t.Fatalf("unexpected active response: %+v", response)
+ }
+ text.WriteString(response.Content)
+ if response.Done {
+ if final != nil {
+ t.Fatal("duplicate final response")
+ }
+ final = &response
+ }
+ }
+ if final == nil || final.EvalCount != count || final.DoneReason != 1 || final.PromptEvalCount != len(request.Tokens) || final.PromptEvalCachedCount == nil || text.Len() != count {
+ t.Fatalf("invalid completion: text=%q final=%+v", text.String(), final)
+ }
+ proposals := 0
+ for _, call := range d.forwards {
+ if call.contextRows > 0 && call.featureWidth != 16 {
+ t.Fatalf("draft did not receive both target taps: %+v", call)
+ }
+ if call.blockRows > 0 {
+ proposals++
+ if call.blockRows < 2 || call.blockRows > min(8, count) {
+ t.Fatalf("proposal exceeded block/budget: %+v budget=%d", call, count)
+ }
+ }
+ }
+ if proposals != d.selections || (count > 1 && proposals == 0) || (count == 1 && proposals != 0) {
+ t.Fatalf("active budget=%d: draft forwards=%d selector calls=%d", count, proposals, d.selections)
+ }
+ checkRippleDFlash2Caches(t, r, prompt)
+ return rippleDFlash2Response{text: text.String(), cached: *final.PromptEvalCachedCount, forwards: d.forwards}
+}
+
+func TestRippleRunnerDFlash2(t *testing.T) {
+ name := writeRippleDFlash2RunnerModel(t)
+ const prompt = "abcdefghi"
+
+ // New wiring: normal combined load, target prefill with trailing draft
+ // caches, target taps, and a real convolution/selector proposal.
+ t.Run("combined_manifest_prefill_proposal", func(t *testing.T) {
+ r, d := loadRippleDFlash2Runner(t, name)
+ got := runRippleDFlash2Request(t, r, prompt, 9)
+ if got.cached != 0 || len(d.forwards) < 2 {
+ t.Fatalf("cold request: cached=%d forwards=%+v", got.cached, d.forwards)
+ }
+ if want := (rippleDraftForward{contextRows: len(prompt) - 1, featureWidth: 16}); d.forwards[0] != want {
+ t.Fatalf("prefill handoff=%+v, want %+v", d.forwards[0], want)
+ }
+ if want := (rippleDraftForward{offset: len(prompt) - 1, blockRows: 8}); d.forwards[1] != want {
+ t.Fatalf("first proposal=%+v, want %+v", d.forwards[1], want)
+ }
+ })
+
+ // Existing runner lifecycle, characterized with the new target/draft pair.
+ // Compare identities only after matching cache-hit and proposal schedules.
+ t.Run("prefix_branch_reuse", func(t *testing.T) {
+ r, _ := loadRippleDFlash2Runner(t, name)
+ runRippleDFlash2Request(t, r, prompt, 9)
+ runRippleDFlash2Request(t, r, prompt, 9) // warm the repeat schedule
+ want := runRippleDFlash2Request(t, r, prompt, 9)
+ if want.cached <= 0 || want.cached >= len(prompt) {
+ t.Fatalf("repeat cached=%d, want prefix plus an unforwarded seed", want.cached)
+ }
+ if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
+ t.Fatalf("matched repeat changed: got=%+v want=%+v", got, want)
+ }
+ const branch = "abcdejklm"
+ if got := runRippleDFlash2Request(t, r, branch, 9); got.cached != 5 {
+ t.Fatalf("branch cached=%d, want shared five-token snapshot", got.cached)
+ }
+ if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
+ t.Fatalf("returning from branch changed matched schedule: got=%+v want=%+v", got, want)
+ }
+ })
+
+ t.Run("parked_to_enabled", func(t *testing.T) {
+ r, d := loadRippleDFlash2Runner(t, name)
+ runRippleRunnerRequest(t, r, prompt)
+ if len(d.forwards) == 0 || d.selections != 0 {
+ t.Fatalf("parked maintenance: forwards=%+v selections=%d", d.forwards, d.selections)
+ }
+ for _, call := range d.forwards {
+ if call.blockRows != 0 || call.contextRows == 0 || call.featureWidth != 16 {
+ t.Fatalf("logprobs must maintain context without proposals: %+v", call)
+ }
+ }
+ checkRippleDFlash2Caches(t, r, prompt)
+ if got := runRippleDFlash2Request(t, r, prompt, 9); got.cached <= 0 {
+ t.Fatal("enabled request did not reuse parked context")
+ }
+ runRippleDFlash2Request(t, r, prompt, 9)
+ want := runRippleDFlash2Request(t, r, prompt, 9)
+ if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
+ t.Fatalf("enabled repeats changed: got=%+v want=%+v", got, want)
+ }
+ })
+
+ t.Run("budget_tail", func(t *testing.T) {
+ r, d := loadRippleDFlash2Runner(t, name)
+ runRippleDFlash2Request(t, r, prompt, 3)
+ if len(d.forwards) < 2 || d.forwards[1].blockRows != 3 {
+ t.Fatalf("shorter-than-convolution proposal: %+v", d.forwards)
+ }
+ runRippleDFlash2Request(t, r, prompt, 1)
+ })
+
+ t.Run("cancellation_recovery", func(t *testing.T) {
+ r, d := loadRippleDFlash2Runner(t, name)
+ request := rippleRunnerRequest(t, r, prompt, 16)
+ request.SamplerOpts.Logprobs, request.SamplerOpts.TopLogprobs = false, 0
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+ defer cancel()
+ request.Ctx = ctx
+ request.Responses = make(chan CompletionResponse)
+ responses := make(chan []CompletionResponse, 1)
+ go func() {
+ var received []CompletionResponse
+ for response := range request.Responses {
+ received = append(received, response)
+ cancel()
+ }
+ responses <- received
+ }()
+ err := r.runRequest(request)
+ close(request.Responses)
+ received := <-responses
+ if !errors.Is(err, context.Canceled) || len(received) == 0 || received[0].Content == "" || d.selections == 0 {
+ t.Fatalf("cancel after real speculative output: err=%v responses=%+v selections=%d", err, received, d.selections)
+ }
+ for _, response := range received {
+ if response.Done || response.Error != nil {
+ t.Fatalf("cancelled generation emitted terminal response: %+v", response)
+ }
+ }
+ checkRippleDFlash2Caches(t, r, prompt)
+ runRippleDFlash2Request(t, r, prompt, 9)
+ runRippleDFlash2Request(t, r, prompt, 9)
+ want := runRippleDFlash2Request(t, r, prompt, 9)
+ if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
+ t.Fatalf("recovery repeats changed: got=%+v want=%+v", got, want)
+ }
+ })
+}
diff --git a/mlxrunner/testdata/generate_qwen35_yarn.py b/mlxrunner/testdata/generate_qwen35_yarn.py
deleted file mode 100644
index ba21d301e..000000000
--- a/mlxrunner/testdata/generate_qwen35_yarn.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# /// script
-# requires-python = ">=3.12,<3.13"
-# dependencies = ["torch==2.6.0", "transformers==5.12.1"]
-# ///
-
-import argparse
-import json
-from pathlib import Path
-
-import torch
-import transformers
-from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
-from transformers.models.qwen3_5.modeling_qwen3_5 import (
- Qwen3_5Attention,
- Qwen3_5TextRotaryEmbedding,
- apply_rotary_pos_emb,
-)
-
-
-def generate():
- torch.set_num_threads(1)
- defaults = {
- "rope_type": "default",
- "rope_theta": 10000000.0,
- "partial_rotary_factor": 0.25,
- "mrope_section": [11, 11, 10],
- }
- yarn = {
- **defaults,
- "rope_type": "yarn",
- "factor": 4.0,
- "original_max_position_embeddings": 262144,
- }
- cases = [
- ("default", 262144, defaults, "float32"),
- ("no_metadata", 262144, None, "float32"),
- ("yarn4", 1048576, yarn, "float32"),
- ("unextended_limit", 262144, yarn, "float32"),
- (
- "implicit_original",
- 262144,
- {k: v for k, v in yarn.items() if k != "original_max_position_embeddings"},
- "float32",
- ),
- ("beta", 1048576, {**yarn, "beta_fast": 16.0, "beta_slow": 2.0}, "float32"),
- ("fractional_range", 1048576, {**yarn, "truncate": False}, "float32"),
- ("mscale", 1048576, {**yarn, "mscale": 2.0, "mscale_all_dim": 1.0}, "float32"),
- ("attention_override", 1048576, {**yarn, "attention_factor": 1.2}, "float32"),
- ("zero_attention", 1048576, {**yarn, "attention_factor": 0.0}, "float32"),
- ("factor_one", 262144, {**yarn, "factor": 1.0}, "float32"),
- (
- "factor_one_attention",
- 262144,
- {**yarn, "factor": 1.0, "attention_factor": 1.2},
- "float32",
- ),
- ("theta", 1048576, {**yarn, "rope_theta": 500000.0}, "float32"),
- (
- "half_rotary",
- 1048576,
- {**yarn, "partial_rotary_factor": 0.5, "mrope_section": [22, 21, 21]},
- "float32",
- ),
- ("sections", 1048576, {**yarn, "mrope_section": [24, 4, 4]}, "float32"),
- ("default_bfloat16", 262144, defaults, "bfloat16"),
- ("yarn_bfloat16", 1048576, yarn, "bfloat16"),
- ]
- sample_tokens = [0, 1, 7, 15]
- sample_dims = [0, 7, 15, 16, 23, 31, 32, 39, 47, 48, 55, 63, 64, 95, 127, 128, 255]
- indices = torch.arange(16)
- text_positions = indices.reshape(1, -1)
- media_positions = torch.stack(
- (32768 + 2 * indices, 7 + indices % 4, 11 + indices // 4)
- ).unsqueeze(1)
- media_positions[:, 0, -1] = media_positions[:, :, :-1].max() + 1
- mask = torch.full((1, 1, 16, 16), float("-inf")).triu(1)
- results = []
- for name, limit, params, dtype_name in cases:
- raw_config = {
- "model_type": "qwen3_5_text",
- "hidden_size": 256,
- "num_hidden_layers": 1,
- "num_attention_heads": 1,
- "num_key_value_heads": 1,
- "head_dim": 256,
- "intermediate_size": 256,
- "rms_norm_eps": 1e-6,
- "linear_num_key_heads": 1,
- "linear_num_value_heads": 1,
- "linear_key_head_dim": 128,
- "linear_value_head_dim": 128,
- "layer_types": ["full_attention"],
- "max_position_embeddings": limit,
- "rope_theta": 10000000.0,
- "partial_rotary_factor": (params or defaults)["partial_rotary_factor"],
- "rope_parameters": params,
- }
- cfg = Qwen3_5TextConfig(
- **{
- k: v
- for k, v in raw_config.items()
- if k not in ("rope_theta", "rope_parameters")
- },
- rope_parameters=dict(params or defaults),
- )
- cfg._attn_implementation = "eager"
- dtype = getattr(torch, dtype_name)
- attention = Qwen3_5Attention(cfg, layer_idx=0).eval().to(dtype)
- rope = Qwen3_5TextRotaryEmbedding(cfg)
- assert rope.inv_freq.numel() * 2 == int(
- cfg.head_dim * (params or defaults)["partial_rotary_factor"]
- )
- with torch.no_grad():
- attention.q_proj.weight.zero_()
- attention.q_proj.weight[:256].copy_(torch.eye(256))
- for linear in (attention.k_proj, attention.v_proj, attention.o_proj):
- linear.weight.copy_(torch.eye(256))
- attention.q_norm.weight.fill_(-0.5)
- attention.k_norm.weight.fill_(-0.5)
- x = (
- (((torch.arange(16 * 256) % 37) - 18).float() / 16)
- .reshape(1, 16, 256)
- .to(dtype)
- )
- raw_keys = attention.k_norm(
- attention.k_proj(x).reshape(1, 16, 1, 256)
- ).transpose(1, 2)
- entry = {
- "name": name,
- "dtype": dtype_name,
- "config": raw_config,
- "inv_freq": rope.inv_freq.tolist(),
- "attention_factor": rope.attention_scaling,
- }
- modes = (
- ("text", text_positions),
- ("mrope", media_positions),
- ("long_text", text_positions + 32768),
- )
- if name == "sections":
- entry["modes"] = ["mrope"]
- modes = (("mrope", media_positions),)
- for mode, positions in modes:
- cos, sin = rope(x, positions)
- output, _ = attention(
- x, position_embeddings=(cos, sin), attention_mask=mask
- )
- entry[mode + "_output"] = [
- output[0, token, dim].float().item()
- for token in sample_tokens
- for dim in sample_dims
- ]
- if mode != "long_text":
- _, keys = apply_rotary_pos_emb(raw_keys, raw_keys, cos, sin)
- entry[mode + "_keys"] = [
- keys[0, 0, token, dim].float().item()
- for token in sample_tokens
- for dim in sample_dims
- ]
- results.append(entry)
- return {
- "source": f"transformers {transformers.__version__}; torch {torch.__version__}; CPU; Qwen3_5TextConfig/Qwen3_5TextRotaryEmbedding/Qwen3_5Attention",
- "sample_tokens": sample_tokens,
- "sample_dims": sample_dims,
- "mrope_positions": media_positions.flatten().tolist(),
- "cases": results,
- }
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("output", type=Path)
- args = parser.parse_args()
- fixture = generate()
- text = json.dumps(fixture, indent=2)
- for case in fixture["cases"]:
- for key in (
- "inv_freq",
- "text_output",
- "long_text_output",
- "text_keys",
- "mrope_output",
- "mrope_keys",
- ):
- if key not in case:
- continue
- multiline = json.dumps(case[key], indent=2)
- body = "\n".join(" " + line for line in multiline.splitlines()[1:-1])
- text = text.replace("[\n" + body + "\n ]", json.dumps(case[key]))
- args.output.parent.mkdir(parents=True, exist_ok=True)
- args.output.write_text(text + "\n")
- print(
- f"Generated {len(fixture['cases'])} reference cases: {args.output} ({args.output.stat().st_size} bytes)"
- )
diff --git a/mlxrunner/verify_test.go b/mlxrunner/verify_test.go
index 35eebd0a6..daa3dd690 100644
--- a/mlxrunner/verify_test.go
+++ b/mlxrunner/verify_test.go
@@ -6,6 +6,12 @@ import (
"github.com/ollama/ollama/mlx"
)
+func TestSupportsRippleArchitecture(t *testing.T) {
+ if !SupportsArchitecture("RicursiveForCausalLM") {
+ t.Fatal("missing model architecture")
+ }
+}
+
func TestCheckRuntimeRequiresLoadedLibrary(t *testing.T) {
_, loadErr := mlx.LoadedLibraryPath()
err := CheckRuntime()
diff --git a/model/parsers/parsers.go b/model/parsers/parsers.go
index 69c387f89..fa82ac9d3 100644
--- a/model/parsers/parsers.go
+++ b/model/parsers/parsers.go
@@ -53,6 +53,8 @@ func ParserForName(name string) Parser {
var p Parser
switch name {
+ case "ripple":
+ p = &RippleParser{}
case "qwen3":
p = &Qwen3Parser{hasThinkingSupport: false, defaultThinking: false}
case "qwen3-thinking":
diff --git a/model/parsers/ripple.go b/model/parsers/ripple.go
new file mode 100644
index 000000000..2ad3fbe84
--- /dev/null
+++ b/model/parsers/ripple.go
@@ -0,0 +1,184 @@
+package parsers
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+
+ "github.com/ollama/ollama/api"
+)
+
+type rippleState int
+
+const (
+ rippleStart rippleState = iota
+ rippleThinkLead
+ rippleThinking
+ rippleThinkEnd
+ rippleContent
+ rippleTool
+)
+
+const (
+ rippleThinkOpen = ""
+ rippleThinkClose = ""
+ rippleToolOpen = ""
+ rippleToolClose = ""
+)
+
+type RippleParser struct {
+ state rippleState
+ buffer string
+ callIndex int
+ toolScan int
+ toolString bool
+ toolEscape bool
+}
+
+func (p *RippleParser) Init(tools []api.Tool, _ *api.Message, _ *api.ThinkValue) []api.Tool {
+ *p = RippleParser{}
+ return tools
+}
+
+func (p *RippleParser) HasToolSupport() bool { return true }
+func (p *RippleParser) HasThinkingSupport() bool { return true }
+
+func (p *RippleParser) ThinkingClose() []string {
+ if p.state == rippleThinkLead || p.state == rippleThinking {
+ return []string{rippleThinkClose}
+ }
+ return nil
+}
+
+func (p *RippleParser) PreservedTokens() []string {
+ return []string{rippleThinkOpen, rippleThinkClose, rippleToolOpen, rippleToolClose}
+}
+
+func (p *RippleParser) Add(s string, done bool) (content string, thinking string, calls []api.ToolCall, err error) {
+ p.buffer += s
+parse:
+ for p.buffer != "" {
+ switch p.state {
+ case rippleStart:
+ if strings.HasPrefix(p.buffer, rippleThinkOpen) {
+ p.buffer = p.buffer[len(rippleThinkOpen):]
+ p.state = rippleThinkLead
+ } else if !done && strings.HasPrefix(rippleThinkOpen, p.buffer) {
+ return content, thinking, calls, nil
+ } else {
+ p.state = rippleContent
+ }
+ case rippleThinkLead:
+ p.buffer = strings.TrimPrefix(p.buffer, "\n")
+ p.state = rippleThinking
+ case rippleThinking:
+ if i := strings.Index(p.buffer, rippleThinkClose); i >= 0 {
+ thinking += strings.TrimSuffix(p.buffer[:i], "\n")
+ p.buffer = p.buffer[i+len(rippleThinkClose):]
+ p.state = rippleThinkEnd
+ } else {
+ n := len(p.buffer)
+ if !done {
+ // The newline before a closing tag belongs to the template.
+ n -= max(overlap(p.buffer, rippleThinkClose), overlap(p.buffer, "\n"+rippleThinkClose))
+ }
+ thinking += p.buffer[:n]
+ p.buffer = p.buffer[n:]
+ return content, thinking, calls, nil
+ }
+ case rippleThinkEnd:
+ p.buffer = strings.TrimPrefix(p.buffer, "\n")
+ p.state = rippleContent
+ case rippleContent:
+ if i := strings.Index(p.buffer, rippleToolOpen); i >= 0 {
+ if p.callIndex == 0 {
+ content += strings.TrimSuffix(p.buffer[:i], "\n")
+ }
+ p.buffer = p.buffer[i+len(rippleToolOpen):]
+ p.state = rippleTool
+ p.toolScan, p.toolString, p.toolEscape = 0, false, false
+ } else {
+ n := len(p.buffer)
+ if !done {
+ n -= max(overlap(p.buffer, rippleToolOpen), overlap(p.buffer, "\n"+rippleToolOpen))
+ }
+ if p.callIndex == 0 {
+ content += p.buffer[:n]
+ }
+ p.buffer = p.buffer[n:]
+ return content, thinking, calls, nil
+ }
+ case rippleTool:
+ if i := p.toolEnd(); i >= 0 {
+ call, parseErr := parseRippleTool(p.buffer[:i])
+ p.buffer = p.buffer[i+len(rippleToolClose):]
+ p.state = rippleContent
+ if parseErr != nil {
+ return content, thinking, calls, parseErr
+ }
+ call.Function.Index = p.callIndex
+ p.callIndex++
+ calls = append(calls, call)
+ } else if !done {
+ return content, thinking, calls, nil
+ } else {
+ break parse
+ }
+ }
+ }
+ if done && p.state == rippleTool {
+ p.buffer = ""
+ p.state = rippleContent
+ return content, thinking, calls, fmt.Errorf("ripple: unterminated tool call")
+ }
+ return content, thinking, calls, nil
+}
+
+// toolEnd ignores closing tags inside JSON strings, including escaped quotes.
+func (p *RippleParser) toolEnd() int {
+ for p.toolScan < len(p.buffer) {
+ i := p.toolScan
+ c := p.buffer[i]
+ if p.toolString {
+ if p.toolEscape {
+ p.toolEscape = false
+ } else if c == '\\' {
+ p.toolEscape = true
+ } else if c == '"' {
+ p.toolString = false
+ }
+ } else if c == '"' {
+ p.toolString = true
+ } else if c == '<' {
+ if strings.HasPrefix(p.buffer[i:], rippleToolClose) {
+ return i
+ }
+ if strings.HasPrefix(rippleToolClose, p.buffer[i:]) {
+ return -1
+ }
+ }
+ p.toolScan++
+ }
+ return -1
+}
+
+func parseRippleTool(raw string) (api.ToolCall, error) {
+ var value struct {
+ Name string `json:"name"`
+ Arguments json.RawMessage `json:"arguments"`
+ }
+ if err := json.Unmarshal([]byte(raw), &value); err != nil {
+ return api.ToolCall{}, fmt.Errorf("ripple: invalid tool call JSON: %w", err)
+ }
+ if value.Name == "" {
+ return api.ToolCall{}, fmt.Errorf("ripple: empty function name")
+ }
+ if !strings.HasPrefix(strings.TrimSpace(string(value.Arguments)), "{") {
+ return api.ToolCall{}, fmt.Errorf("ripple: tool arguments must be a JSON object")
+ }
+ var arguments api.ToolCallFunctionArguments
+ if err := json.Unmarshal(value.Arguments, &arguments); err != nil {
+ return api.ToolCall{}, fmt.Errorf("ripple: invalid tool arguments: %w", err)
+ }
+ return api.ToolCall{Function: api.ToolCallFunction{Name: value.Name, Arguments: arguments}}, nil
+}
diff --git a/model/parsers/ripple_test.go b/model/parsers/ripple_test.go
new file mode 100644
index 000000000..54fc6ca5e
--- /dev/null
+++ b/model/parsers/ripple_test.go
@@ -0,0 +1,228 @@
+package parsers
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/ollama/ollama/api"
+)
+
+func rippleParseChunks(t *testing.T, chunks []string) (string, string, []api.ToolCall, error) {
+ t.Helper()
+ p := ParserForName("ripple")
+ if p == nil {
+ t.Fatal("missing ripple parser")
+ }
+ p.Init(nil, nil, nil)
+ var content, thinking strings.Builder
+ var calls []api.ToolCall
+ for _, chunk := range append(chunks, "") {
+ c, r, tc, err := p.Add(chunk, false)
+ content.WriteString(c)
+ thinking.WriteString(r)
+ calls = append(calls, tc...)
+ if err != nil {
+ return content.String(), thinking.String(), calls, err
+ }
+ }
+ c, r, tc, err := p.Add("", true)
+ content.WriteString(c)
+ thinking.WriteString(r)
+ calls = append(calls, tc...)
+ return content.String(), thinking.String(), calls, err
+}
+
+func rippleChunkings(text string) [][]string {
+ chunks := [][]string{{text}}
+ for i := 0; i <= len(text); i++ {
+ chunks = append(chunks, []string{text[:i], text[i:]})
+ }
+ bytes := make([]string, len(text))
+ for i := range len(text) {
+ bytes[i] = text[i : i+1]
+ }
+ return append(chunks, bytes)
+}
+
+func TestRippleParserThinking(t *testing.T) {
+ for _, tt := range []struct{ name, input, content, thinking string }{
+ {"plain", " \nhello\n", " \nhello\n", ""},
+ {"thinking", "\nreason\n\nanswer", "answer", "reason"},
+ {"no structural newlines", "reasonanswer", "answer", "reason"},
+ {"one newline only", "\n\n", "", ""},
+ {"meaningful whitespace", "\n\n \treason \n\n\n\n answer \n", "\n answer \n", "\n \treason \n"},
+ {"CRLF is not a structural newline", "\r\nreason\r\n\r\nanswer", "\r\nanswer", "\r\nreason\r"},
+ {"nonleading opener", " \nreason\nanswer", " \nreason\nanswer", ""},
+ {"no opener", "reason\n\nanswer", "reason\n\nanswer", ""},
+ {"later tags literal", "rcx", "cx", "r"},
+ {"tool inside thinking stays literal", "\n{}\n\nanswer", "answer", "{}"},
+ {"truncated opener", "\nreason\n\nreason\n", "", "reason\n"},
+ {"opener only", "", "", ""},
+ {"opener and newline", "\n", "", ""},
+ {"close only", "\nreason\n", "", "reason"},
+ {"partial tool text", "text\n\n日本語🙂\n\né\n", "é\n", "日本語🙂"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ for _, chunks := range rippleChunkings(tt.input) {
+ c, r, calls, err := rippleParseChunks(t, chunks)
+ if err != nil || c != tt.content || r != tt.thinking || len(calls) != 0 {
+ t.Fatalf("chunks %q: got (%q, %q, %v, %v), want (%q, %q)", chunks, c, r, calls, err, tt.content, tt.thinking)
+ }
+ }
+ p := ParserForName("ripple")
+ p.Init(nil, nil, nil)
+ c, r, calls, err := p.Add(tt.input, true)
+ if err != nil || c != tt.content || r != tt.thinking || len(calls) != 0 {
+ t.Fatalf("final chunk: (%q, %q, %v, %v)", c, r, calls, err)
+ }
+ c, r, calls, err = p.Add("", true)
+ if err != nil || c != "" || r != "" || len(calls) != 0 {
+ t.Fatal("final drain repeated output")
+ }
+ })
+ }
+}
+
+func TestRippleParserThinkingTransitions(t *testing.T) {
+ p := ParserForName("ripple")
+ p.Init(nil, nil, nil)
+ for _, tt := range []struct {
+ input, content, thinking string
+ close bool
+ }{
+ {"", "", "", true},
+ {"\nreason\n", "", "reason", true},
+ {"", "", "", false},
+ {"\nanswer", "answer", "", false},
+ } {
+ c, r, _, err := p.Add(tt.input, false)
+ if c != tt.content || r != tt.thinking || err != nil {
+ t.Fatalf("%q: (%q, %q, %v)", tt.input, c, r, err)
+ }
+ if close := p.ThinkingClose(); (len(close) != 0) != tt.close || tt.close && close[0] != "" {
+ t.Fatalf("%q: close %v", tt.input, close)
+ }
+ }
+ p.Init(nil, nil, nil)
+ c, r, _, err := p.Add("\nfresh\n\nnew", true)
+ if c != "new" || r != "fresh" || err != nil {
+ t.Fatalf("reset: (%q, %q, %v)", c, r, err)
+ }
+}
+
+func TestRippleParserTools(t *testing.T) {
+ one := "\n" + `{"name": "one", "arguments": {"z": "a\n\"b\"\\<&é", "n": 3, "nested": {"x": true}}}` + "\n"
+ two := "\n" + `{"name": "two", "arguments": {}}` + "\n"
+ for _, tt := range []struct {
+ name, input, content, thinking string
+ calls []api.ToolCall
+ }{
+ {"one", one, "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "one", Arguments: args(`{"z":"a\n\"b\"\\<&é","n":3,"nested":{"x":true}}`)}}}},
+ {"multiple", " \n before \t\n\n" + one + "\n" + two, " \n before \t\n", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "one", Arguments: args(`{"z":"a\n\"b\"\\<&é","n":3,"nested":{"x":true}}`)}}, {Function: api.ToolCallFunction{Index: 1, Name: "two", Arguments: args(`{}`)}}}},
+ {"reasoning and tools", "\n R \n\n\n" + two, "", " R ", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}}},
+ {"preserve spaces before call", " before \t" + two, " before \t", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}}},
+ {"drop post call text", two + "ignored \nalso text\n" + two + "more ignored", "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}, {Function: api.ToolCallFunction{Index: 1, Name: "two", Arguments: args(`{}`)}}}},
+ {"delimiter inside string", `{"name":"two","arguments":{"text":" \"quoted\""}}`, "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{"text":" \"quoted\""}`)}}}},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ for _, chunks := range rippleChunkings(tt.input) {
+ c, r, calls, err := rippleParseChunks(t, chunks)
+ if err != nil || c != tt.content || r != tt.thinking || len(calls) != len(tt.calls) {
+ t.Fatalf("chunks %q: (%q, %q, %v, %v)", chunks, c, r, calls, err)
+ }
+ for i := range calls {
+ if !toolCallEqual(calls[i], tt.calls[i]) {
+ t.Fatalf("call %d = %+v, want %+v", i, calls[i], tt.calls[i])
+ }
+ }
+ }
+ })
+ }
+}
+
+func TestRippleParserMalformedCalls(t *testing.T) {
+ for _, tt := range []struct{ name, input, errorText string }{
+ {"invalid JSON", "{bad}", "JSON"},
+ {"missing name", `{"arguments":{}}`, "name"},
+ {"empty name", `{"name":"","arguments":{}}`, "name"},
+ {"missing arguments", `{"name":"f"}`, "object"},
+ {"string arguments", `{"name":"f","arguments":"{}"}`, "object"},
+ {"array arguments", `{"name":"f","arguments":[]}`, "object"},
+ {"null arguments", `{"name":"f","arguments":null}`, "object"},
+ {"JSON array", `[{"name":"f","arguments":{}}]`, "JSON"},
+ {"multiple objects", `{"name":"f","arguments":{}}{}`, "JSON"},
+ {"truncated call", `{"name":"f","arguments":{}`, "unterminated"},
+ {"truncated closer", `{"name":"f","arguments":{}}{"name":"f","arguments":{}}`, true)
+ if c != "" || err != nil || len(calls) != 1 {
+ t.Fatalf("call: %q, %v, %v", c, calls, err)
+ }
+ p.Init(nil, nil, nil)
+ _, _, _, _ = p.Add("before\n", false)
+ c, _, _, err = p.Add("\nAnswer", " literal", "Text literal", "literal", "é 日本語 🙂"} {
+ t.Run(text, func(t *testing.T) {
+ content, thinking, calls, err := rippleParseChunks(t, []string{text})
+ if err != nil || content != text || thinking != "" || len(calls) != 0 {
+ t.Fatalf("got (%q, %q, %v, %v), want %q", content, thinking, calls, err, text)
+ }
+ })
+ }
+}
+
+func TestRippleParserCapabilities(t *testing.T) {
+ p := ParserForName("ripple")
+ if p == nil {
+ t.Fatal("missing ripple parser")
+ }
+ if !p.HasToolSupport() || !p.HasThinkingSupport() {
+ t.Fatal("missing capabilities")
+ }
+ if got := p.PreservedTokens(); !reflect.DeepEqual(got, []string{"", "", "", ""}) {
+ t.Fatalf("preserved = %v", got)
+ }
+ tools := []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "f"}}}
+ for _, think := range []*api.ThinkValue{nil, {Value: true}, {Value: false}, {Value: "high"}} {
+ if got := p.Init(tools, &api.Message{Role: "assistant", Thinking: "old", Content: "old"}, think); !reflect.DeepEqual(got, tools) {
+ t.Fatalf("changed tools: %v", got)
+ }
+ if got := p.ThinkingClose(); len(got) != 0 {
+ t.Fatalf("generation prompt does not open reasoning: %v", got)
+ }
+ }
+}
diff --git a/model/renderers/renderer.go b/model/renderers/renderer.go
index c175de20a..c11ca9f8a 100644
--- a/model/renderers/renderer.go
+++ b/model/renderers/renderer.go
@@ -59,6 +59,8 @@ func rendererForName(name string) Renderer {
return constructor()
}
switch name {
+ case "ripple":
+ return &RippleRenderer{}
case "qwen3-coder":
renderer := &Qwen3CoderRenderer{}
return renderer
diff --git a/model/renderers/ripple.go b/model/renderers/ripple.go
new file mode 100644
index 000000000..d0f88266e
--- /dev/null
+++ b/model/renderers/ripple.go
@@ -0,0 +1,88 @@
+package renderers
+
+import (
+ "fmt"
+ "strings"
+ "unicode"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/types/model"
+)
+
+type RippleRenderer struct{}
+
+func (r *RippleRenderer) LeadingBOS() string { return "" }
+
+// Ripple can emit reasoning but has no template control to disable or force it.
+func (r *RippleRenderer) Thinking() *model.Thinking { return nil }
+
+func (r *RippleRenderer) Render(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) {
+ if len(messages) == 0 {
+ return "", fmt.Errorf("ripple: expected at least one message")
+ }
+ for _, message := range messages {
+ if len(message.Images) > 0 {
+ return "", fmt.Errorf("ripple: text-only messages required")
+ }
+ }
+ var b strings.Builder
+ if len(tools) > 0 {
+ b.WriteString("<|im_start|>system\n")
+ if messages[0].Role == "system" {
+ b.WriteString(messages[0].Content + "\n\n")
+ }
+ b.WriteString("# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n")
+ for _, tool := range tools {
+ encoded, err := PythonJSON(tool, 0)
+ if err != nil {
+ return "", err
+ }
+ b.WriteString("\n" + encoded)
+ }
+ b.WriteString("\n\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n<|im_end|>\n")
+ }
+ for i, message := range messages {
+ if i == 0 && message.Role == "system" && len(tools) > 0 {
+ continue
+ }
+ content := message.Content
+ if message.Role == "assistant" && message.Thinking != "" && !strings.HasPrefix(strings.TrimLeftFunc(content, func(r rune) bool {
+ return unicode.IsSpace(r) || r >= '\x1c' && r <= '\x1f'
+ }), "") {
+ content = "\n" + message.Thinking + "\n\n" + content
+ }
+ switch message.Role {
+ case "assistant":
+ b.WriteString("<|im_start|>assistant")
+ if content != "" || len(message.ToolCalls) == 0 {
+ b.WriteString("\n" + content)
+ }
+ for _, call := range message.ToolCalls {
+ name, err := PythonJSON(call.Function.Name, 0)
+ if err != nil {
+ return "", err
+ }
+ arguments, err := PythonJSON(call.Function.Arguments, 0)
+ if err != nil {
+ return "", err
+ }
+ b.WriteString("\n\n{\"name\": " + name + ", \"arguments\": " + arguments + "}\n")
+ }
+ b.WriteString("<|im_end|>\n")
+ case "tool":
+ if i == 0 || messages[i-1].Role != "tool" {
+ b.WriteString("<|im_start|>user")
+ }
+ b.WriteString("\n\n" + content + "\n")
+ if i == len(messages)-1 || messages[i+1].Role != "tool" {
+ b.WriteString("<|im_end|>\n")
+ }
+ case "system", "user":
+ b.WriteString("<|im_start|>" + message.Role + "\n")
+ b.WriteString(content)
+ b.WriteString("<|im_end|>\n")
+ }
+ }
+ b.WriteString("<|im_start|>assistant\n")
+ return b.String(), nil
+}
diff --git a/model/renderers/ripple_test.go b/model/renderers/ripple_test.go
new file mode 100644
index 000000000..408ddfeaf
--- /dev/null
+++ b/model/renderers/ripple_test.go
@@ -0,0 +1,292 @@
+package renderers
+
+import (
+ "encoding/json"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/model/parsers"
+)
+
+const rippleTemplate = "testdata/ripple_chat_template.jinja"
+
+func TestRipplePublisherTemplate(t *testing.T) {
+ verifyLFM3TemplateFixture(t, rippleTemplate, "877c2dbf365243dda14738ca250b31398059b35ffd91d4ed0ec68be45ca6ebd3")
+}
+
+func TestRippleRendererTools(t *testing.T) {
+ call := api.ToolCall{Function: api.ToolCallFunction{Name: "f\"<&é", Arguments: args(`{"z":"line\n\"quote\"\\<&é","a":[true,null,3]}`)}}
+ tests := []struct {
+ name string
+ messages []api.Message
+ want string
+ }{
+ {
+ "call alone",
+ []api.Message{{Role: "assistant", ToolCalls: []api.ToolCall{call}}},
+ "<|im_start|>assistant\n\n" + `{"name": "f\"<&é", "arguments": {"z": "line\n\"quote\"\\<&é", "a": [true, null, 3]}}` + "\n<|im_end|>\n<|im_start|>assistant\n",
+ },
+ {
+ "multiple calls with reasoning",
+ []api.Message{{Role: "assistant", Thinking: "R", Content: " C \n", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "one"}}, {Function: api.ToolCallFunction{Name: "two"}}}}},
+ "<|im_start|>assistant\n\nR\n\n C \n\n\n{\"name\": \"one\", \"arguments\": {}}\n\n\n{\"name\": \"two\", \"arguments\": {}}\n<|im_end|>\n<|im_start|>assistant\n",
+ },
+ {
+ "grouped results",
+ []api.Message{{Role: "tool", Content: " A\n"}, {Role: "tool", Content: "B"}, {Role: "user", Content: "Next"}, {Role: "tool", Content: "C"}},
+ "<|im_start|>user\n\n A\n\n\n\nB\n<|im_end|>\n<|im_start|>user\nNext<|im_end|>\n<|im_start|>user\n\nC\n<|im_end|>\n<|im_start|>assistant\n",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := RenderWithRenderer("ripple", tt.messages, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != tt.want {
+ t.Fatalf("got %q\nwant %q", got, tt.want)
+ }
+ if os.Getenv("VERIFY_JINJA2") != "" {
+ want := renderQwenJinja(t, rippleTemplate, tt.messages, nil, nil, "")
+ if got != want {
+ t.Fatalf("Go %q\nPython %q", got, want)
+ }
+ }
+ })
+ }
+}
+
+func TestRippleRendererToolInstructions(t *testing.T) {
+ tool := api.Tool{Type: "function", Function: api.ToolFunction{Name: "execute_bash", Parameters: api.ToolFunctionParameters{Type: "object"}}}
+ for _, system := range []string{"", "Be precise."} {
+ messages := []api.Message{{Role: "system", Content: system}, {Role: "user", Content: "pwd"}}
+ got, err := RenderWithRenderer("ripple", messages, []api.Tool{tool}, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := "<|im_start|>system\n" + system + "\n\n" + `# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within XML tags:
+
+{"type": "function", "function": {"name": "execute_bash", "parameters": {"type": "object", "properties": null}}}
+
+
+For each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:
+
+{"name": "execute_bash", "arguments": {"command": "pwd"}}
+<|im_end|>
+<|im_start|>user
+pwd<|im_end|>
+<|im_start|>assistant
+`
+ if got != want {
+ t.Fatalf("got %q\nwant %q", got, want)
+ }
+ if os.Getenv("VERIFY_JINJA2") != "" {
+ want := renderQwenJinja(t, rippleTemplate, messages, []api.Tool{tool}, nil, "")
+ if got != want {
+ t.Fatalf("Go %q\nPython %q", got, want)
+ }
+ }
+ }
+}
+
+func TestRippleRendererInvalidInput(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ messages []api.Message
+ tools []api.Tool
+ errorText string
+ }{
+ {name: "empty transcript", errorText: "at least one message"},
+ {name: "empty transcript with tools", tools: []api.Tool{{Type: "function"}}, errorText: "at least one message"},
+ {name: "image", messages: []api.Message{{Role: "user", Content: "describe", Images: []api.ImageData{{1}}}}, errorText: "text-only"},
+ {name: "invalid tool schema", messages: []api.Message{{Role: "user"}}, tools: []api.Tool{{Items: make(chan int)}}, errorText: "unsupported type"},
+ {name: "invalid tool arguments", messages: []api.Message{{Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "f", Arguments: testArgs(map[string]any{"x": make(chan int)})}}}}}, errorText: "unsupported type"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := RenderWithRenderer("ripple", tt.messages, tt.tools, nil)
+ if got != "" || err == nil || !strings.Contains(err.Error(), tt.errorText) {
+ t.Fatalf("got (%q, %v), want %s", got, err, tt.errorText)
+ }
+ })
+ }
+}
+
+func TestRippleRendererToolResultContinuation(t *testing.T) {
+ messages := []api.Message{
+ {Role: "user", Content: "Run the program and report what it prints."},
+ {Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "execute_bash", Arguments: args(`{"command":"go run ./cmd/demo"}`)}}}},
+ {Role: "tool", Content: `{"exit_code":0,"stdout":"fmt.Println(\"done\")","stderr":""}`},
+ {Role: "assistant", Thinking: "The command succeeded; I can report its output.", Content: "The program prints `done`."},
+ }
+ tools := []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "execute_bash"}}}
+ want := "<|im_start|>system\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"execute_bash\", \"parameters\": {\"type\": \"\", \"properties\": null}}}\n\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n<|im_end|>\n<|im_start|>user\nRun the program and report what it prints.<|im_end|>\n<|im_start|>assistant\n\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"go run ./cmd/demo\"}}\n<|im_end|>\n<|im_start|>user\n\n{\"exit_code\":0,\"stdout\":\"fmt.Println(\\\"done\\\")\",\"stderr\":\"\"}\n<|im_end|>\n<|im_start|>assistant\n\nThe command succeeded; I can report its output.\n\nThe program prints `done`.<|im_end|>\n<|im_start|>assistant\n"
+ got, err := (&RippleRenderer{}).Render(messages, tools, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != want {
+ t.Fatalf("render mismatch:\ngot:\n%s\nwant:\n%s", got, want)
+ }
+}
+
+func TestRippleRendererJinjaParity(t *testing.T) {
+ if os.Getenv("VERIFY_JINJA2") == "" {
+ t.Skip("set VERIFY_JINJA2=1 and JINJA_PYTHON for publisher template parity")
+ }
+ values := api.NewToolCallFunctionArguments()
+ values.Set("z", json.Number("3.0"))
+ values.Set("a", json.RawMessage(`{"z": 1e-7, "a": ["\u2028\u2029<&é", true, null]}`))
+ call := api.ToolCall{Function: api.ToolCallFunction{Name: "f\"\\\n<&é", Arguments: values}}
+ tool := api.Tool{Type: "function", Function: api.ToolFunction{Name: "f", Description: "Use \"quoted\" <&é\n", Parameters: api.ToolFunctionParameters{Type: "object", Properties: propsMap(`{"z":{"type":"integer"},"a":{"type":"string"}}`), Required: []string{"z"}}}}
+ for _, tt := range []struct {
+ name string
+ messages []api.Message
+ tools []api.Tool
+ think *api.ThinkValue
+ }{
+ {name: "plain", messages: []api.Message{{Role: "user", Content: " \nHi\t "}}},
+ {name: "empty user", messages: []api.Message{{Role: "user"}}},
+ {name: "initial and later systems", messages: []api.Message{{Role: "system", Content: "A"}, {Role: "system", Content: "B"}, {Role: "user", Content: "C"}, {Role: "system", Content: "D"}}},
+ {name: "tools without system", messages: []api.Message{{Role: "user", Content: "Call"}}, tools: []api.Tool{tool, tool}},
+ {name: "old reasoning retained", messages: []api.Message{{Role: "assistant", Thinking: "old", Content: "first"}, {Role: "user", Content: "next"}, {Role: "assistant", Thinking: "new", Content: "second"}, {Role: "user", Content: "again"}}},
+ {name: "no duplicate wrapper", messages: []api.Message{{Role: "assistant", Thinking: "ignored", Content: "\x1c\t\u2003oldanswer"}}},
+ {name: "nonleading tag", messages: []api.Message{{Role: "assistant", Thinking: "new", Content: "wordsliteral"}}},
+ {name: "thinking false has no invented prefix", messages: []api.Message{{Role: "assistant", Thinking: "retain", Content: "answer"}}, think: &api.ThinkValue{Value: false}},
+ {name: "thinking true no forced opening", messages: []api.Message{{Role: "user", Content: "Hi"}}, think: &api.ThinkValue{Value: true}},
+ {name: "tools JSON numbers escaping and order", messages: []api.Message{{Role: "assistant", Thinking: "Reason", Content: " \n ", ToolCalls: []api.ToolCall{call}}, {Role: "tool", Content: " result \n"}, {Role: "tool", Content: "other"}}, tools: []api.Tool{tool}},
+ {name: "tools reasoning only", messages: []api.Message{{Role: "assistant", Thinking: "Reason", ToolCalls: []api.ToolCall{call}}}},
+ {name: "group boundary ignores unknown role", messages: []api.Message{{Role: "tool", Content: "one"}, {Role: "ignored"}, {Role: "tool", Content: "two"}}},
+ {
+ name: "tool result followed by assistant decision",
+ tools: []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "execute_bash"}}},
+ messages: []api.Message{
+ {Role: "user", Content: "Run the program and report what it prints."},
+ {Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "execute_bash", Arguments: args(`{"command":"go run ./cmd/demo"}`)}}}},
+ {Role: "tool", Content: `{"exit_code":0,"stdout":"fmt.Println(\"done\")","stderr":""}`},
+ {Role: "assistant", Thinking: "The command succeeded; I can report its output.", Content: "The program prints `done`."},
+ },
+ },
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := RenderWithRenderer("ripple", tt.messages, tt.tools, tt.think)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := renderQwenJinja(t, rippleTemplate, tt.messages, tt.tools, tt.think, "")
+ if got != want {
+ t.Fatalf("Go %q\nPython %q", got, want)
+ }
+ })
+ }
+ python := os.Getenv("JINJA_PYTHON")
+ if python == "" {
+ t.Fatal("set JINJA_PYTHON for negative publisher-template checks")
+ }
+ for _, messages := range []any{[]any{}, []any{map[string]any{"role": "user", "content": []any{map[string]any{"type": "image"}}}}} {
+ output, err := lfm3RunJinja2(t, python, rippleTemplate, map[string]any{"messages": messages, "add_generation_prompt": true})
+ if err == nil {
+ t.Fatalf("publisher accepted unsupported transcript: %q", output)
+ }
+ }
+}
+
+func TestRippleRendererParserRoundTrip(t *testing.T) {
+ for _, content := range []string{"", " ", "\n", "\n\n", " \t\n ", "answer\n", "日本語\n"} {
+ for _, thinking := range []string{"", "R", " \n", "\n\nR\t\n"} {
+ for _, calls := range [][]api.ToolCall{nil, {{Function: api.ToolCallFunction{Name: "f", Arguments: args(`{"text":"<&é\n\"quoted\""}`)}}}, {{Function: api.ToolCallFunction{Name: "f"}}, {Function: api.ToolCallFunction{Name: "g"}}}} {
+ message := api.Message{Role: "assistant", Content: content, Thinking: thinking, ToolCalls: calls}
+ original, err := RenderWithRenderer("ripple", []api.Message{message}, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ body := strings.TrimSuffix(strings.TrimPrefix(original, "<|im_start|>assistant\n"), "<|im_end|>\n<|im_start|>assistant\n")
+ p := parsers.ParserForName("ripple")
+ p.Init(nil, nil, nil)
+ parsed := api.Message{Role: "assistant"}
+ for i := 0; i <= len(body); i++ {
+ chunk := ""
+ if i < len(body) {
+ chunk = body[i : i+1]
+ }
+ c, r, tc, err := p.Add(chunk, i == len(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ parsed.Content += c
+ parsed.Thinking += r
+ parsed.ToolCalls = append(parsed.ToolCalls, tc...)
+ }
+ got, err := RenderWithRenderer("ripple", []api.Message{parsed}, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != original {
+ t.Fatalf("roundtrip got %q\nwant %q", got, original)
+ }
+ }
+ }
+ }
+}
+
+func TestRippleRendererReasoningHistory(t *testing.T) {
+ for _, tt := range []struct{ name, content, reasoning, want string }{
+ {"separate", " Answer \n", " Reason \n", "\n Reason \n\n\n Answer \n"},
+ {"already tagged", "\noriginal\n\nAnswer", "ignored", "\noriginal\n\nAnswer"},
+ {"leading whitespace already tagged", "\x1c\t\u2003originalAnswer", "ignored", "\x1c\t\u2003originalAnswer"},
+ {"nonleading tag", "Text literal", "Reason", "\nReason\n\nText literal"},
+ {"empty reasoning", "Answer", "", "Answer"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ for _, think := range []*api.ThinkValue{nil, {Value: true}, {Value: false}, {Value: "high"}} {
+ got, err := RenderWithRenderer("ripple", []api.Message{{Role: "assistant", Content: tt.content, Thinking: tt.reasoning}, {Role: "user", Content: "Next"}}, nil, think)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := "<|im_start|>assistant\n" + tt.want + "<|im_end|>\n<|im_start|>user\nNext<|im_end|>\n<|im_start|>assistant\n"
+ if got != want {
+ t.Fatalf("got %q\nwant %q", got, want)
+ }
+ }
+ })
+ }
+}
+
+func TestRippleRendererMessages(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ messages []api.Message
+ want string
+ }{
+ {"plain", []api.Message{{Role: "user", Content: " Hi\n"}}, "<|im_start|>user\n Hi\n<|im_end|>\n<|im_start|>assistant\n"},
+ {"system", []api.Message{{Role: "system", Content: "Be precise."}, {Role: "user", Content: "Hi"}}, "<|im_start|>system\nBe precise.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n"},
+ {"empty system", []api.Message{{Role: "system"}}, "<|im_start|>system\n<|im_end|>\n<|im_start|>assistant\n"},
+ {"later system", []api.Message{{Role: "user", Content: "Hi"}, {Role: "system", Content: "Now"}}, "<|im_start|>user\nHi<|im_end|>\n<|im_start|>system\nNow<|im_end|>\n<|im_start|>assistant\n"},
+ {"assistant is closed not prefilled", []api.Message{{Role: "assistant", Content: "Part"}}, "<|im_start|>assistant\nPart<|im_end|>\n<|im_start|>assistant\n"},
+ {"unknown role ignored", []api.Message{{Role: "developer", Content: "ignored"}}, "<|im_start|>assistant\n"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := RenderWithRenderer("ripple", tt.messages, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != tt.want {
+ t.Fatalf("got %q\nwant %q", got, tt.want)
+ }
+ })
+ }
+ if rendererForName("ripple") == nil {
+ t.Fatal("missing ripple renderer")
+ }
+ if got := LeadingBOSForRenderer("ripple"); got != "" {
+ t.Fatalf("BOS = %q", got)
+ }
+ if got := ThinkingForRenderer("ripple"); got != nil {
+ t.Fatalf("unexpected thinking control: %+v", got)
+ }
+}
diff --git a/model/renderers/testdata/ripple_chat_template.jinja b/model/renderers/testdata/ripple_chat_template.jinja
new file mode 100644
index 000000000..e4b0bd675
--- /dev/null
+++ b/model/renderers/testdata/ripple_chat_template.jinja
@@ -0,0 +1,52 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- set content = message.content or '' %}
+ {%- if message.role == 'assistant' and message.reasoning_content and not (clear_thinking | default(false)) and not content.lstrip().startswith('') %}
+ {%- set content = '\n' + message.reasoning_content + '\n\n' + content %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if content %}
+ {{- '\n' + content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": ' }}
+ {{- tool_call.name | tojson }}
+ {{- ', "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' + message.content + '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}