Add ripple preview support to the MLX runner (#19592)

Add the Ripple model, renderer, parser and import registration.
Support per-layer NoPE in DFlash drafts.
Fuse MLP activation, value blending and attention gating.
Add reference fixtures and model/runner regression tests.
This commit is contained in:
Patrick Devine 2026-11-07 16:36:53 -08:00 • committed by GitHub
parent 5e983d4f80
commit d63b702609
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 3878 additions and 200 deletions

View file

@ -282,6 +282,8 @@ func parserNameForConfig(modelDir string, cfg sourceModelConfig, chatTemplate st
func parserNameForIdentifier(modelDir, s, chatTemplate string) (string, error) {
s = strings.ToLower(s)
switch {
case s == "ricursiveforcausallm" || s == "ricursive":
return "ripple", nil
case strings.Contains(s, "flint"):
return "flint", nil
case strings.HasPrefix(s, "lfm3"):
@ -322,6 +324,8 @@ func rendererNameForConfig(modelDir string, cfg sourceModelConfig, chatTemplate
func rendererNameForIdentifier(modelDir, s, chatTemplate string) (string, error) {
s = strings.ToLower(s)
switch {
case s == "ricursiveforcausallm" || s == "ricursive":
return "ripple", nil
case strings.Contains(s, "flint"):
return "flint", nil
case strings.HasPrefix(s, "lfm3"):

View file

@ -102,6 +102,23 @@ func TestInferSafetensorsConfigFamilies(t *testing.T) {
wantRenderer string
wantCaps []string
}{
{
name: "ripple architecture",
config: `{"architectures":["RicursiveForCausalLM"],"model_type":"ricursive"}`,
chatTemplate: `{% for message in messages %}{{ message.content }}{% endfor %}`,
wantParser: "ripple",
wantRenderer: "ripple",
wantCaps: []string{"completion", "tools", "thinking"},
},
{
name: "ripple model type",
config: `{"model_type":"ricursive"}`,
chatTemplate: `<think>{{ reasoning_content }}</think>{{ content }}`,
standalone: true,
wantParser: "ripple",
wantRenderer: "ripple",
wantCaps: []string{"completion", "tools", "thinking"},
},
{
name: "qwen3",
config: `{"architectures":["Qwen3ForCausalLM"]}`,

View file

@ -23,5 +23,6 @@ import (
_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5"
_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5_moe"
_ "github.com/ollama/ollama/mlxrunner/model/qwen4_exp"
_ "github.com/ollama/ollama/mlxrunner/model/ripple"
_ "github.com/ollama/ollama/mlxrunner/model/strands"
)

View file

@ -3,12 +3,14 @@ package dflash
import (
"fmt"
"math"
"slices"
"testing"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlx/mlxtest"
"github.com/ollama/ollama/mlxrunner/batch"
"github.com/ollama/ollama/mlxrunner/cache"
"github.com/ollama/ollama/mlxrunner/model"
"github.com/ollama/ollama/mlxrunner/nn"
)
@ -120,6 +122,298 @@ func TestRepeatedBlockRollback(t *testing.T) {
}
}
// The scalar oracle uses nonzero queries and checks both cache K/V and output,
// so skipping only context K, block K, or block Q cannot satisfy the contract.
func TestDraftPerLayerNoRoPE(t *testing.T) {
for _, policy := range []struct {
name, field string
noRoPE [2]bool
}{
{"omitted", "", [2]bool{}},
{"empty", `,"no_rope_layers":[]`, [2]bool{}},
{"full_nope", `,"no_rope_layers":[1]`, [2]bool{false, true}},
{"sliding_nope", `,"no_rope_layers":[0]`, [2]bool{true, false}},
} {
for _, path := range []struct {
name string
context, block bool
}{
{"context_only", true, false},
{"block_only", false, true},
{"combined", true, true},
} {
mlxtest.RunSubtest(t, policy.name+"/"+path.name, func(t *mlxtest.T) {
mlx.Scoped(func() {
m := newDraftRotaryTestModel(t, policy.field)
caches := m.NewCaches()
for _, c := range caches {
defer c.Free()
}
var oracle draftRotaryOracle
// Seed the real caches without invoking either rotary path.
// Block-only must not fail because context maintenance is broken.
seed := []float32{0.25, -0.5, 0.75, 0.25, -0.5, 1}
for i, c := range caches {
c.(cache.Attention).Update(&batch.Batch{SeqOffsets: []int32{0}},
mlx.Zeros(mlx.DTypeFloat32, 1, 1, 3, 2), mlx.FromValues(seed, 1, 1, 3, 2))
oracle.keys[i] = make([]float64, len(seed))
for _, v := range seed {
oracle.values[i] = append(oracle.values[i], float64(v))
}
}
for round := range 2 {
start := caches[0].Offset()
b := &batch.Batch{SeqOffsets: []int32{int32(start)}}
var context, block []float64
if path.context {
context = []float64{1, 2, -3, 1}
b.Hidden = mlx.FromValues([]float32{1, 2, -3, 1}, 1, 2, 2)
}
if path.block {
block = []float64{1, 2, -2, 1, 0.5, -1}
b.InputIDs = mlx.FromValues([]int32{0, 1, 2}, 1, 3)
b.SeqQueryLens = []int32{3}
}
want := oracle.forward(context, block, start, policy.noRoPE)
hidden, aux := m.Forward(b, nil, caches)
label := fmt.Sprintf("round %d offset %d", round, start)
if !path.block {
if hidden != nil || aux != nil {
t.Fatal("context-only forward returned block hidden states")
}
} else {
if hidden == nil || aux != hidden {
t.Fatal("block forward did not return shared hidden/aux states")
}
checkDraftRotaryArray(t, label+" hidden", hidden, want)
}
for i, c := range caches {
if wantOffset := len(oracle.keys[i]) / 2; c.Offset() != wantOffset {
t.Fatalf("%s layer %d cache offset=%d want=%d", label, i, c.Offset(), wantOffset)
}
state := c.State()
if len(state) != 2 {
t.Fatalf("%s layer %d state roots=%d want=2", label, i, len(state))
}
checkDraftRotaryArray(t, fmt.Sprintf("%s layer %d keys", label, i), state[0], oracle.keys[i])
checkDraftRotaryArray(t, fmt.Sprintf("%s layer %d values", label, i), state[1], oracle.values[i])
}
}
})
})
}
}
}
type windowedDraftTarget struct {
*draftRotaryTarget
window int
}
func (t *windowedDraftTarget) DraftWindow() int { return t.window }
func checkDraftWindowValues(t *mlxtest.T, label string, array *mlx.Array, want []float64) {
t.Helper()
array = mlx.Contiguous(array, false).AsType(mlx.DTypeFloat32)
mlx.Eval(array)
got := array.Floats()
wantFloat := make([]float32, len(want))
for i, v := range want {
wantFloat[i] = float32(v)
}
slices.Sort(got)
slices.Sort(wantFloat)
if !slices.Equal(got, wantFloat) {
t.Fatalf("%s values = %v, want %v", label, got, wantFloat)
}
}
func TestDraftWindowCache(t *testing.T) {
for _, tc := range []struct {
name string
target draftTarget
initial, wrapped [][]float64
}{
{
name: "default", target: &draftRotaryTarget{},
initial: [][]float64{{2, 3}, {0, 1, 2, 3}},
wrapped: [][]float64{{3, 4}, {0, 1, 2, 3, 4}},
},
{
name: "bounded", target: &windowedDraftTarget{draftRotaryTarget: &draftRotaryTarget{}, window: 3},
initial: [][]float64{{2, 3}, {1, 2, 3}},
wrapped: [][]float64{{3, 4}, {2, 3, 4}},
},
} {
t.Run(tc.name, func(t *testing.T) {
mlxtest.Run(t, func(t *mlxtest.T) {
m := &Model{
Config: &Config{SlidingWindow: 2},
Layers: []*Layer{{IsSliding: true}, {IsSliding: false}},
target: tc.target,
}
caches := m.NewCaches()
defer func() {
for _, c := range caches {
c.Free()
}
}()
initial := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{4}}
values := mlx.FromValues([]float32{0, 1, 2, 3}, 1, 1, 4, 1)
for i, c := range caches {
c.(cache.Attention).Update(initial, values, values)
checkDraftWindowValues(t, fmt.Sprintf("initial cache %d", i), c.(cache.Attention).View(initial).V(), tc.initial[i])
c.PrepareSnapshots([]int{4})
next := &batch.Batch{SeqOffsets: []int32{4}, SeqQueryLens: []int32{1}}
value := mlx.FromValue(float32(4))
value = mlx.Reshape(value, 1, 1, 1, 1)
c.(cache.Attention).Update(next, value, value)
checkDraftWindowValues(t, fmt.Sprintf("wrapped cache %d", i), c.(cache.Attention).View(next).V(), tc.wrapped[i])
snapshots := c.TakeSnapshots()
if len(snapshots) != 1 {
t.Fatalf("cache %d snapshots = %d", i, len(snapshots))
}
if !c.Restore(snapshots[0], 4) {
t.Fatalf("cache %d could not restore", i)
}
if snapshots[0] != nil {
snapshots[0].Close()
}
if c.Offset() != 4 {
t.Fatalf("cache %d offset = %d", i, c.Offset())
}
checkDraftWindowValues(t, fmt.Sprintf("restored cache %d", i), c.(cache.Attention).View(initial).V(), tc.initial[i])
}
})
})
}
}
type draftRotaryTarget struct {
model.Model
embeddings *mlx.Array
}
func (m *draftRotaryTarget) TokenEmbeddings(ids *mlx.Array) *mlx.Array {
return m.embeddings.TakeAxis(ids, 0)
}
func (*draftRotaryTarget) RawLogits(x *mlx.Array) *mlx.Array { return x }
func (*draftRotaryTarget) SetAuxHiddenLayers([]int) {}
func (*draftRotaryTarget) NumLayers() int { return 2 }
func newDraftRotaryTestModel(t *mlxtest.T, field string) *Model {
t.Helper()
cfg, err := parseConfig(draftRotaryTestConfig(field))
if err != nil {
t.Fatal(err)
}
m := &Model{
Config: cfg, Layers: make([]*Layer, cfg.NumHiddenLayers), tensorPrefix: "draft.",
target: &draftRotaryTarget{embeddings: mlx.FromValues([]float32{1, 2, -2, 1, 0.5, -1}, 3, 2)},
}
identity := mlx.FromValues([]float32{1, 0, 0, 1}, 2, 2)
ones := mlx.FromValues([]float32{1, 1}, 2)
zero := mlx.Zeros(mlx.DTypeFloat32, 2, 2)
tensors := map[string]*mlx.Array{
"draft.fc.weight": identity, "draft.hidden_norm.weight": ones, "draft.norm.weight": ones,
}
for i := range m.Layers {
prefix := fmt.Sprintf("draft.layers.%d.", i)
for _, name := range []string{"q_proj", "k_proj", "v_proj", "o_proj"} {
tensors[prefix+"self_attn."+name+".weight"] = identity
}
for _, name := range []string{"input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm"} {
tensors[prefix+name+".weight"] = ones
}
for _, name := range []string{"gate_proj", "up_proj", "down_proj"} {
tensors[prefix+"mlp."+name+".weight"] = zero
}
}
if err := m.LoadWeights(tensors); err != nil {
t.Fatal(err)
}
return m
}
func checkDraftRotaryArray(t *mlxtest.T, label string, array *mlx.Array, want []float64) {
t.Helper()
// Cache state can be a strided view; Floats requires contiguous storage.
array = mlx.Contiguous(array, false).AsType(mlx.DTypeFloat32)
mlx.Eval(array)
got := array.Floats()
if len(got) != len(want) {
t.Fatalf("%s length=%d want=%d", label, len(got), len(want))
}
for i, v := range got {
if math.IsNaN(float64(v)) || math.Abs(float64(v)-want[i]) > 2e-3*math.Max(1, math.Abs(want[i])) {
t.Errorf("%s[%d]: got %g want %g", label, i, v, want[i])
return
}
}
}
// This two-wide scalar model has identity projections, unit norms, zero MLPs,
// and a window larger than its history. It does not call production RoPE/SDPA.
type draftRotaryOracle struct {
keys, values [2][]float64
}
func draftRotaryNorm(x []float64) []float64 {
out := make([]float64, len(x))
for r := 0; r < len(x); r += 2 {
inv := 1 / math.Sqrt((x[r]*x[r]+x[r+1]*x[r+1])/2+1e-6)
out[r], out[r+1] = x[r]*inv, x[r+1]*inv
}
return out
}
func draftRotaryRotate(x []float64, start int) []float64 {
out := slices.Clone(x)
for r := 0; r < len(x); r += 2 {
s, c := math.Sincos(float64(start + r/2))
out[r], out[r+1] = x[r]*c-x[r+1]*s, x[r]*s+x[r+1]*c
}
return out
}
func (o *draftRotaryOracle) forward(context, block []float64, start int, noRoPE [2]bool) []float64 {
contextValues := draftRotaryNorm(context)
x := slices.Clone(block)
for layer := range o.keys {
contextKeys := draftRotaryNorm(contextValues)
values := draftRotaryNorm(x)
q := draftRotaryNorm(values)
k := slices.Clone(q)
if !noRoPE[layer] {
contextKeys = draftRotaryRotate(contextKeys, start)
q = draftRotaryRotate(q, start+len(context)/2)
k = draftRotaryRotate(k, start+len(context)/2)
}
o.keys[layer] = append(o.keys[layer], contextKeys...)
o.keys[layer] = append(o.keys[layer], k...)
o.values[layer] = append(o.values[layer], contextValues...)
o.values[layer] = append(o.values[layer], values...)
for r := 0; r < len(x); r += 2 {
scores := make([]float64, len(o.keys[layer])/2)
for j := range scores {
scores[j] = (q[r]*o.keys[layer][2*j] + q[r+1]*o.keys[layer][2*j+1]) / math.Sqrt(2)
}
maxScore := slices.Max(scores)
total := 0.0
for j, score := range scores {
scores[j] = math.Exp(score - maxScore)
total += scores[j]
}
for j, score := range scores {
x[r] += score / total * o.values[layer][2*j]
x[r+1] += score / total * o.values[layer][2*j+1]
}
}
}
return draftRotaryNorm(x)
}
// meanAttention is one head of zero queries over width-wide rows: every output
// row is the mean of the value rows it may see, and values copy their input.
func meanAttention(width int, window int32) (*Config, *Attention) {

View file

@ -7,6 +7,7 @@ import (
"encoding/json"
"fmt"
"math"
"slices"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlxrunner/batch"
@ -49,6 +50,7 @@ type Config struct {
Scale float32
SlidingWindow int32
LayerTypes []string
NoRoPELayers []int
BlockSize int
MaskTokenID int32
@ -143,6 +145,7 @@ type Attention struct {
OProj nn.LinearLayer
QNorm *nn.RMSNorm
KNorm *nn.RMSNorm
NoRoPE bool
// ctxInputNorm applies the layer's input norm to context rows (laguna).
ctxInputNorm *nn.RMSNorm
@ -183,6 +186,7 @@ func parseConfig(data []byte) (*Config, error) {
VocabSize int32 `json:"vocab_size"`
HasOwnLMHead bool `json:"has_own_lm_head"`
LayerTypes []string `json:"layer_types"`
NoRoPELayers []int `json:"no_rope_layers"`
SlidingWindow int32 `json:"sliding_window"`
RopeInterleaved *bool `json:"rope_interleaved"`
}
@ -200,6 +204,7 @@ func parseConfig(data []byte) (*Config, error) {
RopeTheta: raw.RopeTheta,
SlidingWindow: raw.SlidingWindow,
LayerTypes: raw.LayerTypes,
NoRoPELayers: raw.NoRoPELayers,
BlockSize: raw.DFlashConfig.BlockSize,
VocabSize: raw.VocabSize,
HasOwnLMHead: raw.HasOwnLMHead,
@ -217,6 +222,11 @@ func parseConfig(data []byte) (*Config, error) {
if cfg.NumHiddenLayers > maxDraftLayers {
return nil, fmt.Errorf("dflash num_hidden_layers %d exceeds supported maximum %d", cfg.NumHiddenLayers, maxDraftLayers)
}
for _, id := range cfg.NoRoPELayers {
if id < 0 || id >= int(cfg.NumHiddenLayers) {
return nil, fmt.Errorf("dflash no_rope_layers index %d out of range for %d layers", id, cfg.NumHiddenLayers)
}
}
// The fused projection holds the query, key and value rows.
if (int64(cfg.NumAttentionHeads)+2*int64(cfg.NumKeyValueHeads))*int64(cfg.HeadDim) > math.MaxInt32 {
return nil, fmt.Errorf("dflash attention projection dimensions exceed int32")
@ -391,8 +401,9 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {
layer := &Layer{
IsSliding: m.LayerTypes[i] == "sliding_attention",
Attention: &Attention{
GProj: linears.Make(layerPrefix + ".self_attn.g_proj"),
OProj: linears.Make(layerPrefix + ".self_attn.o_proj"),
NoRoPE: slices.Contains(m.NoRoPELayers, i),
GProj: linears.Make(layerPrefix + ".self_attn.g_proj"),
OProj: linears.Make(layerPrefix + ".self_attn.o_proj"),
},
MLP: &MLP{
DownProj: linears.Make(layerPrefix + ".mlp.down_proj"),
@ -583,9 +594,19 @@ func (m *Model) BlockParams() (int, int32) { return m.BlockSize, m.MaskTokenID }
// NewCaches builds the per-layer context caches.
func (m *Model) NewCaches() []cache.Cache {
caches := make([]cache.Cache, len(m.Layers))
draftWindow := 0
if target, ok := m.target.(interface{ DraftWindow() int }); ok {
draftWindow = target.DraftWindow()
}
for i, layer := range m.Layers {
if layer.IsSliding {
caches[i] = cache.NewRotatingKVCache(int(m.SlidingWindow))
window := int(m.SlidingWindow)
if draftWindow > 0 {
window = min(window, draftWindow)
}
caches[i] = cache.NewRotatingKVCache(window)
} else if draftWindow > 0 {
caches[i] = cache.NewRotatingKVCache(draftWindow)
} else {
caches[i] = cache.NewKVCache()
}
@ -681,7 +702,9 @@ func (a *Attention) contextKV(hctx *mlx.Array, positions *mlx.Array, cfg *Config
k = a.KNorm.Forward(k, cfg.RMSNormEps)
k = mlx.Transpose(k, 0, 2, 1, 3)
v = mlx.Transpose(v, 0, 2, 1, 3)
k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
if !a.NoRoPE {
k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
}
return k, v
}
@ -703,8 +726,10 @@ func (a *Attention) Forward(x, ctxK, ctxV *mlx.Array, c cache.Cache, bb *batch.B
k = mlx.Transpose(k, 0, 2, 1, 3)
v = mlx.Transpose(v, 0, 2, 1, 3)
q = mlx.RoPEWithBase(q, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
if !a.NoRoPE {
q = mlx.RoPEWithBase(q, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
}
if ctxK != nil {
k = ctxK.Concatenate(2, k)

View file

@ -241,6 +241,89 @@ func TestSelectorConditionalProbabilities(t *testing.T) {
}
}
func TestFourTapConvolutionOracle(t *testing.T) {
for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} {
for _, length := range []int{1, 2, 3, 4, 8, 16} {
mlxtest.RunSubtest(t, fmt.Sprintf("%v length %d", dtype, length), func(t *mlxtest.T) {
const width, group, taps = 32, 16, 4
values := func(n int) []float32 {
v := make([]float32, n)
for i := range v {
v[i] = float32(i%7-3) / 8
}
return v
}
h, d, s := values(length*width), values(length*taps*width/group), values(taps*width)
want := convolutionOracle(h, d, s, 1, length, width, taps, width/group, func(v float32) float32 {
return rounded(v, dtype == mlx.DTypeBFloat16)
})
inputs := []*mlx.Array{mlx.FromValues(h, 1, length, width).AsType(dtype), mlx.FromValues(d, 1, length, taps, width/group).AsType(dtype), mlx.FromValues(s, taps, width).AsType(dtype)}
for _, got := range []*mlx.Array{convolveGraph(inputs[0], inputs[1], inputs[2]), convolve(inputs[0], inputs[1], inputs[2])} {
got = mlx.Contiguous(got.AsType(mlx.DTypeFloat32), false)
mlx.Eval(got)
if !slices.Equal(got.Floats(), want) {
t.Fatal("four-tap convolution differs from scalar oracle")
}
}
})
}
}
}
func TestSelectorTop32Probabilities(t *testing.T) {
mlxtest.Run(t, func(t *mlxtest.T) {
const vocab, rank, k, length = 40, 256, 32, 3
pred, succ, projection := make([]float32, vocab*rank), make([]float32, vocab*rank), make([]float32, rank)
for id := range vocab {
for j := range rank {
pred[id*rank+j] = float32(id%3-1) / 32
succ[id*rank+j] = float32(id%5-2) / 64
}
}
for j := range projection {
projection[j] = 0.5
}
logits := make([]float32, length*vocab)
for i := range logits {
logits[i] = -float32(i%vocab) / 8
}
m := &model2{Selector: &candidateSelector{
Projection: nn.NewLinear(mlx.FromValues(projection, rank, 1).AsType(mlx.DTypeBFloat16), nil),
Predecessor: mlx.FromValues(pred, vocab, rank).AsType(mlx.DTypeBFloat16),
Successor: mlx.FromValues(succ, vocab, rank).AsType(mlx.DTypeBFloat16), TopK: k,
}}
sampler := sample.New(16)
defer sampler.Free()
sampler.Add(0, sample.Options{Temperature: 1, UseSeed: true, Seed: 42}, nil)
tokens, ids, probs := m.Select(mlx.FromValues([]float32{1, 1, 1}, 1, length, 1).AsType(mlx.DTypeBFloat16), mlx.FromValues(logits, 1, length, vocab).AsType(mlx.DTypeBFloat16), mlx.FromValues([]int32{0}, 1), func(ids, scores *mlx.Array) (*mlx.Array, *mlx.Array) {
tokens, distribution := sampler.SampleCandidates(0, ids, scores)
return tokens, distribution.Probs
})
mlx.Eval(tokens, ids, probs)
path, support, q := tokens.Ints(), ids.Ints(), mlx.Contiguous(probs, false).Floats()
previous := 0
for row := range length {
weights, total := make([]float64, k), float64(0)
for id := range k {
weights[id] = math.Exp(-float64(id)/8 + float64((previous%3-1)*(id%5-2))/16)
total += weights[id]
}
for j, id := range support[row*k : (row+1)*k] {
if id < 0 || id >= k {
t.Fatalf("unexpected candidate %d", id)
}
if want := weights[id] / total; math.Abs(float64(q[row*k+j])-want) > 1e-6 {
t.Fatalf("row %d candidate %d probability=%g want=%g", row, id, q[row*k+j], want)
}
}
if !slices.Contains(support[row*k:(row+1)*k], path[row]) {
t.Fatal("selected token outside proposal support")
}
previous = int(path[row])
}
})
}
func TestDFlash2ConfigValidation(t *testing.T) {
valid := Config{HiddenSize: 5120, ConvGroupSize: 16, ConvKernelSize: 2, BlockSize: 8, SelectorRank: 256, SelectorTopK: 16, VocabSize: 248320}
for _, tt := range []struct {

View file

@ -2,9 +2,11 @@ package dflash
import (
"encoding/json"
"fmt"
"maps"
"math"
"slices"
"strings"
"testing"
)
@ -50,6 +52,50 @@ func TestDraftConfigBoundsAndCausality(t *testing.T) {
}
}
// Keep the policy in JSON so these tests exercise parsing and loader wiring.
func draftRotaryTestConfig(noRoPEField string) []byte {
return []byte(fmt.Sprintf(`{
"hidden_size": 2, "num_hidden_layers": 2,
"num_attention_heads": 1, "num_key_value_heads": 1, "head_dim": 2,
"rms_norm_eps": 0.000001, "rope_theta": 10000,
"layer_types": ["sliding_attention", "full_attention"], "sliding_window": 64,
"dflash_config": {"block_size": 4, "mask_token_id": 0, "target_layer_ids": [0], "causal": false}
%s
}`, noRoPEField))
}
func TestDraftConfigNoRoPELayers(t *testing.T) {
for _, tt := range []struct {
name, field string
valid bool
}{
{"omitted", "", true},
{"empty", `,"no_rope_layers":[]`, true},
{"first", `,"no_rope_layers":[0]`, true},
{"last", `,"no_rope_layers":[1]`, true},
{"unordered", `,"no_rope_layers":[1,0]`, true},
{"negative", `,"no_rope_layers":[-1]`, false},
{"past_last", `,"no_rope_layers":[2]`, false},
{"invalid_after_valid", `,"no_rope_layers":[0,2]`, false},
} {
t.Run(tt.name, func(t *testing.T) {
_, err := parseConfig(draftRotaryTestConfig(tt.field))
if tt.valid {
if err != nil {
t.Fatal(err)
}
return
}
if err == nil {
t.Fatalf("parseConfig accepted invalid %s", tt.field)
}
if !strings.Contains(err.Error(), "no_rope_layers") {
t.Fatalf("error %q does not identify no_rope_layers", err)
}
})
}
}
func FuzzDraftConfig(f *testing.F) {
f.Add([]byte(`{"hidden_size":32,"num_hidden_layers":2,"num_attention_heads":4,"num_key_value_heads":2,"head_dim":8,"is_causal":false,"vocab_size":64,"dflash_config":{"block_size":8,"mask_token_id":7,"target_layer_ids":[1,3],"conv_group_size":16,"conv_kernel_size":2,"selector_rank":4,"selector_top_k":2}}`))
f.Add([]byte(`{"num_hidden_layers":-1}`))

View file

@ -0,0 +1,156 @@
package ripple
import (
"math"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlxrunner/batch"
"github.com/ollama/ollama/mlxrunner/cache"
"github.com/ollama/ollama/mlxrunner/nn"
)
func (m *Model) Forward(b *batch.Batch, caches []cache.Cache) (hidden, auxHidden *mlx.Array) {
if caches != nil && len(caches) < len(m.Layers) {
panic("ripple: cache count must cover target layers")
}
if caches != nil && b.InputIDs.Dim(0) != 1 {
panic("ripple: cached batches require one sequence")
}
h := scaleEmbedding(m.EmbedTokens.Forward(b.InputIDs), m.EmbedScale)
seed := h
positions := mlx.Add(mlx.Reshape(mlx.FromValues(b.SeqOffsets, len(b.SeqOffsets)), int32(len(b.SeqOffsets)), 1), mlx.Arange(0, float64(b.InputIDs.Dim(1)), 1, mlx.DTypeInt32))
cos, sin := rotaryTables(positions, m.HeadDim, m.RopeTheta, h.DType())
var features []*mlx.Array
for i, layer := range m.Layers {
var c cache.Cache
if caches != nil {
c = caches[i]
}
h = layer.Forward(h, seed, b, c, cos, sin, m.LayerTypes[i] == "sliding_attention", m.Config)
if next := len(features); next < len(m.auxHiddenLayers) && i == m.auxHiddenLayers[next] {
features = append(features, h)
}
}
h = rmsNorm(h, m.Norm, m.RMSNormEps)
if len(features) > 0 {
return h, mlx.Concatenate(features, -1)
}
return h, h
}
func (l *Layer) Forward(x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array {
u := rmsNorm(x, l.InputNorm, cfg.RMSNormEps)
a := l.Attention.Forward(u, seed, b, c, cos, sin, local, cfg)
h := mlx.Add(x, rmsNorm(a, l.PostAttentionNorm, cfg.RMSNormEps))
f := l.MLP.Forward(rmsNorm(h, l.PreFFNNorm, cfg.RMSNormEps), cfg.FFNClampThreshold)
return mlx.Add(h, rmsNorm(f, l.PostFFNNorm, cfg.RMSNormEps))
}
func (a *Attention) Forward(x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array {
B, L := int32(x.Dim(0)), int32(x.Dim(1))
q := mlx.Reshape(a.QProj.Forward(x), B, L, cfg.NumAttentionHeads, cfg.HeadDim)
k := mlx.Reshape(a.KProj.Forward(x), B, L, cfg.NumKeyValueHeads, cfg.HeadDim)
v := mlx.Reshape(a.VProj.Forward(blendValues(seed, x, a.ValueResLambda)), B, L, cfg.NumKeyValueHeads, cfg.HeadDim)
q = mlx.Transpose(rmsNorm(q, a.QNorm, cfg.RMSNormEps), 0, 2, 1, 3)
k = mlx.Transpose(rmsNorm(k, a.KNorm, cfg.RMSNormEps), 0, 2, 1, 3)
v = mlx.Transpose(v, 0, 2, 1, 3)
if local {
q, k = applyRotary(q, cos, sin), applyRotary(k, cos, sin)
}
mask := nn.CausalMask()
var kv nn.SDPAOption
if c != nil {
kv = nn.WithKVHistory(c.(cache.Attention).Update(b, k, v))
} else {
kv = nn.WithKV(k, v, b.SeqQueryLens)
if local {
mask = mask.Intersect(nn.SlidingWindowMask(b, int(L), cfg.SlidingWindow, k.DType()))
}
}
out := nn.ScaledDotProductAttention(b, q, cfg.Scale, kv, nn.WithMask(mask))
out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim)
return a.OProj.Forward(gatedAttention(out, a.GProj.Forward(x)))
}
// MLP is the clipped SwiGLU feed-forward block.
type MLP struct {
GateProj, UpProj, DownProj nn.LinearLayer
}
var ffnActivation = mlx.Compile2("SwiGLUF32", func(gate, up *mlx.Array) *mlx.Array {
f := gate.AsType(mlx.DTypeFloat32)
silu := mlx.Mul(f, mlx.Sigmoid(f)).AsType(gate.DType())
return mlx.Mul(silu, up)
}, mlx.Shapeless())
var clippedFFNActivation = mlx.Compile3("ClippedSwiGLUF32", func(gate, up, threshold *mlx.Array) *mlx.Array {
gate = mlx.Minimum(mlx.Maximum(gate, mlx.FromValue(float32(math.Inf(-1)))), threshold).AsType(gate.DType())
up = mlx.Minimum(mlx.Maximum(up, threshold.Negative()), threshold).AsType(up.DType())
return ffnActivation(gate, up)
}, mlx.Shapeless())
func (m *MLP) Forward(x *mlx.Array, threshold *float32) *mlx.Array {
gate, up := m.GateProj.Forward(x), m.UpProj.Forward(x)
var activated *mlx.Array
if threshold != nil && *threshold > 0 {
activated = clippedFFNActivation(gate, up, mlx.FromValue(*threshold))
} else {
activated = ffnActivation(gate, up)
}
return m.DownProj.Forward(activated)
}
func rotaryTables(positions *mlx.Array, dim int32, theta float32, dtype mlx.DType) (*mlx.Array, *mlx.Array) {
inv := make([]float32, dim/2)
for i := range inv {
inv[i] = 1 / float32(math.Pow(float64(theta), float64(float32(2*i)/float32(dim))))
}
freq := mlx.Mul(mlx.ExpandDims(positions.AsType(mlx.DTypeFloat32), -1), mlx.FromValues(inv, len(inv)))
freq = mlx.Concatenate([]*mlx.Array{freq, freq}, -1)
return mlx.Cos(freq).AsType(dtype), mlx.Sin(freq).AsType(dtype)
}
func applyRotary(x, cos, sin *mlx.Array) *mlx.Array {
f := x.AsType(mlx.DTypeFloat32)
half := x.Dim(-1) / 2
first := f.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(0, half))
second := f.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(half, x.Dim(-1)))
rotated := mlx.Concatenate([]*mlx.Array{second.Negative(), first}, -1)
c := mlx.ExpandDims(cos.AsType(mlx.DTypeFloat32), 1)
s := mlx.ExpandDims(sin.AsType(mlx.DTypeFloat32), 1)
return mlx.Add(mlx.Mul(f, c), mlx.Mul(rotated, s)).AsType(x.DType())
}
func rmsNorm(x, weight *mlx.Array, eps float32) *mlx.Array {
// Round the normalized activation before multiplying the learned weight.
return mlx.Mul(mlx.RMSNormFn(x, nil, eps), weight)
}
func scaleEmbedding(x *mlx.Array, scale *float64) *mlx.Array {
if scale == nil {
return x
}
return mlx.MulScalar(x.AsType(mlx.DTypeFloat32), float32(*scale)).AsType(x.DType())
}
func attentionGate(x *mlx.Array) *mlx.Array {
return mlx.Sigmoid(x.AsType(mlx.DTypeFloat32)).AsType(x.DType())
}
var gatedAttention = mlx.Compile2("AttentionOutputGate", func(out, gate *mlx.Array) *mlx.Array {
return mlx.Mul(out, attentionGate(gate))
}, mlx.Shapeless())
func blendValues(seed, x, lambda *mlx.Array) *mlx.Array {
// Sigmoid rounds to the parameter dtype; lerp has one activation rounding.
a := mlx.Sigmoid(lambda.AsType(mlx.DTypeFloat32)).AsType(lambda.DType()).AsType(x.DType()).AsType(mlx.DTypeFloat32)
return interpolateValues(seed, x, a)
}
var interpolateValues = mlx.Compile3("ValueBlend", func(seed, x, a *mlx.Array) *mlx.Array {
start, end := seed.AsType(mlx.DTypeFloat32), x.AsType(mlx.DTypeFloat32)
delta := mlx.Sub(end, start)
low := mlx.Add(start, mlx.Mul(a, delta))
high := mlx.Sub(end, mlx.Mul(mlx.Sub(mlx.FromValue(float32(1)), a), delta))
return mlx.Where(a.Less(mlx.FromValue(float32(0.5))), low, high).AsType(x.DType())
}, mlx.Shapeless())

View file

@ -0,0 +1,318 @@
// Package ripple implements the Ricursive transformer.
package ripple
import (
"encoding/json"
"fmt"
"math"
"slices"
"strings"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlx/quant"
"github.com/ollama/ollama/mlxrunner/cache"
"github.com/ollama/ollama/mlxrunner/nn"
"github.com/ollama/ollama/mlxrunner/tokenizer"
"github.com/ollama/ollama/mlxrunner/model"
)
func init() { model.Register("RicursiveForCausalLM", newModel) }
// Model retains all loaded arrays; value-residual seeds belong to Forward.
type Model struct {
*Config
EmbedTokens nn.EmbeddingLayer
Layers []*Layer
Norm *mlx.Array
LMHead nn.LinearLayer
tok *tokenizer.Tokenizer
auxHiddenLayers []int
draftTensorPrefix string
}
type Layer struct {
Attention *Attention
MLP *MLP
InputNorm, PostAttentionNorm, PreFFNNorm, PostFFNNorm *mlx.Array
}
type Attention struct {
QProj, KProj, VProj, OProj, GProj nn.LinearLayer
QNorm, KNorm, ValueResLambda *mlx.Array
}
func newModel(root *model.Root) (model.Model, error) {
cfg, err := loadConfig(root)
if err != nil {
return nil, err
}
configData, err := root.Manifest.ReadConfig("config.json")
if err != nil {
return nil, err
}
tokData, err := root.Manifest.ReadConfig("tokenizer.json")
if err != nil {
return nil, fmt.Errorf("load tokenizer: %w", err)
}
tc := &tokenizer.TokenizerConfig{ConfigJSON: configData}
tc.GenerationConfigJSON, _ = root.Manifest.ReadConfig("generation_config.json")
tc.TokenizerConfigJSON, _ = root.Manifest.ReadConfig("tokenizer_config.json")
tok, err := tokenizer.LoadFromBytesWithConfig(tokData, tc)
if err != nil {
return nil, fmt.Errorf("parse tokenizer: %w", err)
}
m := &Model{Config: cfg, Layers: make([]*Layer, cfg.NumHiddenLayers), tok: tok}
if root.Draft != nil {
m.draftTensorPrefix = root.Draft.TensorPrefix
if m.draftTensorPrefix == "" {
m.draftTensorPrefix = "draft."
}
}
return m, nil
}
func (m *Model) Tokenizer() *tokenizer.Tokenizer { return m.tok }
func (m *Model) MaxContextLength() int { return int(m.MaxPositionEmbeddings) }
func (m *Model) Unembed(x *mlx.Array) *mlx.Array { return m.LMHead.Forward(x) }
func (m *Model) NumLayers() int { return len(m.Layers) }
func (m *Model) SetAuxHiddenLayers(layers []int) { m.auxHiddenLayers = slices.Clone(layers) }
func (m *Model) TokenEmbeddings(ids *mlx.Array) *mlx.Array { return m.EmbedTokens.Forward(ids) }
func (m *Model) RawLogits(hidden *mlx.Array) *mlx.Array { return m.LMHead.Forward(hidden) }
// DraftWindow bounds draft attention to its training context.
func (*Model) DraftWindow() int { return 16384 }
func (m *Model) NewCaches() []cache.Cache {
out := make([]cache.Cache, len(m.LayerTypes))
for i, kind := range m.LayerTypes {
if kind == "sliding_attention" {
out[i] = cache.NewRotatingKVCache(m.SlidingWindow)
} else {
out[i] = cache.NewKVCache()
}
}
return out
}
func (m *Model) weightShapes() map[string][]int {
h, d, f, v := int(m.HiddenSize), int(m.HeadDim), int(m.IntermediateSize), int(m.VocabSize)
q, k := int(m.NumAttentionHeads)*d, int(m.NumKeyValueHeads)*d
shapes := map[string][]int{"model.embed_tokens.weight": {v, h}, "lm_head.weight": {v, h}, "model.norm.weight": {h}}
for i := range m.NumHiddenLayers {
p := fmt.Sprintf("model.layers.%d.", i)
for name, shape := range map[string][]int{
"input_layernorm.weight": {h}, "post_attention_layernorm.weight": {h}, "pre_feedforward_layernorm.weight": {h}, "post_feedforward_layernorm.weight": {h},
"self_attn.q_proj.weight": {q, h}, "self_attn.k_proj.weight": {k, h}, "self_attn.v_proj.weight": {k, h}, "self_attn.o_proj.weight": {h, q}, "self_attn.g_proj.weight": {q, h},
"self_attn.q_norm.weight": {d}, "self_attn.k_norm.weight": {d}, "self_attn.value_res_lambda": {1},
"mlp.gate_proj.weight": {f, h}, "mlp.up_proj.weight": {f, h}, "mlp.down_proj.weight": {h, f},
} {
shapes[p+name] = shape
}
}
return shapes
}
func floatTensor(a *mlx.Array) bool {
return a.DType() == mlx.DTypeFloat32 || a.DType() == mlx.DTypeBFloat16 || a.DType() == mlx.DTypeFloat16
}
func (m *Model) validateWeights(tensors map[string]*mlx.Array) error {
allowed := make(map[string]bool)
for name, shape := range m.weightShapes() {
w := tensors[name]
if w == nil {
return fmt.Errorf("missing weight: %s", name)
}
allowed[name] = true
scales := tensors[name+"_scale"]
if scales == nil {
if !floatTensor(w) || !slices.Equal(w.Dims(), shape) {
return fmt.Errorf("invalid weight %s: dtype %v shape %v, want float %v", name, w.DType(), w.Dims(), shape)
}
if m.TensorQuant[name] != nil {
return fmt.Errorf("missing scale for quantized weight %s", name)
}
continue
}
if len(shape) != 2 {
return fmt.Errorf("cannot quantize %s", name)
}
if tq := m.TensorQuant[name]; tq != nil && quant.Canonical(tq.QuantType) == "" {
return fmt.Errorf("unsupported quantization for %s: %q", name, tq.QuantType)
}
gs, bits, mode := model.ResolveLinearQuantParams(m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant, name, w, scales)
if gs <= 0 || (bits != 4 && bits != 8) || (mode != "affine" && mode != "nvfp4" && mode != "mxfp4" && mode != "mxfp8") {
return fmt.Errorf("invalid quantization for %s", name)
}
if (mode == "nvfp4" && (gs != 16 || bits != 4)) || (mode == "mxfp4" && (gs != 32 || bits != 4)) || (mode == "mxfp8" && (gs != 32 || bits != 8)) {
return fmt.Errorf("invalid quantization group/bits for %s", name)
}
packed, ss := []int{shape[0], shape[1] * bits / 32}, []int{shape[0], shape[1] / gs}
if shape[1]%gs != 0 || shape[1]*bits%32 != 0 || w.DType() != mlx.DTypeUint32 || !slices.Equal(w.Dims(), packed) || !slices.Equal(scales.Dims(), ss) {
return fmt.Errorf("invalid quantized shape for %s: weight %v scale %v, want %v %v", name, w.Dims(), scales.Dims(), packed, ss)
}
allowed[name+"_scale"] = true
if mode == "affine" {
qb := tensors[name+"_qbias"]
if !floatTensor(scales) || qb == nil || !floatTensor(qb) || !slices.Equal(qb.Dims(), ss) {
return fmt.Errorf("invalid affine scale/bias for %s", name)
}
allowed[name+"_qbias"] = true
} else if scales.DType() != mlx.DTypeUint8 {
return fmt.Errorf("invalid packed scale dtype for %s", name)
}
for _, suffix := range []string{".global_scale", "_scale_2", ".input_scale", ".input_global_scale"} {
if a := tensors[name+suffix]; a != nil {
if mode != "nvfp4" || !floatTensor(a) || a.Size() != 1 {
return fmt.Errorf("invalid %s%s", name, suffix)
}
allowed[name+suffix] = true
}
}
}
for name := range tensors {
if !allowed[name] && !(m.draftTensorPrefix != "" && strings.HasPrefix(name, m.draftTensorPrefix)) {
return fmt.Errorf("unexpected weight: %s", name)
}
}
return nil
}
func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {
if err := m.validateWeights(tensors); err != nil {
return err
}
linears := model.NewLinearFactory(tensors, m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant)
m.EmbedTokens = model.MakeEmbeddingLayer(tensors, "model.embed_tokens", m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant)
m.LMHead = linears.Make("lm_head")
m.Norm = tensors["model.norm.weight"]
for i := range m.Layers {
p := fmt.Sprintf("model.layers.%d.", i)
m.Layers[i] = &Layer{
InputNorm: tensors[p+"input_layernorm.weight"], PostAttentionNorm: tensors[p+"post_attention_layernorm.weight"], PreFFNNorm: tensors[p+"pre_feedforward_layernorm.weight"], PostFFNNorm: tensors[p+"post_feedforward_layernorm.weight"],
Attention: &Attention{QProj: linears.Make(p + "self_attn.q_proj"), KProj: linears.Make(p + "self_attn.k_proj"), VProj: linears.Make(p + "self_attn.v_proj"), OProj: linears.Make(p + "self_attn.o_proj"), GProj: linears.Make(p + "self_attn.g_proj"), QNorm: tensors[p+"self_attn.q_norm.weight"], KNorm: tensors[p+"self_attn.k_norm.weight"], ValueResLambda: tensors[p+"self_attn.value_res_lambda"]},
MLP: &MLP{GateProj: linears.Make(p + "mlp.gate_proj"), UpProj: linears.Make(p + "mlp.up_proj"), DownProj: linears.Make(p + "mlp.down_proj")},
}
}
return nil
}
type Config struct {
HiddenSize int32 `json:"hidden_size"`
IntermediateSize int32 `json:"intermediate_size"`
NumHiddenLayers int32 `json:"num_hidden_layers"`
NumAttentionHeads int32 `json:"num_attention_heads"`
NumKeyValueHeads int32 `json:"num_key_value_heads"`
HeadDim int32 `json:"head_dim"`
VocabSize int32 `json:"vocab_size"`
MaxPositionEmbeddings int32 `json:"max_position_embeddings"`
RMSNormEps float32 `json:"rms_norm_eps"`
RopeTheta float32 `json:"rope_theta"`
SlidingWindow int `json:"sliding_window"`
LayerTypes []string `json:"layer_types"`
HiddenAct string `json:"hidden_act"`
UseGatedAttention bool `json:"use_gated_attention"`
GateFullPrecision bool `json:"gate_full_precision"`
UseValRes bool `json:"use_valres"`
TieWordEmbeddings bool `json:"tie_word_embeddings"`
AttentionBias bool `json:"attention_bias"`
MLPBias bool `json:"mlp_bias"`
FFNClampThreshold *float32 `json:"ffn_clamp_threshold"`
EmbedScale *float64 `json:"embed_scale"`
RopeScaling json.RawMessage `json:"rope_scaling"`
RopeParameters json.RawMessage `json:"rope_parameters"`
QuantGroupSize int `json:"-"`
QuantBits int `json:"-"`
QuantMode string `json:"-"`
TensorQuant map[string]*model.TensorQuantInfo `json:"-"`
Scale float32 `json:"-"`
}
func loadConfig(root *model.Root) (*Config, error) {
data, err := root.Manifest.ReadConfig("config.json")
if err != nil {
return nil, fmt.Errorf("load config: %w", err)
}
cfg := &Config{HiddenAct: "silu", RMSNormEps: 1e-6, RopeTheta: 10000, SlidingWindow: 1024, UseGatedAttention: true, GateFullPrecision: true}
if err := json.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("parse config: %w", err)
}
for _, v := range []struct {
name string
value int32
}{
{"hidden_size", cfg.HiddenSize},
{"intermediate_size", cfg.IntermediateSize},
{"num_hidden_layers", cfg.NumHiddenLayers},
{"num_attention_heads", cfg.NumAttentionHeads},
{"num_key_value_heads", cfg.NumKeyValueHeads},
{"head_dim", cfg.HeadDim},
{"vocab_size", cfg.VocabSize},
{"max_position_embeddings", cfg.MaxPositionEmbeddings},
} {
if v.value <= 0 {
return nil, fmt.Errorf("invalid %s: %d", v.name, v.value)
}
}
if cfg.HeadDim%2 != 0 {
return nil, fmt.Errorf("head_dim must be even")
}
if cfg.NumAttentionHeads%cfg.NumKeyValueHeads != 0 {
return nil, fmt.Errorf("num_attention_heads must be divisible by num_key_value_heads")
}
if int64(cfg.NumAttentionHeads)*int64(cfg.HeadDim) > math.MaxInt32 {
return nil, fmt.Errorf("num_attention_heads * head_dim overflows")
}
if cfg.RMSNormEps <= 0 {
return nil, fmt.Errorf("rms_norm_eps must be positive")
}
if cfg.RopeTheta <= 0 {
return nil, fmt.Errorf("rope_theta must be positive")
}
if cfg.SlidingWindow <= 0 {
return nil, fmt.Errorf("sliding_window must be positive")
}
if len(cfg.LayerTypes) != int(cfg.NumHiddenLayers) {
return nil, fmt.Errorf("layer_types must have num_hidden_layers entries")
}
for i, kind := range cfg.LayerTypes {
if kind != "sliding_attention" && kind != "full_attention" {
return nil, fmt.Errorf("unsupported layer_types[%d]: %q", i, kind)
}
}
if cfg.HiddenAct != "silu" {
return nil, fmt.Errorf("unsupported hidden_act: %q", cfg.HiddenAct)
}
for _, v := range []struct {
name string
unsupported bool
}{
{"use_gated_attention", !cfg.UseGatedAttention},
{"gate_full_precision", !cfg.GateFullPrecision},
{"use_valres", !cfg.UseValRes},
{"tie_word_embeddings", cfg.TieWordEmbeddings},
{"attention_bias", cfg.AttentionBias},
{"mlp_bias", cfg.MLPBias},
{"rope_scaling", len(cfg.RopeScaling) > 0 && string(cfg.RopeScaling) != "null"},
{"rope_parameters", len(cfg.RopeParameters) > 0 && string(cfg.RopeParameters) != "null"},
} {
if v.unsupported {
return nil, fmt.Errorf("unsupported %s", v.name)
}
}
if cfg.FFNClampThreshold != nil && *cfg.FFNClampThreshold < 0 {
return nil, fmt.Errorf("ffn_clamp_threshold must be nonnegative")
}
if cfg.EmbedScale != nil && (math.IsInf(*cfg.EmbedScale, 0) || math.IsNaN(*cfg.EmbedScale) || math.Abs(*cfg.EmbedScale) > math.MaxFloat32) {
return nil, fmt.Errorf("invalid embed_scale")
}
cfg.Scale = float32(1 / math.Sqrt(float64(cfg.HeadDim)))
cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode = model.QuantizationParams(root.QuantType())
if gs := root.GroupSize(); gs > 0 {
cfg.QuantGroupSize = gs
}
cfg.TensorQuant = root.AllTensorQuant()
return cfg, nil
}

File diff suppressed because it is too large Load diff

681
mlxrunner/ripple_test.go Normal file
View file

@ -0,0 +1,681 @@
package mlxrunner
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/llm"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlx/mlxtest"
"github.com/ollama/ollama/mlxrunner/batch"
"github.com/ollama/ollama/mlxrunner/cache"
"github.com/ollama/ollama/mlxrunner/model"
"github.com/ollama/ollama/mlxrunner/model/ripple"
"github.com/ollama/ollama/mlxrunner/sample"
modeltypes "github.com/ollama/ollama/types/model"
)
type rippleRunnerTensor struct {
data []float32
shape []int
}
func rippleRunnerFixture() (map[string]any, map[string]rippleRunnerTensor) {
cfg := map[string]any{
"architectures": []string{"RicursiveForCausalLM"},
"vocab_size": 19, "hidden_size": 8, "intermediate_size": 12,
"num_hidden_layers": 3, "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 4,
"layer_types": []string{"sliding_attention", "full_attention", "sliding_attention"},
"sliding_window": 3, "rms_norm_eps": 1e-6, "rope_theta": 10000.0,
"use_valres": true, "use_gated_attention": true, "gate_full_precision": true,
"ffn_clamp_threshold": 20.0, "embed_scale": 71.55417527999327,
"max_position_embeddings": 64, "tie_word_embeddings": false,
}
shapes := map[string][]int{
"model.embed_tokens.weight": {19, 8}, "lm_head.weight": {19, 8}, "model.norm.weight": {8},
}
for i := range 3 {
prefix := fmt.Sprintf("model.layers.%d.", i)
for name, shape := range map[string][]int{
"input_layernorm.weight": {8}, "post_attention_layernorm.weight": {8},
"pre_feedforward_layernorm.weight": {8}, "post_feedforward_layernorm.weight": {8},
"self_attn.q_proj.weight": {16, 8}, "self_attn.k_proj.weight": {8, 8},
"self_attn.v_proj.weight": {8, 8}, "self_attn.o_proj.weight": {8, 16},
"self_attn.g_proj.weight": {16, 8}, "self_attn.q_norm.weight": {4},
"self_attn.k_norm.weight": {4}, "self_attn.value_res_lambda": {1},
"mlp.gate_proj.weight": {12, 8}, "mlp.up_proj.weight": {12, 8}, "mlp.down_proj.weight": {8, 12},
} {
shapes[prefix+name] = shape
}
}
weights := make(map[string]rippleRunnerTensor)
for name, shape := range shapes {
size := 1
for _, dim := range shape {
size *= dim
}
values := make([]float32, size)
hash := sha256.Sum256([]byte(name))
phase := int(binary.BigEndian.Uint32(hash[:4]) % 101)
if strings.HasSuffix(name, "value_res_lambda") {
layer := strings.Split(name, ".")[2]
values[0] = map[string]float32{"0": -1.25, "1": 0.75, "2": 1.5}[layer]
} else {
for i := range values {
v := float32((i*37+phase)%101-50) / 50
switch {
case strings.Contains(name, "norm.weight"):
scaled := v * 0.375
v = 1 + scaled
case strings.Contains(name, "embed_tokens"):
v *= 0.045
case strings.Contains(name, ".mlp.gate_proj") || strings.Contains(name, ".mlp.up_proj"):
v *= 18
default:
v *= 0.3
}
values[i] = v
}
}
weights[name] = rippleRunnerTensor{data: values, shape: shape}
}
return cfg, weights
}
func writeRippleRunnerModel(t *testing.T) string {
t.Helper()
mlxtest.SkipIfUnavailable(t)
dir := t.TempDir()
t.Setenv("OLLAMA_MODELS", dir)
fixtureConfig, fixtureWeights := rippleRunnerFixture()
config, err := json.Marshal(fixtureConfig)
if err != nil {
t.Fatal(err)
}
vocab := make(map[string]int)
for i := range fixtureConfig["vocab_size"].(int) {
vocab[string(rune('a'+i))] = i
}
tokenizer, err := json.Marshal(map[string]any{
"model": map[string]any{"type": "BPE", "vocab": vocab, "merges": []string{}},
})
if err != nil {
t.Fatal(err)
}
name := "ripple-runner-test"
mlxtest.Run(t, func(t *mlxtest.T) {
mlx.Scoped(func() {
weights := make(map[string]*mlx.Array)
for name, tensor := range fixtureWeights {
weights[name] = mlx.FromValues(tensor.data, tensor.shape...)
}
path := filepath.Join(dir, "model.safetensors")
if err := mlx.SaveSafetensors(path, weights); err != nil {
t.Fatal(err)
}
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
tensors, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor)
if err != nil {
t.Fatal(err)
}
tensors.Name = "model.safetensors"
layers := []manifest.Layer{tensors}
for name, data := range map[string][]byte{"config.json": config, "tokenizer.json": tokenizer} {
layer, err := manifest.NewLayer(bytes.NewReader(data), manifest.MediaTypeImageJSON)
if err != nil {
t.Fatal(err)
}
layer.Name = name
layers = append(layers, layer)
}
meta, err := manifest.NewLayer(strings.NewReader(`{"model_format":"safetensors","model_family":"ripple"}`), manifest.MediaTypeImageConfig)
if err != nil {
t.Fatal(err)
}
if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), meta, layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil {
t.Fatal(err)
}
})
})
return name
}
func loadRippleRunner(t *testing.T, name string) *Runner {
t.Helper()
r := &Runner{mlxThread: mlxtest.Worker(t)}
mlxtest.Run(t, func(t *mlxtest.T) {
if err := r.Load(name); err != nil {
t.Fatal(err)
}
if _, ok := r.Model.(*ripple.Model); !ok {
t.Fatalf("loaded %T, want *ripple.Model", r.Model)
}
})
t.Cleanup(func() {
mlxtest.Run(t, func(t *mlxtest.T) {
r.cache.close()
r.Close()
})
})
return r
}
func rippleRunnerMemory(t *testing.T) (active int) {
t.Helper()
mlxtest.Run(t, func(t *mlxtest.T) {
mlx.DefaultStream().Synchronize()
mlx.ClearCache()
active = mlx.ActiveMemory()
})
return active
}
func rippleRunnerRequest(t *testing.T, r *Runner, prompt string, count int) Request {
t.Helper()
request := Request{
CompletionRequest: CompletionRequest{Prompt: prompt, Options: api.Options{NumPredict: count}},
Pipeline: r.TextGenerationPipeline,
SamplerOpts: sample.Options{
Temperature: 0, RepeatLastN: 8, RepeatPenalty: 1.1,
Seed: 1, UseSeed: true, Logprobs: true, TopLogprobs: 3,
},
Responses: make(chan CompletionResponse, count+1),
}
if err := r.Prepare(&request); err != nil {
t.Fatal(err)
}
if len(request.Tokens) != len(prompt) {
t.Fatalf("tokenized %q into %d tokens, want %d", prompt, len(request.Tokens), len(prompt))
}
return request
}
func runRippleRunnerRequest(t *testing.T, r *Runner, prompt string) (string, []llm.Logprob, int) {
t.Helper()
request := rippleRunnerRequest(t, r, prompt, 4)
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
defer cancel()
request.Ctx = ctx
if err := r.runRequest(request); err != nil {
t.Fatal(err)
}
close(request.Responses)
var text strings.Builder
var logprobs []llm.Logprob
var final *CompletionResponse
for response := range request.Responses {
if response.Error != nil {
t.Fatal(response.Error)
}
if response.Done {
if final != nil {
t.Fatal("duplicate final response")
}
final = &response
}
text.WriteString(response.Content)
logprobs = append(logprobs, response.Logprobs...)
}
if final == nil || final.EvalCount != 4 || final.DoneReason != 1 || final.PromptEvalCount != len(request.Tokens) || final.PromptEvalCachedCount == nil {
t.Fatalf("invalid final response: %+v", final)
}
if text.Len() != 4 || len(logprobs) != 4 {
t.Fatalf("empty or incomplete model output: text=%q logprobs=%d", text.String(), len(logprobs))
}
for _, lp := range logprobs {
if math.IsNaN(lp.Logprob) || math.IsInf(lp.Logprob, 0) || lp.Logprob >= 0 || len(lp.TopLogprobs) != 3 {
t.Fatalf("invalid model log probability: %+v", lp)
}
if lp.TopLogprobs[0].Logprob == lp.TopLogprobs[2].Logprob {
t.Fatalf("model produced uniform logits: %+v", lp)
}
}
return text.String(), logprobs, *final.PromptEvalCachedCount
}
func TestRippleRunnerLoadClose(t *testing.T) {
name := writeRippleRunnerModel(t)
warm := loadRippleRunner(t, name)
mlxtest.Run(t, func(t *mlxtest.T) { warm.Close() })
baseline := rippleRunnerMemory(t)
for range 3 {
r := loadRippleRunner(t, name)
if loaded := rippleRunnerMemory(t); loaded <= baseline {
t.Fatalf("load allocated no weights: loaded=%d baseline=%d", loaded, baseline)
}
mlxtest.Run(t, func(t *mlxtest.T) { r.Close() })
if got := rippleRunnerMemory(t); got != baseline {
t.Fatalf("load/close retained %d bytes above warmed baseline %d", got-baseline, baseline)
}
}
}
func TestRippleRunnerRequests(t *testing.T) {
name := writeRippleRunnerModel(t)
cold, reused := loadRippleRunner(t, name), loadRippleRunner(t, name)
const prompt = "ihgfedcba"
wantText, wantLogprobs, cached := runRippleRunnerRequest(t, cold, prompt)
if cached != 0 {
t.Fatalf("fresh runner reused %d tokens", cached)
}
runRippleRunnerRequest(t, reused, "abcdefghi")
gotText, gotLogprobs, cached := runRippleRunnerRequest(t, reused, prompt)
// These cold forwards have identical chunk schedules; cache-hit forwards do not.
if cached != 0 || gotText != wantText || !reflect.DeepEqual(gotLogprobs, wantLogprobs) {
t.Fatalf("independent request inherited state: cached=%d text=%q want=%q", cached, gotText, wantText)
}
wantText, wantLogprobs, cached = runRippleRunnerRequest(t, reused, prompt)
if cached <= 0 || cached >= len(prompt) {
t.Fatalf("repeat reused %d tokens, want a nonempty prefix with a decode seed", cached)
}
baseline := rippleRunnerMemory(t)
// Settled allocator readings can vary by a few small buffers between requests.
const memorySlack = 256
for range 16 {
gotText, gotLogprobs, gotCached := runRippleRunnerRequest(t, reused, prompt)
if gotCached != cached || gotText != wantText || !reflect.DeepEqual(gotLogprobs, wantLogprobs) {
t.Fatal("identical cache-hit schedule changed the response")
}
if got := rippleRunnerMemory(t); got > baseline+memorySlack {
t.Fatalf("repeated request retained %d bytes above warmed baseline %d", got-baseline, baseline)
}
}
request := rippleRunnerRequest(t, reused, prompt, 32)
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
defer cancel()
request.Ctx = ctx
request.Responses = make(chan CompletionResponse)
responses := make(chan []CompletionResponse, 1)
go func() {
var received []CompletionResponse
for response := range request.Responses {
received = append(received, response)
cancel()
}
responses <- received
}()
err := reused.runRequest(request)
close(request.Responses)
received := <-responses
if !errors.Is(err, context.Canceled) || len(received) == 0 || received[0].Content == "" {
t.Fatalf("cancel after real output: err=%v responses=%+v", err, received)
}
for _, response := range received {
if response.Done {
t.Fatal("cancelled generation emitted a successful final response")
}
}
// Retry through the same worker, sampler, and prefix-cache owner.
runRippleRunnerRequest(t, reused, prompt)
runRippleRunnerRequest(t, reused, prompt)
baseline = rippleRunnerMemory(t)
runRippleRunnerRequest(t, reused, prompt)
if got := rippleRunnerMemory(t); got > baseline+memorySlack {
t.Fatalf("post-cancellation request retained %d bytes", got-baseline)
}
}
func writeRippleDFlash2RunnerModel(t *testing.T) string {
t.Helper()
name := writeRippleRunnerModel(t)
m, err := manifest.ParseNamedManifest(modeltypes.ParseName(name))
if err != nil {
t.Fatal(err)
}
const draftConfig = `{
"architectures": ["DFlash2DraftModel"],
"hidden_size": 8, "num_hidden_layers": 2, "intermediate_size": 16,
"num_attention_heads": 2, "num_key_value_heads": 1, "head_dim": 4,
"rms_norm_eps": 0.000001, "rope_theta": 10000,
"vocab_size": 19, "num_target_layers": 3, "block_size": 8,
"layer_types": ["sliding_attention", "full_attention"],
"sliding_window": 4, "no_rope_layers": [1], "is_causal": false,
"dflash_config": {
"mask_token_id": 18, "target_layer_ids": [0, 2],
"conv_group_size": 2, "conv_kernel_size": 4,
"selector_rank": 4, "selector_top_k": 4
}
}`
config, err := manifest.NewLayer(strings.NewReader(draftConfig), manifest.MediaTypeImageJSON)
if err != nil {
t.Fatal(err)
}
config.Name = "draft/config.json"
layers := append(m.Layers, config)
path := filepath.Join(t.TempDir(), "draft.safetensors")
mlxtest.Run(t, func(t *mlxtest.T) {
mlx.Scoped(func() {
weights := make(map[string]*mlx.Array)
add := func(name string, dims ...int) {
n, phase := 1, 0
for _, dim := range dims {
n *= dim
}
for _, c := range name {
phase += int(c)
}
values := make([]float32, n)
for i := range values {
values[i] = float32((i*17+phase)%31-15) / 128
if len(dims) == 1 {
values[i] = 1
}
}
weights["draft."+name] = mlx.FromValues(values, dims...)
}
add("fc.weight", 8, 2*8)
add("hidden_norm.weight", 8)
add("norm.weight", 8)
for i := range 2 {
prefix := fmt.Sprintf("layers.%d.", i)
add(prefix+"input_layernorm.weight", 8)
add(prefix+"post_attention_layernorm.weight", 8)
add(prefix+"self_attn.q_norm.weight", 4)
add(prefix+"self_attn.k_norm.weight", 4)
add(prefix+"self_attn.q_proj.weight", 8, 8)
add(prefix+"self_attn.k_proj.weight", 4, 8)
add(prefix+"self_attn.v_proj.weight", 4, 8)
add(prefix+"self_attn.o_proj.weight", 8, 8)
add(prefix+"mlp.gate_proj.weight", 16, 8)
add(prefix+"mlp.up_proj.weight", 16, 8)
add(prefix+"mlp.down_proj.weight", 8, 16)
for _, conv := range []string{"attention_conv", "mlp_conv"} {
add(prefix+conv+".base_kernel", 2, 4, 8)
add(prefix+conv+".kernel_projection.weight", 2*4*8/2, 8)
}
}
add("candidate_selector.predecessor_codebook", 19, 4)
add("candidate_selector.successor_codebook", 19, 4)
add("candidate_selector.hidden_projection.weight", 4, 8)
if err := mlx.SaveSafetensors(path, weights); err != nil {
t.Fatal(err)
}
})
})
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
tensors, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor)
if err != nil {
t.Fatal(err)
}
tensors.Name = "draft/model.safetensors"
layers = append(layers, tensors)
metadata, err := json.Marshal(modeltypes.ConfigV2{
ModelFormat: "safetensors", ModelFamily: "ripple",
Draft: &modeltypes.Draft{
ModelFormat: "safetensors", Architecture: "DFlash2DraftModel",
TensorPrefix: "draft.", Config: "draft/config.json",
},
})
if err != nil {
t.Fatal(err)
}
meta, err := manifest.NewLayer(bytes.NewReader(metadata), manifest.MediaTypeImageConfig)
if err != nil {
t.Fatal(err)
}
name = "ripple-dflash2-runner-test"
if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), meta, layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil {
t.Fatal(err)
}
return name
}
type rippleDraftForward struct {
offset, contextRows, featureWidth, blockRows int
}
// Observe shapes and calls only; every native operand and selector stays real.
type rippleDFlash2Observer struct {
model.BlockSelector
forwards []rippleDraftForward
selections int
}
func (d *rippleDFlash2Observer) Forward(b *batch.Batch, targets, drafts []cache.Cache) (*mlx.Array, *mlx.Array) {
call := rippleDraftForward{offset: int(b.SeqOffsets[0])}
if b.Hidden != nil {
call.contextRows, call.featureWidth = b.Hidden.Dim(1), b.Hidden.Dim(2)
}
if b.InputIDs != nil {
call.blockRows = b.InputIDs.Dim(1)
}
d.forwards = append(d.forwards, call)
return d.BlockSelector.Forward(b, targets, drafts)
}
func (d *rippleDFlash2Observer) Select(hidden, logits, anchor *mlx.Array, choose func(ids, scores *mlx.Array) (*mlx.Array, *mlx.Array)) (*mlx.Array, *mlx.Array, *mlx.Array) {
d.selections++
return d.BlockSelector.Select(hidden, logits, anchor, choose)
}
func loadRippleDFlash2Runner(t *testing.T, name string) (*Runner, *rippleDFlash2Observer) {
t.Helper()
r := loadRippleRunner(t, name)
if r.spec == nil {
t.Fatal("combined manifest did not attach a draft")
}
draft, ok := r.spec.draft.(model.BlockSelector)
if !ok {
t.Fatalf("loaded draft %T, want DFlash2 BlockSelector", r.spec.draft)
}
if block, mask := draft.BlockParams(); block != 8 || mask != 18 {
t.Fatalf("draft block=%d mask=%d, want 8 and 18", block, mask)
}
if len(r.spec.targets) != 3 || len(r.spec.draftKV) != 2 || len(r.cache.caches) != 5 {
t.Fatalf("cache partitions: target=%d draft=%d combined=%d", len(r.spec.targets), len(r.spec.draftKV), len(r.cache.caches))
}
if _, ok := r.spec.draftKV[0].(*cache.RotatingKVCache); !ok {
t.Fatalf("local draft cache is %T", r.spec.draftKV[0])
}
if _, ok := r.spec.draftKV[1].(*cache.RotatingKVCache); !ok {
t.Fatalf("full draft cache is %T", r.spec.draftKV[1])
}
observer := &rippleDFlash2Observer{BlockSelector: draft}
r.spec.draft = observer
return r, observer
}
func checkRippleDFlash2Caches(t *testing.T, r *Runner, prompt string) {
t.Helper()
mlxtest.Run(t, func(t *mlxtest.T) {
offset := r.spec.targets[0].Offset()
if offset < len(prompt) {
t.Fatalf("target stopped before prompt completion: offset=%d prompt=%d", offset, len(prompt))
}
for i, c := range r.cache.caches {
if c.Offset() != offset {
t.Fatalf("cache %d offset=%d, want settled target offset=%d", i, c.Offset(), offset)
}
}
})
}
type rippleDFlash2Response struct {
text string
cached int
forwards []rippleDraftForward
}
func runRippleDFlash2Request(t *testing.T, r *Runner, prompt string, count int) rippleDFlash2Response {
t.Helper()
d := r.spec.draft.(*rippleDFlash2Observer)
d.forwards, d.selections = nil, 0
request := rippleRunnerRequest(t, r, prompt, count)
// The shared helper requests logprobs, which would silently park drafting.
request.SamplerOpts.Logprobs, request.SamplerOpts.TopLogprobs = false, 0
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
defer cancel()
request.Ctx = ctx
if err := r.runRequest(request); err != nil {
t.Fatal(err)
}
close(request.Responses)
var text strings.Builder
var final *CompletionResponse
for response := range request.Responses {
if response.Error != nil || len(response.Logprobs) != 0 {
t.Fatalf("unexpected active response: %+v", response)
}
text.WriteString(response.Content)
if response.Done {
if final != nil {
t.Fatal("duplicate final response")
}
final = &response
}
}
if final == nil || final.EvalCount != count || final.DoneReason != 1 || final.PromptEvalCount != len(request.Tokens) || final.PromptEvalCachedCount == nil || text.Len() != count {
t.Fatalf("invalid completion: text=%q final=%+v", text.String(), final)
}
proposals := 0
for _, call := range d.forwards {
if call.contextRows > 0 && call.featureWidth != 16 {
t.Fatalf("draft did not receive both target taps: %+v", call)
}
if call.blockRows > 0 {
proposals++
if call.blockRows < 2 || call.blockRows > min(8, count) {
t.Fatalf("proposal exceeded block/budget: %+v budget=%d", call, count)
}
}
}
if proposals != d.selections || (count > 1 && proposals == 0) || (count == 1 && proposals != 0) {
t.Fatalf("active budget=%d: draft forwards=%d selector calls=%d", count, proposals, d.selections)
}
checkRippleDFlash2Caches(t, r, prompt)
return rippleDFlash2Response{text: text.String(), cached: *final.PromptEvalCachedCount, forwards: d.forwards}
}
func TestRippleRunnerDFlash2(t *testing.T) {
name := writeRippleDFlash2RunnerModel(t)
const prompt = "abcdefghi"
// New wiring: normal combined load, target prefill with trailing draft
// caches, target taps, and a real convolution/selector proposal.
t.Run("combined_manifest_prefill_proposal", func(t *testing.T) {
r, d := loadRippleDFlash2Runner(t, name)
got := runRippleDFlash2Request(t, r, prompt, 9)
if got.cached != 0 || len(d.forwards) < 2 {
t.Fatalf("cold request: cached=%d forwards=%+v", got.cached, d.forwards)
}
if want := (rippleDraftForward{contextRows: len(prompt) - 1, featureWidth: 16}); d.forwards[0] != want {
t.Fatalf("prefill handoff=%+v, want %+v", d.forwards[0], want)
}
if want := (rippleDraftForward{offset: len(prompt) - 1, blockRows: 8}); d.forwards[1] != want {
t.Fatalf("first proposal=%+v, want %+v", d.forwards[1], want)
}
})
// Existing runner lifecycle, characterized with the new target/draft pair.
// Compare identities only after matching cache-hit and proposal schedules.
t.Run("prefix_branch_reuse", func(t *testing.T) {
r, _ := loadRippleDFlash2Runner(t, name)
runRippleDFlash2Request(t, r, prompt, 9)
runRippleDFlash2Request(t, r, prompt, 9) // warm the repeat schedule
want := runRippleDFlash2Request(t, r, prompt, 9)
if want.cached <= 0 || want.cached >= len(prompt) {
t.Fatalf("repeat cached=%d, want prefix plus an unforwarded seed", want.cached)
}
if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
t.Fatalf("matched repeat changed: got=%+v want=%+v", got, want)
}
const branch = "abcdejklm"
if got := runRippleDFlash2Request(t, r, branch, 9); got.cached != 5 {
t.Fatalf("branch cached=%d, want shared five-token snapshot", got.cached)
}
if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
t.Fatalf("returning from branch changed matched schedule: got=%+v want=%+v", got, want)
}
})
t.Run("parked_to_enabled", func(t *testing.T) {
r, d := loadRippleDFlash2Runner(t, name)
runRippleRunnerRequest(t, r, prompt)
if len(d.forwards) == 0 || d.selections != 0 {
t.Fatalf("parked maintenance: forwards=%+v selections=%d", d.forwards, d.selections)
}
for _, call := range d.forwards {
if call.blockRows != 0 || call.contextRows == 0 || call.featureWidth != 16 {
t.Fatalf("logprobs must maintain context without proposals: %+v", call)
}
}
checkRippleDFlash2Caches(t, r, prompt)
if got := runRippleDFlash2Request(t, r, prompt, 9); got.cached <= 0 {
t.Fatal("enabled request did not reuse parked context")
}
runRippleDFlash2Request(t, r, prompt, 9)
want := runRippleDFlash2Request(t, r, prompt, 9)
if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
t.Fatalf("enabled repeats changed: got=%+v want=%+v", got, want)
}
})
t.Run("budget_tail", func(t *testing.T) {
r, d := loadRippleDFlash2Runner(t, name)
runRippleDFlash2Request(t, r, prompt, 3)
if len(d.forwards) < 2 || d.forwards[1].blockRows != 3 {
t.Fatalf("shorter-than-convolution proposal: %+v", d.forwards)
}
runRippleDFlash2Request(t, r, prompt, 1)
})
t.Run("cancellation_recovery", func(t *testing.T) {
r, d := loadRippleDFlash2Runner(t, name)
request := rippleRunnerRequest(t, r, prompt, 16)
request.SamplerOpts.Logprobs, request.SamplerOpts.TopLogprobs = false, 0
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
defer cancel()
request.Ctx = ctx
request.Responses = make(chan CompletionResponse)
responses := make(chan []CompletionResponse, 1)
go func() {
var received []CompletionResponse
for response := range request.Responses {
received = append(received, response)
cancel()
}
responses <- received
}()
err := r.runRequest(request)
close(request.Responses)
received := <-responses
if !errors.Is(err, context.Canceled) || len(received) == 0 || received[0].Content == "" || d.selections == 0 {
t.Fatalf("cancel after real speculative output: err=%v responses=%+v selections=%d", err, received, d.selections)
}
for _, response := range received {
if response.Done || response.Error != nil {
t.Fatalf("cancelled generation emitted terminal response: %+v", response)
}
}
checkRippleDFlash2Caches(t, r, prompt)
runRippleDFlash2Request(t, r, prompt, 9)
runRippleDFlash2Request(t, r, prompt, 9)
want := runRippleDFlash2Request(t, r, prompt, 9)
if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
t.Fatalf("recovery repeats changed: got=%+v want=%+v", got, want)
}
})
}

View file

@ -1,194 +0,0 @@
# /// script
# requires-python = ">=3.12,<3.13"
# dependencies = ["torch==2.6.0", "transformers==5.12.1"]
# ///
import argparse
import json
from pathlib import Path
import torch
import transformers
from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
from transformers.models.qwen3_5.modeling_qwen3_5 import (
Qwen3_5Attention,
Qwen3_5TextRotaryEmbedding,
apply_rotary_pos_emb,
)
def generate():
torch.set_num_threads(1)
defaults = {
"rope_type": "default",
"rope_theta": 10000000.0,
"partial_rotary_factor": 0.25,
"mrope_section": [11, 11, 10],
}
yarn = {
**defaults,
"rope_type": "yarn",
"factor": 4.0,
"original_max_position_embeddings": 262144,
}
cases = [
("default", 262144, defaults, "float32"),
("no_metadata", 262144, None, "float32"),
("yarn4", 1048576, yarn, "float32"),
("unextended_limit", 262144, yarn, "float32"),
(
"implicit_original",
262144,
{k: v for k, v in yarn.items() if k != "original_max_position_embeddings"},
"float32",
),
("beta", 1048576, {**yarn, "beta_fast": 16.0, "beta_slow": 2.0}, "float32"),
("fractional_range", 1048576, {**yarn, "truncate": False}, "float32"),
("mscale", 1048576, {**yarn, "mscale": 2.0, "mscale_all_dim": 1.0}, "float32"),
("attention_override", 1048576, {**yarn, "attention_factor": 1.2}, "float32"),
("zero_attention", 1048576, {**yarn, "attention_factor": 0.0}, "float32"),
("factor_one", 262144, {**yarn, "factor": 1.0}, "float32"),
(
"factor_one_attention",
262144,
{**yarn, "factor": 1.0, "attention_factor": 1.2},
"float32",
),
("theta", 1048576, {**yarn, "rope_theta": 500000.0}, "float32"),
(
"half_rotary",
1048576,
{**yarn, "partial_rotary_factor": 0.5, "mrope_section": [22, 21, 21]},
"float32",
),
("sections", 1048576, {**yarn, "mrope_section": [24, 4, 4]}, "float32"),
("default_bfloat16", 262144, defaults, "bfloat16"),
("yarn_bfloat16", 1048576, yarn, "bfloat16"),
]
sample_tokens = [0, 1, 7, 15]
sample_dims = [0, 7, 15, 16, 23, 31, 32, 39, 47, 48, 55, 63, 64, 95, 127, 128, 255]
indices = torch.arange(16)
text_positions = indices.reshape(1, -1)
media_positions = torch.stack(
(32768 + 2 * indices, 7 + indices % 4, 11 + indices // 4)
).unsqueeze(1)
media_positions[:, 0, -1] = media_positions[:, :, :-1].max() + 1
mask = torch.full((1, 1, 16, 16), float("-inf")).triu(1)
results = []
for name, limit, params, dtype_name in cases:
raw_config = {
"model_type": "qwen3_5_text",
"hidden_size": 256,
"num_hidden_layers": 1,
"num_attention_heads": 1,
"num_key_value_heads": 1,
"head_dim": 256,
"intermediate_size": 256,
"rms_norm_eps": 1e-6,
"linear_num_key_heads": 1,
"linear_num_value_heads": 1,
"linear_key_head_dim": 128,
"linear_value_head_dim": 128,
"layer_types": ["full_attention"],
"max_position_embeddings": limit,
"rope_theta": 10000000.0,
"partial_rotary_factor": (params or defaults)["partial_rotary_factor"],
"rope_parameters": params,
}
cfg = Qwen3_5TextConfig(
**{
k: v
for k, v in raw_config.items()
if k not in ("rope_theta", "rope_parameters")
},
rope_parameters=dict(params or defaults),
)
cfg._attn_implementation = "eager"
dtype = getattr(torch, dtype_name)
attention = Qwen3_5Attention(cfg, layer_idx=0).eval().to(dtype)
rope = Qwen3_5TextRotaryEmbedding(cfg)
assert rope.inv_freq.numel() * 2 == int(
cfg.head_dim * (params or defaults)["partial_rotary_factor"]
)
with torch.no_grad():
attention.q_proj.weight.zero_()
attention.q_proj.weight[:256].copy_(torch.eye(256))
for linear in (attention.k_proj, attention.v_proj, attention.o_proj):
linear.weight.copy_(torch.eye(256))
attention.q_norm.weight.fill_(-0.5)
attention.k_norm.weight.fill_(-0.5)
x = (
(((torch.arange(16 * 256) % 37) - 18).float() / 16)
.reshape(1, 16, 256)
.to(dtype)
)
raw_keys = attention.k_norm(
attention.k_proj(x).reshape(1, 16, 1, 256)
).transpose(1, 2)
entry = {
"name": name,
"dtype": dtype_name,
"config": raw_config,
"inv_freq": rope.inv_freq.tolist(),
"attention_factor": rope.attention_scaling,
}
modes = (
("text", text_positions),
("mrope", media_positions),
("long_text", text_positions + 32768),
)
if name == "sections":
entry["modes"] = ["mrope"]
modes = (("mrope", media_positions),)
for mode, positions in modes:
cos, sin = rope(x, positions)
output, _ = attention(
x, position_embeddings=(cos, sin), attention_mask=mask
)
entry[mode + "_output"] = [
output[0, token, dim].float().item()
for token in sample_tokens
for dim in sample_dims
]
if mode != "long_text":
_, keys = apply_rotary_pos_emb(raw_keys, raw_keys, cos, sin)
entry[mode + "_keys"] = [
keys[0, 0, token, dim].float().item()
for token in sample_tokens
for dim in sample_dims
]
results.append(entry)
return {
"source": f"transformers {transformers.__version__}; torch {torch.__version__}; CPU; Qwen3_5TextConfig/Qwen3_5TextRotaryEmbedding/Qwen3_5Attention",
"sample_tokens": sample_tokens,
"sample_dims": sample_dims,
"mrope_positions": media_positions.flatten().tolist(),
"cases": results,
}
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("output", type=Path)
args = parser.parse_args()
fixture = generate()
text = json.dumps(fixture, indent=2)
for case in fixture["cases"]:
for key in (
"inv_freq",
"text_output",
"long_text_output",
"text_keys",
"mrope_output",
"mrope_keys",
):
if key not in case:
continue
multiline = json.dumps(case[key], indent=2)
body = "\n".join(" " + line for line in multiline.splitlines()[1:-1])
text = text.replace("[\n" + body + "\n ]", json.dumps(case[key]))
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(text + "\n")
print(
f"Generated {len(fixture['cases'])} reference cases: {args.output} ({args.output.stat().st_size} bytes)"
)

View file

@ -6,6 +6,12 @@ import (
"github.com/ollama/ollama/mlx"
)
func TestSupportsRippleArchitecture(t *testing.T) {
if !SupportsArchitecture("RicursiveForCausalLM") {
t.Fatal("missing model architecture")
}
}
func TestCheckRuntimeRequiresLoadedLibrary(t *testing.T) {
_, loadErr := mlx.LoadedLibraryPath()
err := CheckRuntime()

View file

@ -53,6 +53,8 @@ func ParserForName(name string) Parser {
var p Parser
switch name {
case "ripple":
p = &RippleParser{}
case "qwen3":
p = &Qwen3Parser{hasThinkingSupport: false, defaultThinking: false}
case "qwen3-thinking":

184
model/parsers/ripple.go Normal file
View file

@ -0,0 +1,184 @@
package parsers
import (
"encoding/json"
"fmt"
"strings"
"github.com/ollama/ollama/api"
)
type rippleState int
const (
rippleStart rippleState = iota
rippleThinkLead
rippleThinking
rippleThinkEnd
rippleContent
rippleTool
)
const (
rippleThinkOpen = "<think>"
rippleThinkClose = "</think>"
rippleToolOpen = "<tool_call>"
rippleToolClose = "</tool_call>"
)
type RippleParser struct {
state rippleState
buffer string
callIndex int
toolScan int
toolString bool
toolEscape bool
}
func (p *RippleParser) Init(tools []api.Tool, _ *api.Message, _ *api.ThinkValue) []api.Tool {
*p = RippleParser{}
return tools
}
func (p *RippleParser) HasToolSupport() bool { return true }
func (p *RippleParser) HasThinkingSupport() bool { return true }
func (p *RippleParser) ThinkingClose() []string {
if p.state == rippleThinkLead || p.state == rippleThinking {
return []string{rippleThinkClose}
}
return nil
}
func (p *RippleParser) PreservedTokens() []string {
return []string{rippleThinkOpen, rippleThinkClose, rippleToolOpen, rippleToolClose}
}
func (p *RippleParser) Add(s string, done bool) (content string, thinking string, calls []api.ToolCall, err error) {
p.buffer += s
parse:
for p.buffer != "" {
switch p.state {
case rippleStart:
if strings.HasPrefix(p.buffer, rippleThinkOpen) {
p.buffer = p.buffer[len(rippleThinkOpen):]
p.state = rippleThinkLead
} else if !done && strings.HasPrefix(rippleThinkOpen, p.buffer) {
return content, thinking, calls, nil
} else {
p.state = rippleContent
}
case rippleThinkLead:
p.buffer = strings.TrimPrefix(p.buffer, "\n")
p.state = rippleThinking
case rippleThinking:
if i := strings.Index(p.buffer, rippleThinkClose); i >= 0 {
thinking += strings.TrimSuffix(p.buffer[:i], "\n")
p.buffer = p.buffer[i+len(rippleThinkClose):]
p.state = rippleThinkEnd
} else {
n := len(p.buffer)
if !done {
// The newline before a closing tag belongs to the template.
n -= max(overlap(p.buffer, rippleThinkClose), overlap(p.buffer, "\n"+rippleThinkClose))
}
thinking += p.buffer[:n]
p.buffer = p.buffer[n:]
return content, thinking, calls, nil
}
case rippleThinkEnd:
p.buffer = strings.TrimPrefix(p.buffer, "\n")
p.state = rippleContent
case rippleContent:
if i := strings.Index(p.buffer, rippleToolOpen); i >= 0 {
if p.callIndex == 0 {
content += strings.TrimSuffix(p.buffer[:i], "\n")
}
p.buffer = p.buffer[i+len(rippleToolOpen):]
p.state = rippleTool
p.toolScan, p.toolString, p.toolEscape = 0, false, false
} else {
n := len(p.buffer)
if !done {
n -= max(overlap(p.buffer, rippleToolOpen), overlap(p.buffer, "\n"+rippleToolOpen))
}
if p.callIndex == 0 {
content += p.buffer[:n]
}
p.buffer = p.buffer[n:]
return content, thinking, calls, nil
}
case rippleTool:
if i := p.toolEnd(); i >= 0 {
call, parseErr := parseRippleTool(p.buffer[:i])
p.buffer = p.buffer[i+len(rippleToolClose):]
p.state = rippleContent
if parseErr != nil {
return content, thinking, calls, parseErr
}
call.Function.Index = p.callIndex
p.callIndex++
calls = append(calls, call)
} else if !done {
return content, thinking, calls, nil
} else {
break parse
}
}
}
if done && p.state == rippleTool {
p.buffer = ""
p.state = rippleContent
return content, thinking, calls, fmt.Errorf("ripple: unterminated tool call")
}
return content, thinking, calls, nil
}
// toolEnd ignores closing tags inside JSON strings, including escaped quotes.
func (p *RippleParser) toolEnd() int {
for p.toolScan < len(p.buffer) {
i := p.toolScan
c := p.buffer[i]
if p.toolString {
if p.toolEscape {
p.toolEscape = false
} else if c == '\\' {
p.toolEscape = true
} else if c == '"' {
p.toolString = false
}
} else if c == '"' {
p.toolString = true
} else if c == '<' {
if strings.HasPrefix(p.buffer[i:], rippleToolClose) {
return i
}
if strings.HasPrefix(rippleToolClose, p.buffer[i:]) {
return -1
}
}
p.toolScan++
}
return -1
}
func parseRippleTool(raw string) (api.ToolCall, error) {
var value struct {
Name string `json:"name"`
Arguments json.RawMessage `json:"arguments"`
}
if err := json.Unmarshal([]byte(raw), &value); err != nil {
return api.ToolCall{}, fmt.Errorf("ripple: invalid tool call JSON: %w", err)
}
if value.Name == "" {
return api.ToolCall{}, fmt.Errorf("ripple: empty function name")
}
if !strings.HasPrefix(strings.TrimSpace(string(value.Arguments)), "{") {
return api.ToolCall{}, fmt.Errorf("ripple: tool arguments must be a JSON object")
}
var arguments api.ToolCallFunctionArguments
if err := json.Unmarshal(value.Arguments, &arguments); err != nil {
return api.ToolCall{}, fmt.Errorf("ripple: invalid tool arguments: %w", err)
}
return api.ToolCall{Function: api.ToolCallFunction{Name: value.Name, Arguments: arguments}}, nil
}

View file

@ -0,0 +1,228 @@
package parsers
import (
"reflect"
"strings"
"testing"
"github.com/ollama/ollama/api"
)
func rippleParseChunks(t *testing.T, chunks []string) (string, string, []api.ToolCall, error) {
t.Helper()
p := ParserForName("ripple")
if p == nil {
t.Fatal("missing ripple parser")
}
p.Init(nil, nil, nil)
var content, thinking strings.Builder
var calls []api.ToolCall
for _, chunk := range append(chunks, "") {
c, r, tc, err := p.Add(chunk, false)
content.WriteString(c)
thinking.WriteString(r)
calls = append(calls, tc...)
if err != nil {
return content.String(), thinking.String(), calls, err
}
}
c, r, tc, err := p.Add("", true)
content.WriteString(c)
thinking.WriteString(r)
calls = append(calls, tc...)
return content.String(), thinking.String(), calls, err
}
func rippleChunkings(text string) [][]string {
chunks := [][]string{{text}}
for i := 0; i <= len(text); i++ {
chunks = append(chunks, []string{text[:i], text[i:]})
}
bytes := make([]string, len(text))
for i := range len(text) {
bytes[i] = text[i : i+1]
}
return append(chunks, bytes)
}
func TestRippleParserThinking(t *testing.T) {
for _, tt := range []struct{ name, input, content, thinking string }{
{"plain", " \nhello\n", " \nhello\n", ""},
{"thinking", "<think>\nreason\n</think>\nanswer", "answer", "reason"},
{"no structural newlines", "<think>reason</think>answer", "answer", "reason"},
{"one newline only", "<think>\n</think>\n", "", ""},
{"meaningful whitespace", "<think>\n\n \treason \n\n</think>\n\n answer \n", "\n answer \n", "\n \treason \n"},
{"CRLF is not a structural newline", "<think>\r\nreason\r\n</think>\r\nanswer", "\r\nanswer", "\r\nreason\r"},
{"nonleading opener", " \n<think>reason</think>\nanswer", " \n<think>reason</think>\nanswer", ""},
{"no opener", "reason\n</think>\nanswer", "reason\n</think>\nanswer", ""},
{"later tags literal", "<think>r</think>c<think>x</think>", "c<think>x</think>", "r"},
{"tool inside thinking stays literal", "<think>\n<tool_call>{}</tool_call>\n</think>\nanswer", "answer", "<tool_call>{}</tool_call>"},
{"truncated opener", "<thin", "<thin", ""},
{"truncated reasoning", "<think>\nreason\n</thi", "", "reason\n</thi"},
{"trailing newline drained", "<think>\nreason\n", "", "reason\n"},
{"opener only", "<think>", "", ""},
{"opener and newline", "<think>\n", "", ""},
{"close only", "<think>\nreason\n</think>", "", "reason"},
{"partial tool text", "text\n<tool_", "text\n<tool_", ""},
{"unicode", "<think>\n日本語🙂\n</think>\né\n", "é\n", "日本語🙂"},
} {
t.Run(tt.name, func(t *testing.T) {
for _, chunks := range rippleChunkings(tt.input) {
c, r, calls, err := rippleParseChunks(t, chunks)
if err != nil || c != tt.content || r != tt.thinking || len(calls) != 0 {
t.Fatalf("chunks %q: got (%q, %q, %v, %v), want (%q, %q)", chunks, c, r, calls, err, tt.content, tt.thinking)
}
}
p := ParserForName("ripple")
p.Init(nil, nil, nil)
c, r, calls, err := p.Add(tt.input, true)
if err != nil || c != tt.content || r != tt.thinking || len(calls) != 0 {
t.Fatalf("final chunk: (%q, %q, %v, %v)", c, r, calls, err)
}
c, r, calls, err = p.Add("", true)
if err != nil || c != "" || r != "" || len(calls) != 0 {
t.Fatal("final drain repeated output")
}
})
}
}
func TestRippleParserThinkingTransitions(t *testing.T) {
p := ParserForName("ripple")
p.Init(nil, nil, nil)
for _, tt := range []struct {
input, content, thinking string
close bool
}{
{"<thi", "", "", false},
{"nk>", "", "", true},
{"\nreason\n", "", "reason", true},
{"</thi", "", "", true},
{"nk>", "", "", false},
{"\nanswer", "answer", "", false},
} {
c, r, _, err := p.Add(tt.input, false)
if c != tt.content || r != tt.thinking || err != nil {
t.Fatalf("%q: (%q, %q, %v)", tt.input, c, r, err)
}
if close := p.ThinkingClose(); (len(close) != 0) != tt.close || tt.close && close[0] != "</think>" {
t.Fatalf("%q: close %v", tt.input, close)
}
}
p.Init(nil, nil, nil)
c, r, _, err := p.Add("<think>\nfresh\n</think>\nnew", true)
if c != "new" || r != "fresh" || err != nil {
t.Fatalf("reset: (%q, %q, %v)", c, r, err)
}
}
func TestRippleParserTools(t *testing.T) {
one := "<tool_call>\n" + `{"name": "one", "arguments": {"z": "a\n\"b\"\\<&é", "n": 3, "nested": {"x": true}}}` + "\n</tool_call>"
two := "<tool_call>\n" + `{"name": "two", "arguments": {}}` + "\n</tool_call>"
for _, tt := range []struct {
name, input, content, thinking string
calls []api.ToolCall
}{
{"one", one, "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "one", Arguments: args(`{"z":"a\n\"b\"\\<&é","n":3,"nested":{"x":true}}`)}}}},
{"multiple", " \n before \t\n\n" + one + "\n" + two, " \n before \t\n", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "one", Arguments: args(`{"z":"a\n\"b\"\\<&é","n":3,"nested":{"x":true}}`)}}, {Function: api.ToolCallFunction{Index: 1, Name: "two", Arguments: args(`{}`)}}}},
{"reasoning and tools", "<think>\n R \n</think>\n\n" + two, "", " R ", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}}},
{"preserve spaces before call", " before \t" + two, " before \t", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}}},
{"drop post call text", two + "ignored \n<think>also text</think>\n" + two + "more ignored", "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}, {Function: api.ToolCallFunction{Index: 1, Name: "two", Arguments: args(`{}`)}}}},
{"delimiter inside string", `<tool_call>{"name":"two","arguments":{"text":"</tool_call> <tool_call> \"quoted\""}}</tool_call>`, "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{"text":"</tool_call> <tool_call> \"quoted\""}`)}}}},
} {
t.Run(tt.name, func(t *testing.T) {
for _, chunks := range rippleChunkings(tt.input) {
c, r, calls, err := rippleParseChunks(t, chunks)
if err != nil || c != tt.content || r != tt.thinking || len(calls) != len(tt.calls) {
t.Fatalf("chunks %q: (%q, %q, %v, %v)", chunks, c, r, calls, err)
}
for i := range calls {
if !toolCallEqual(calls[i], tt.calls[i]) {
t.Fatalf("call %d = %+v, want %+v", i, calls[i], tt.calls[i])
}
}
}
})
}
}
func TestRippleParserMalformedCalls(t *testing.T) {
for _, tt := range []struct{ name, input, errorText string }{
{"invalid JSON", "<tool_call>{bad}</tool_call>", "JSON"},
{"missing name", `<tool_call>{"arguments":{}}</tool_call>`, "name"},
{"empty name", `<tool_call>{"name":"","arguments":{}}</tool_call>`, "name"},
{"missing arguments", `<tool_call>{"name":"f"}</tool_call>`, "object"},
{"string arguments", `<tool_call>{"name":"f","arguments":"{}"}</tool_call>`, "object"},
{"array arguments", `<tool_call>{"name":"f","arguments":[]}</tool_call>`, "object"},
{"null arguments", `<tool_call>{"name":"f","arguments":null}</tool_call>`, "object"},
{"JSON array", `<tool_call>[{"name":"f","arguments":{}}]</tool_call>`, "JSON"},
{"multiple objects", `<tool_call>{"name":"f","arguments":{}}{}</tool_call>`, "JSON"},
{"truncated call", `<tool_call>{"name":"f","arguments":{}`, "unterminated"},
{"truncated closer", `<tool_call>{"name":"f","arguments":{}}</tool_`, "unterminated"},
} {
t.Run(tt.name, func(t *testing.T) {
for _, chunks := range rippleChunkings(tt.input) {
_, _, calls, err := rippleParseChunks(t, chunks)
if err == nil || !strings.Contains(err.Error(), tt.errorText) || len(calls) != 0 {
t.Fatalf("chunks %q: calls %v, error %v; want %s", chunks, calls, err, tt.errorText)
}
}
})
}
}
func TestRippleParserToolNewlineHoldback(t *testing.T) {
p := ParserForName("ripple")
p.Init(nil, nil, nil)
c, _, _, err := p.Add("before\n\n", false)
if c != "before\n" || err != nil {
t.Fatalf("first chunk: %q, %v", c, err)
}
c, _, _, err = p.Add("<tool_", false)
if c != "" || err != nil {
t.Fatalf("partial delimiter: %q, %v", c, err)
}
c, _, calls, err := p.Add(`call>{"name":"f","arguments":{}}</tool_call>`, true)
if c != "" || err != nil || len(calls) != 1 {
t.Fatalf("call: %q, %v, %v", c, calls, err)
}
p.Init(nil, nil, nil)
_, _, _, _ = p.Add("before\n", false)
c, _, _, err = p.Add("<tool_", true)
if c != "\n<tool_" || err != nil {
t.Fatalf("drain: %q, %v", c, err)
}
}
func TestRippleParserPlain(t *testing.T) {
for _, text := range []string{"", "Hello", " \nanswer\n\n", "Reason without opener</think>\nAnswer", " <think>literal</think>", "Text <think>literal</think>", "<thinking>literal", "é 日本語 🙂"} {
t.Run(text, func(t *testing.T) {
content, thinking, calls, err := rippleParseChunks(t, []string{text})
if err != nil || content != text || thinking != "" || len(calls) != 0 {
t.Fatalf("got (%q, %q, %v, %v), want %q", content, thinking, calls, err, text)
}
})
}
}
func TestRippleParserCapabilities(t *testing.T) {
p := ParserForName("ripple")
if p == nil {
t.Fatal("missing ripple parser")
}
if !p.HasToolSupport() || !p.HasThinkingSupport() {
t.Fatal("missing capabilities")
}
if got := p.PreservedTokens(); !reflect.DeepEqual(got, []string{"<think>", "</think>", "<tool_call>", "</tool_call>"}) {
t.Fatalf("preserved = %v", got)
}
tools := []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "f"}}}
for _, think := range []*api.ThinkValue{nil, {Value: true}, {Value: false}, {Value: "high"}} {
if got := p.Init(tools, &api.Message{Role: "assistant", Thinking: "old", Content: "old"}, think); !reflect.DeepEqual(got, tools) {
t.Fatalf("changed tools: %v", got)
}
if got := p.ThinkingClose(); len(got) != 0 {
t.Fatalf("generation prompt does not open reasoning: %v", got)
}
}
}

View file

@ -59,6 +59,8 @@ func rendererForName(name string) Renderer {
return constructor()
}
switch name {
case "ripple":
return &RippleRenderer{}
case "qwen3-coder":
renderer := &Qwen3CoderRenderer{}
return renderer

88
model/renderers/ripple.go Normal file
View file

@ -0,0 +1,88 @@
package renderers
import (
"fmt"
"strings"
"unicode"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/types/model"
)
type RippleRenderer struct{}
func (r *RippleRenderer) LeadingBOS() string { return "" }
// Ripple can emit reasoning but has no template control to disable or force it.
func (r *RippleRenderer) Thinking() *model.Thinking { return nil }
func (r *RippleRenderer) Render(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) {
if len(messages) == 0 {
return "", fmt.Errorf("ripple: expected at least one message")
}
for _, message := range messages {
if len(message.Images) > 0 {
return "", fmt.Errorf("ripple: text-only messages required")
}
}
var b strings.Builder
if len(tools) > 0 {
b.WriteString("<|im_start|>system\n")
if messages[0].Role == "system" {
b.WriteString(messages[0].Content + "\n\n")
}
b.WriteString("# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>")
for _, tool := range tools {
encoded, err := PythonJSON(tool, 0)
if err != nil {
return "", err
}
b.WriteString("\n" + encoded)
}
b.WriteString("\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n")
}
for i, message := range messages {
if i == 0 && message.Role == "system" && len(tools) > 0 {
continue
}
content := message.Content
if message.Role == "assistant" && message.Thinking != "" && !strings.HasPrefix(strings.TrimLeftFunc(content, func(r rune) bool {
return unicode.IsSpace(r) || r >= '\x1c' && r <= '\x1f'
}), "<think>") {
content = "<think>\n" + message.Thinking + "\n</think>\n" + content
}
switch message.Role {
case "assistant":
b.WriteString("<|im_start|>assistant")
if content != "" || len(message.ToolCalls) == 0 {
b.WriteString("\n" + content)
}
for _, call := range message.ToolCalls {
name, err := PythonJSON(call.Function.Name, 0)
if err != nil {
return "", err
}
arguments, err := PythonJSON(call.Function.Arguments, 0)
if err != nil {
return "", err
}
b.WriteString("\n<tool_call>\n{\"name\": " + name + ", \"arguments\": " + arguments + "}\n</tool_call>")
}
b.WriteString("<|im_end|>\n")
case "tool":
if i == 0 || messages[i-1].Role != "tool" {
b.WriteString("<|im_start|>user")
}
b.WriteString("\n<tool_response>\n" + content + "\n</tool_response>")
if i == len(messages)-1 || messages[i+1].Role != "tool" {
b.WriteString("<|im_end|>\n")
}
case "system", "user":
b.WriteString("<|im_start|>" + message.Role + "\n")
b.WriteString(content)
b.WriteString("<|im_end|>\n")
}
}
b.WriteString("<|im_start|>assistant\n")
return b.String(), nil
}

View file

@ -0,0 +1,292 @@
package renderers
import (
"encoding/json"
"os"
"strings"
"testing"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/model/parsers"
)
const rippleTemplate = "testdata/ripple_chat_template.jinja"
func TestRipplePublisherTemplate(t *testing.T) {
verifyLFM3TemplateFixture(t, rippleTemplate, "877c2dbf365243dda14738ca250b31398059b35ffd91d4ed0ec68be45ca6ebd3")
}
func TestRippleRendererTools(t *testing.T) {
call := api.ToolCall{Function: api.ToolCallFunction{Name: "f\"<&é", Arguments: args(`{"z":"line\n\"quote\"\\<&é","a":[true,null,3]}`)}}
tests := []struct {
name string
messages []api.Message
want string
}{
{
"call alone",
[]api.Message{{Role: "assistant", ToolCalls: []api.ToolCall{call}}},
"<|im_start|>assistant\n<tool_call>\n" + `{"name": "f\"<&é", "arguments": {"z": "line\n\"quote\"\\<&é", "a": [true, null, 3]}}` + "\n</tool_call><|im_end|>\n<|im_start|>assistant\n",
},
{
"multiple calls with reasoning",
[]api.Message{{Role: "assistant", Thinking: "R", Content: " C \n", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "one"}}, {Function: api.ToolCallFunction{Name: "two"}}}}},
"<|im_start|>assistant\n<think>\nR\n</think>\n C \n\n<tool_call>\n{\"name\": \"one\", \"arguments\": {}}\n</tool_call>\n<tool_call>\n{\"name\": \"two\", \"arguments\": {}}\n</tool_call><|im_end|>\n<|im_start|>assistant\n",
},
{
"grouped results",
[]api.Message{{Role: "tool", Content: " A\n"}, {Role: "tool", Content: "B"}, {Role: "user", Content: "Next"}, {Role: "tool", Content: "C"}},
"<|im_start|>user\n<tool_response>\n A\n\n</tool_response>\n<tool_response>\nB\n</tool_response><|im_end|>\n<|im_start|>user\nNext<|im_end|>\n<|im_start|>user\n<tool_response>\nC\n</tool_response><|im_end|>\n<|im_start|>assistant\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := RenderWithRenderer("ripple", tt.messages, nil, nil)
if err != nil {
t.Fatal(err)
}
if got != tt.want {
t.Fatalf("got %q\nwant %q", got, tt.want)
}
if os.Getenv("VERIFY_JINJA2") != "" {
want := renderQwenJinja(t, rippleTemplate, tt.messages, nil, nil, "")
if got != want {
t.Fatalf("Go %q\nPython %q", got, want)
}
}
})
}
}
func TestRippleRendererToolInstructions(t *testing.T) {
tool := api.Tool{Type: "function", Function: api.ToolFunction{Name: "execute_bash", Parameters: api.ToolFunctionParameters{Type: "object"}}}
for _, system := range []string{"", "Be precise."} {
messages := []api.Message{{Role: "system", Content: system}, {Role: "user", Content: "pwd"}}
got, err := RenderWithRenderer("ripple", messages, []api.Tool{tool}, nil)
if err != nil {
t.Fatal(err)
}
want := "<|im_start|>system\n" + system + "\n\n" + `# Tools
You may call one or more functions to assist with the user query.
You are provided with function signatures within <tools> XML tags:
<tools>
{"type": "function", "function": {"name": "execute_bash", "parameters": {"type": "object", "properties": null}}}
</tools>
For each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:
<tool_call>
{"name": "execute_bash", "arguments": {"command": "pwd"}}
</tool_call><|im_end|>
<|im_start|>user
pwd<|im_end|>
<|im_start|>assistant
`
if got != want {
t.Fatalf("got %q\nwant %q", got, want)
}
if os.Getenv("VERIFY_JINJA2") != "" {
want := renderQwenJinja(t, rippleTemplate, messages, []api.Tool{tool}, nil, "")
if got != want {
t.Fatalf("Go %q\nPython %q", got, want)
}
}
}
}
func TestRippleRendererInvalidInput(t *testing.T) {
for _, tt := range []struct {
name string
messages []api.Message
tools []api.Tool
errorText string
}{
{name: "empty transcript", errorText: "at least one message"},
{name: "empty transcript with tools", tools: []api.Tool{{Type: "function"}}, errorText: "at least one message"},
{name: "image", messages: []api.Message{{Role: "user", Content: "describe", Images: []api.ImageData{{1}}}}, errorText: "text-only"},
{name: "invalid tool schema", messages: []api.Message{{Role: "user"}}, tools: []api.Tool{{Items: make(chan int)}}, errorText: "unsupported type"},
{name: "invalid tool arguments", messages: []api.Message{{Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "f", Arguments: testArgs(map[string]any{"x": make(chan int)})}}}}}, errorText: "unsupported type"},
} {
t.Run(tt.name, func(t *testing.T) {
got, err := RenderWithRenderer("ripple", tt.messages, tt.tools, nil)
if got != "" || err == nil || !strings.Contains(err.Error(), tt.errorText) {
t.Fatalf("got (%q, %v), want %s", got, err, tt.errorText)
}
})
}
}
func TestRippleRendererToolResultContinuation(t *testing.T) {
messages := []api.Message{
{Role: "user", Content: "Run the program and report what it prints."},
{Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "execute_bash", Arguments: args(`{"command":"go run ./cmd/demo"}`)}}}},
{Role: "tool", Content: `{"exit_code":0,"stdout":"fmt.Println(\"done\")","stderr":""}`},
{Role: "assistant", Thinking: "The command succeeded; I can report its output.", Content: "The program prints `done`."},
}
tools := []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "execute_bash"}}}
want := "<|im_start|>system\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>\n{\"type\": \"function\", \"function\": {\"name\": \"execute_bash\", \"parameters\": {\"type\": \"\", \"properties\": null}}}\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n<|im_start|>user\nRun the program and report what it prints.<|im_end|>\n<|im_start|>assistant\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"go run ./cmd/demo\"}}\n</tool_call><|im_end|>\n<|im_start|>user\n<tool_response>\n{\"exit_code\":0,\"stdout\":\"fmt.Println(\\\"done\\\")\",\"stderr\":\"\"}\n</tool_response><|im_end|>\n<|im_start|>assistant\n<think>\nThe command succeeded; I can report its output.\n</think>\nThe program prints `done`.<|im_end|>\n<|im_start|>assistant\n"
got, err := (&RippleRenderer{}).Render(messages, tools, nil)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("render mismatch:\ngot:\n%s\nwant:\n%s", got, want)
}
}
func TestRippleRendererJinjaParity(t *testing.T) {
if os.Getenv("VERIFY_JINJA2") == "" {
t.Skip("set VERIFY_JINJA2=1 and JINJA_PYTHON for publisher template parity")
}
values := api.NewToolCallFunctionArguments()
values.Set("z", json.Number("3.0"))
values.Set("a", json.RawMessage(`{"z": 1e-7, "a": ["\u2028\u2029<&é", true, null]}`))
call := api.ToolCall{Function: api.ToolCallFunction{Name: "f\"\\\n<&é", Arguments: values}}
tool := api.Tool{Type: "function", Function: api.ToolFunction{Name: "f", Description: "Use \"quoted\" <&é\n", Parameters: api.ToolFunctionParameters{Type: "object", Properties: propsMap(`{"z":{"type":"integer"},"a":{"type":"string"}}`), Required: []string{"z"}}}}
for _, tt := range []struct {
name string
messages []api.Message
tools []api.Tool
think *api.ThinkValue
}{
{name: "plain", messages: []api.Message{{Role: "user", Content: " \nHi\t "}}},
{name: "empty user", messages: []api.Message{{Role: "user"}}},
{name: "initial and later systems", messages: []api.Message{{Role: "system", Content: "A"}, {Role: "system", Content: "B"}, {Role: "user", Content: "C"}, {Role: "system", Content: "D"}}},
{name: "tools without system", messages: []api.Message{{Role: "user", Content: "Call"}}, tools: []api.Tool{tool, tool}},
{name: "old reasoning retained", messages: []api.Message{{Role: "assistant", Thinking: "old", Content: "first"}, {Role: "user", Content: "next"}, {Role: "assistant", Thinking: "new", Content: "second"}, {Role: "user", Content: "again"}}},
{name: "no duplicate wrapper", messages: []api.Message{{Role: "assistant", Thinking: "ignored", Content: "\x1c\t\u2003<think>old</think>answer"}}},
{name: "nonleading tag", messages: []api.Message{{Role: "assistant", Thinking: "new", Content: "words<think>literal</think>"}}},
{name: "thinking false has no invented prefix", messages: []api.Message{{Role: "assistant", Thinking: "retain", Content: "answer"}}, think: &api.ThinkValue{Value: false}},
{name: "thinking true no forced opening", messages: []api.Message{{Role: "user", Content: "Hi"}}, think: &api.ThinkValue{Value: true}},
{name: "tools JSON numbers escaping and order", messages: []api.Message{{Role: "assistant", Thinking: "Reason", Content: " \n ", ToolCalls: []api.ToolCall{call}}, {Role: "tool", Content: " result \n"}, {Role: "tool", Content: "other"}}, tools: []api.Tool{tool}},
{name: "tools reasoning only", messages: []api.Message{{Role: "assistant", Thinking: "Reason", ToolCalls: []api.ToolCall{call}}}},
{name: "group boundary ignores unknown role", messages: []api.Message{{Role: "tool", Content: "one"}, {Role: "ignored"}, {Role: "tool", Content: "two"}}},
{
name: "tool result followed by assistant decision",
tools: []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "execute_bash"}}},
messages: []api.Message{
{Role: "user", Content: "Run the program and report what it prints."},
{Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "execute_bash", Arguments: args(`{"command":"go run ./cmd/demo"}`)}}}},
{Role: "tool", Content: `{"exit_code":0,"stdout":"fmt.Println(\"done\")","stderr":""}`},
{Role: "assistant", Thinking: "The command succeeded; I can report its output.", Content: "The program prints `done`."},
},
},
} {
t.Run(tt.name, func(t *testing.T) {
got, err := RenderWithRenderer("ripple", tt.messages, tt.tools, tt.think)
if err != nil {
t.Fatal(err)
}
want := renderQwenJinja(t, rippleTemplate, tt.messages, tt.tools, tt.think, "")
if got != want {
t.Fatalf("Go %q\nPython %q", got, want)
}
})
}
python := os.Getenv("JINJA_PYTHON")
if python == "" {
t.Fatal("set JINJA_PYTHON for negative publisher-template checks")
}
for _, messages := range []any{[]any{}, []any{map[string]any{"role": "user", "content": []any{map[string]any{"type": "image"}}}}} {
output, err := lfm3RunJinja2(t, python, rippleTemplate, map[string]any{"messages": messages, "add_generation_prompt": true})
if err == nil {
t.Fatalf("publisher accepted unsupported transcript: %q", output)
}
}
}
func TestRippleRendererParserRoundTrip(t *testing.T) {
for _, content := range []string{"", " ", "\n", "\n\n", " \t\n ", "answer\n", "日本語\n"} {
for _, thinking := range []string{"", "R", " \n", "\n\nR\t\n"} {
for _, calls := range [][]api.ToolCall{nil, {{Function: api.ToolCallFunction{Name: "f", Arguments: args(`{"text":"<&é\n\"quoted\""}`)}}}, {{Function: api.ToolCallFunction{Name: "f"}}, {Function: api.ToolCallFunction{Name: "g"}}}} {
message := api.Message{Role: "assistant", Content: content, Thinking: thinking, ToolCalls: calls}
original, err := RenderWithRenderer("ripple", []api.Message{message}, nil, nil)
if err != nil {
t.Fatal(err)
}
body := strings.TrimSuffix(strings.TrimPrefix(original, "<|im_start|>assistant\n"), "<|im_end|>\n<|im_start|>assistant\n")
p := parsers.ParserForName("ripple")
p.Init(nil, nil, nil)
parsed := api.Message{Role: "assistant"}
for i := 0; i <= len(body); i++ {
chunk := ""
if i < len(body) {
chunk = body[i : i+1]
}
c, r, tc, err := p.Add(chunk, i == len(body))
if err != nil {
t.Fatal(err)
}
parsed.Content += c
parsed.Thinking += r
parsed.ToolCalls = append(parsed.ToolCalls, tc...)
}
got, err := RenderWithRenderer("ripple", []api.Message{parsed}, nil, nil)
if err != nil {
t.Fatal(err)
}
if got != original {
t.Fatalf("roundtrip got %q\nwant %q", got, original)
}
}
}
}
}
func TestRippleRendererReasoningHistory(t *testing.T) {
for _, tt := range []struct{ name, content, reasoning, want string }{
{"separate", " Answer \n", " Reason \n", "<think>\n Reason \n\n</think>\n Answer \n"},
{"already tagged", "<think>\noriginal\n</think>\nAnswer", "ignored", "<think>\noriginal\n</think>\nAnswer"},
{"leading whitespace already tagged", "\x1c\t\u2003<think>original</think>Answer", "ignored", "\x1c\t\u2003<think>original</think>Answer"},
{"nonleading tag", "Text <think>literal</think>", "Reason", "<think>\nReason\n</think>\nText <think>literal</think>"},
{"empty reasoning", "Answer", "", "Answer"},
} {
t.Run(tt.name, func(t *testing.T) {
for _, think := range []*api.ThinkValue{nil, {Value: true}, {Value: false}, {Value: "high"}} {
got, err := RenderWithRenderer("ripple", []api.Message{{Role: "assistant", Content: tt.content, Thinking: tt.reasoning}, {Role: "user", Content: "Next"}}, nil, think)
if err != nil {
t.Fatal(err)
}
want := "<|im_start|>assistant\n" + tt.want + "<|im_end|>\n<|im_start|>user\nNext<|im_end|>\n<|im_start|>assistant\n"
if got != want {
t.Fatalf("got %q\nwant %q", got, want)
}
}
})
}
}
func TestRippleRendererMessages(t *testing.T) {
for _, tt := range []struct {
name string
messages []api.Message
want string
}{
{"plain", []api.Message{{Role: "user", Content: " Hi\n"}}, "<|im_start|>user\n Hi\n<|im_end|>\n<|im_start|>assistant\n"},
{"system", []api.Message{{Role: "system", Content: "Be precise."}, {Role: "user", Content: "Hi"}}, "<|im_start|>system\nBe precise.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n"},
{"empty system", []api.Message{{Role: "system"}}, "<|im_start|>system\n<|im_end|>\n<|im_start|>assistant\n"},
{"later system", []api.Message{{Role: "user", Content: "Hi"}, {Role: "system", Content: "Now"}}, "<|im_start|>user\nHi<|im_end|>\n<|im_start|>system\nNow<|im_end|>\n<|im_start|>assistant\n"},
{"assistant is closed not prefilled", []api.Message{{Role: "assistant", Content: "Part"}}, "<|im_start|>assistant\nPart<|im_end|>\n<|im_start|>assistant\n"},
{"unknown role ignored", []api.Message{{Role: "developer", Content: "ignored"}}, "<|im_start|>assistant\n"},
} {
t.Run(tt.name, func(t *testing.T) {
got, err := RenderWithRenderer("ripple", tt.messages, nil, nil)
if err != nil {
t.Fatal(err)
}
if got != tt.want {
t.Fatalf("got %q\nwant %q", got, tt.want)
}
})
}
if rendererForName("ripple") == nil {
t.Fatal("missing ripple renderer")
}
if got := LeadingBOSForRenderer("ripple"); got != "" {
t.Fatalf("BOS = %q", got)
}
if got := ThinkingForRenderer("ripple"); got != nil {
t.Fatalf("unexpected thinking control: %+v", got)
}
}

View file

@ -0,0 +1,52 @@
{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0]['role'] == 'system' %}
{{- messages[0]['content'] + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0]['role'] == 'system' %}
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- for message in messages %}
{%- set content = message.content or '' %}
{%- if message.role == 'assistant' and message.reasoning_content and not (clear_thinking | default(false)) and not content.lstrip().startswith('<think>') %}
{%- set content = '<think>\n' + message.reasoning_content + '\n</think>\n' + content %}
{%- endif %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{{- '<|im_start|>' + message.role }}
{%- if content %}
{{- '\n' + content }}
{%- endif %}
{%- for tool_call in message.tool_calls %}
{%- if tool_call.function is defined %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '\n<tool_call>\n{"name": ' }}
{{- tool_call.name | tojson }}
{{- ', "arguments": ' }}
{{- tool_call.arguments | tojson }}
{{- '}\n</tool_call>' }}
{%- endfor %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' + message.content + '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- endif %}