mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
Add ripple preview support to the MLX runner (#19592)
Add the Ripple model, renderer, parser and import registration. Support per-layer NoPE in DFlash drafts. Fuse MLP activation, value blending and attention gating. Add reference fixtures and model/runner regression tests.
This commit is contained in:
parent
5e983d4f80
commit
d63b702609
20 changed files with 3878 additions and 200 deletions
|
|
@ -282,6 +282,8 @@ func parserNameForConfig(modelDir string, cfg sourceModelConfig, chatTemplate st
|
|||
func parserNameForIdentifier(modelDir, s, chatTemplate string) (string, error) {
|
||||
s = strings.ToLower(s)
|
||||
switch {
|
||||
case s == "ricursiveforcausallm" || s == "ricursive":
|
||||
return "ripple", nil
|
||||
case strings.Contains(s, "flint"):
|
||||
return "flint", nil
|
||||
case strings.HasPrefix(s, "lfm3"):
|
||||
|
|
@ -322,6 +324,8 @@ func rendererNameForConfig(modelDir string, cfg sourceModelConfig, chatTemplate
|
|||
func rendererNameForIdentifier(modelDir, s, chatTemplate string) (string, error) {
|
||||
s = strings.ToLower(s)
|
||||
switch {
|
||||
case s == "ricursiveforcausallm" || s == "ricursive":
|
||||
return "ripple", nil
|
||||
case strings.Contains(s, "flint"):
|
||||
return "flint", nil
|
||||
case strings.HasPrefix(s, "lfm3"):
|
||||
|
|
|
|||
|
|
@ -102,6 +102,23 @@ func TestInferSafetensorsConfigFamilies(t *testing.T) {
|
|||
wantRenderer string
|
||||
wantCaps []string
|
||||
}{
|
||||
{
|
||||
name: "ripple architecture",
|
||||
config: `{"architectures":["RicursiveForCausalLM"],"model_type":"ricursive"}`,
|
||||
chatTemplate: `{% for message in messages %}{{ message.content }}{% endfor %}`,
|
||||
wantParser: "ripple",
|
||||
wantRenderer: "ripple",
|
||||
wantCaps: []string{"completion", "tools", "thinking"},
|
||||
},
|
||||
{
|
||||
name: "ripple model type",
|
||||
config: `{"model_type":"ricursive"}`,
|
||||
chatTemplate: `<think>{{ reasoning_content }}</think>{{ content }}`,
|
||||
standalone: true,
|
||||
wantParser: "ripple",
|
||||
wantRenderer: "ripple",
|
||||
wantCaps: []string{"completion", "tools", "thinking"},
|
||||
},
|
||||
{
|
||||
name: "qwen3",
|
||||
config: `{"architectures":["Qwen3ForCausalLM"]}`,
|
||||
|
|
|
|||
|
|
@ -23,5 +23,6 @@ import (
|
|||
_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5"
|
||||
_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5_moe"
|
||||
_ "github.com/ollama/ollama/mlxrunner/model/qwen4_exp"
|
||||
_ "github.com/ollama/ollama/mlxrunner/model/ripple"
|
||||
_ "github.com/ollama/ollama/mlxrunner/model/strands"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@ package dflash
|
|||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/ollama/ollama/mlx"
|
||||
"github.com/ollama/ollama/mlx/mlxtest"
|
||||
"github.com/ollama/ollama/mlxrunner/batch"
|
||||
"github.com/ollama/ollama/mlxrunner/cache"
|
||||
"github.com/ollama/ollama/mlxrunner/model"
|
||||
"github.com/ollama/ollama/mlxrunner/nn"
|
||||
)
|
||||
|
||||
|
|
@ -120,6 +122,298 @@ func TestRepeatedBlockRollback(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// The scalar oracle uses nonzero queries and checks both cache K/V and output,
|
||||
// so skipping only context K, block K, or block Q cannot satisfy the contract.
|
||||
func TestDraftPerLayerNoRoPE(t *testing.T) {
|
||||
for _, policy := range []struct {
|
||||
name, field string
|
||||
noRoPE [2]bool
|
||||
}{
|
||||
{"omitted", "", [2]bool{}},
|
||||
{"empty", `,"no_rope_layers":[]`, [2]bool{}},
|
||||
{"full_nope", `,"no_rope_layers":[1]`, [2]bool{false, true}},
|
||||
{"sliding_nope", `,"no_rope_layers":[0]`, [2]bool{true, false}},
|
||||
} {
|
||||
for _, path := range []struct {
|
||||
name string
|
||||
context, block bool
|
||||
}{
|
||||
{"context_only", true, false},
|
||||
{"block_only", false, true},
|
||||
{"combined", true, true},
|
||||
} {
|
||||
mlxtest.RunSubtest(t, policy.name+"/"+path.name, func(t *mlxtest.T) {
|
||||
mlx.Scoped(func() {
|
||||
m := newDraftRotaryTestModel(t, policy.field)
|
||||
caches := m.NewCaches()
|
||||
for _, c := range caches {
|
||||
defer c.Free()
|
||||
}
|
||||
var oracle draftRotaryOracle
|
||||
// Seed the real caches without invoking either rotary path.
|
||||
// Block-only must not fail because context maintenance is broken.
|
||||
seed := []float32{0.25, -0.5, 0.75, 0.25, -0.5, 1}
|
||||
for i, c := range caches {
|
||||
c.(cache.Attention).Update(&batch.Batch{SeqOffsets: []int32{0}},
|
||||
mlx.Zeros(mlx.DTypeFloat32, 1, 1, 3, 2), mlx.FromValues(seed, 1, 1, 3, 2))
|
||||
oracle.keys[i] = make([]float64, len(seed))
|
||||
for _, v := range seed {
|
||||
oracle.values[i] = append(oracle.values[i], float64(v))
|
||||
}
|
||||
}
|
||||
for round := range 2 {
|
||||
start := caches[0].Offset()
|
||||
b := &batch.Batch{SeqOffsets: []int32{int32(start)}}
|
||||
var context, block []float64
|
||||
if path.context {
|
||||
context = []float64{1, 2, -3, 1}
|
||||
b.Hidden = mlx.FromValues([]float32{1, 2, -3, 1}, 1, 2, 2)
|
||||
}
|
||||
if path.block {
|
||||
block = []float64{1, 2, -2, 1, 0.5, -1}
|
||||
b.InputIDs = mlx.FromValues([]int32{0, 1, 2}, 1, 3)
|
||||
b.SeqQueryLens = []int32{3}
|
||||
}
|
||||
want := oracle.forward(context, block, start, policy.noRoPE)
|
||||
hidden, aux := m.Forward(b, nil, caches)
|
||||
label := fmt.Sprintf("round %d offset %d", round, start)
|
||||
if !path.block {
|
||||
if hidden != nil || aux != nil {
|
||||
t.Fatal("context-only forward returned block hidden states")
|
||||
}
|
||||
} else {
|
||||
if hidden == nil || aux != hidden {
|
||||
t.Fatal("block forward did not return shared hidden/aux states")
|
||||
}
|
||||
checkDraftRotaryArray(t, label+" hidden", hidden, want)
|
||||
}
|
||||
for i, c := range caches {
|
||||
if wantOffset := len(oracle.keys[i]) / 2; c.Offset() != wantOffset {
|
||||
t.Fatalf("%s layer %d cache offset=%d want=%d", label, i, c.Offset(), wantOffset)
|
||||
}
|
||||
state := c.State()
|
||||
if len(state) != 2 {
|
||||
t.Fatalf("%s layer %d state roots=%d want=2", label, i, len(state))
|
||||
}
|
||||
checkDraftRotaryArray(t, fmt.Sprintf("%s layer %d keys", label, i), state[0], oracle.keys[i])
|
||||
checkDraftRotaryArray(t, fmt.Sprintf("%s layer %d values", label, i), state[1], oracle.values[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type windowedDraftTarget struct {
|
||||
*draftRotaryTarget
|
||||
window int
|
||||
}
|
||||
|
||||
func (t *windowedDraftTarget) DraftWindow() int { return t.window }
|
||||
|
||||
func checkDraftWindowValues(t *mlxtest.T, label string, array *mlx.Array, want []float64) {
|
||||
t.Helper()
|
||||
array = mlx.Contiguous(array, false).AsType(mlx.DTypeFloat32)
|
||||
mlx.Eval(array)
|
||||
got := array.Floats()
|
||||
wantFloat := make([]float32, len(want))
|
||||
for i, v := range want {
|
||||
wantFloat[i] = float32(v)
|
||||
}
|
||||
slices.Sort(got)
|
||||
slices.Sort(wantFloat)
|
||||
if !slices.Equal(got, wantFloat) {
|
||||
t.Fatalf("%s values = %v, want %v", label, got, wantFloat)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDraftWindowCache(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
target draftTarget
|
||||
initial, wrapped [][]float64
|
||||
}{
|
||||
{
|
||||
name: "default", target: &draftRotaryTarget{},
|
||||
initial: [][]float64{{2, 3}, {0, 1, 2, 3}},
|
||||
wrapped: [][]float64{{3, 4}, {0, 1, 2, 3, 4}},
|
||||
},
|
||||
{
|
||||
name: "bounded", target: &windowedDraftTarget{draftRotaryTarget: &draftRotaryTarget{}, window: 3},
|
||||
initial: [][]float64{{2, 3}, {1, 2, 3}},
|
||||
wrapped: [][]float64{{3, 4}, {2, 3, 4}},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
m := &Model{
|
||||
Config: &Config{SlidingWindow: 2},
|
||||
Layers: []*Layer{{IsSliding: true}, {IsSliding: false}},
|
||||
target: tc.target,
|
||||
}
|
||||
caches := m.NewCaches()
|
||||
defer func() {
|
||||
for _, c := range caches {
|
||||
c.Free()
|
||||
}
|
||||
}()
|
||||
initial := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{4}}
|
||||
values := mlx.FromValues([]float32{0, 1, 2, 3}, 1, 1, 4, 1)
|
||||
for i, c := range caches {
|
||||
c.(cache.Attention).Update(initial, values, values)
|
||||
checkDraftWindowValues(t, fmt.Sprintf("initial cache %d", i), c.(cache.Attention).View(initial).V(), tc.initial[i])
|
||||
c.PrepareSnapshots([]int{4})
|
||||
next := &batch.Batch{SeqOffsets: []int32{4}, SeqQueryLens: []int32{1}}
|
||||
value := mlx.FromValue(float32(4))
|
||||
value = mlx.Reshape(value, 1, 1, 1, 1)
|
||||
c.(cache.Attention).Update(next, value, value)
|
||||
checkDraftWindowValues(t, fmt.Sprintf("wrapped cache %d", i), c.(cache.Attention).View(next).V(), tc.wrapped[i])
|
||||
snapshots := c.TakeSnapshots()
|
||||
if len(snapshots) != 1 {
|
||||
t.Fatalf("cache %d snapshots = %d", i, len(snapshots))
|
||||
}
|
||||
if !c.Restore(snapshots[0], 4) {
|
||||
t.Fatalf("cache %d could not restore", i)
|
||||
}
|
||||
if snapshots[0] != nil {
|
||||
snapshots[0].Close()
|
||||
}
|
||||
if c.Offset() != 4 {
|
||||
t.Fatalf("cache %d offset = %d", i, c.Offset())
|
||||
}
|
||||
checkDraftWindowValues(t, fmt.Sprintf("restored cache %d", i), c.(cache.Attention).View(initial).V(), tc.initial[i])
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type draftRotaryTarget struct {
|
||||
model.Model
|
||||
embeddings *mlx.Array
|
||||
}
|
||||
|
||||
func (m *draftRotaryTarget) TokenEmbeddings(ids *mlx.Array) *mlx.Array {
|
||||
return m.embeddings.TakeAxis(ids, 0)
|
||||
}
|
||||
|
||||
func (*draftRotaryTarget) RawLogits(x *mlx.Array) *mlx.Array { return x }
|
||||
func (*draftRotaryTarget) SetAuxHiddenLayers([]int) {}
|
||||
func (*draftRotaryTarget) NumLayers() int { return 2 }
|
||||
|
||||
func newDraftRotaryTestModel(t *mlxtest.T, field string) *Model {
|
||||
t.Helper()
|
||||
cfg, err := parseConfig(draftRotaryTestConfig(field))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := &Model{
|
||||
Config: cfg, Layers: make([]*Layer, cfg.NumHiddenLayers), tensorPrefix: "draft.",
|
||||
target: &draftRotaryTarget{embeddings: mlx.FromValues([]float32{1, 2, -2, 1, 0.5, -1}, 3, 2)},
|
||||
}
|
||||
identity := mlx.FromValues([]float32{1, 0, 0, 1}, 2, 2)
|
||||
ones := mlx.FromValues([]float32{1, 1}, 2)
|
||||
zero := mlx.Zeros(mlx.DTypeFloat32, 2, 2)
|
||||
tensors := map[string]*mlx.Array{
|
||||
"draft.fc.weight": identity, "draft.hidden_norm.weight": ones, "draft.norm.weight": ones,
|
||||
}
|
||||
for i := range m.Layers {
|
||||
prefix := fmt.Sprintf("draft.layers.%d.", i)
|
||||
for _, name := range []string{"q_proj", "k_proj", "v_proj", "o_proj"} {
|
||||
tensors[prefix+"self_attn."+name+".weight"] = identity
|
||||
}
|
||||
for _, name := range []string{"input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm"} {
|
||||
tensors[prefix+name+".weight"] = ones
|
||||
}
|
||||
for _, name := range []string{"gate_proj", "up_proj", "down_proj"} {
|
||||
tensors[prefix+"mlp."+name+".weight"] = zero
|
||||
}
|
||||
}
|
||||
if err := m.LoadWeights(tensors); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func checkDraftRotaryArray(t *mlxtest.T, label string, array *mlx.Array, want []float64) {
|
||||
t.Helper()
|
||||
// Cache state can be a strided view; Floats requires contiguous storage.
|
||||
array = mlx.Contiguous(array, false).AsType(mlx.DTypeFloat32)
|
||||
mlx.Eval(array)
|
||||
got := array.Floats()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("%s length=%d want=%d", label, len(got), len(want))
|
||||
}
|
||||
for i, v := range got {
|
||||
if math.IsNaN(float64(v)) || math.Abs(float64(v)-want[i]) > 2e-3*math.Max(1, math.Abs(want[i])) {
|
||||
t.Errorf("%s[%d]: got %g want %g", label, i, v, want[i])
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This two-wide scalar model has identity projections, unit norms, zero MLPs,
|
||||
// and a window larger than its history. It does not call production RoPE/SDPA.
|
||||
type draftRotaryOracle struct {
|
||||
keys, values [2][]float64
|
||||
}
|
||||
|
||||
func draftRotaryNorm(x []float64) []float64 {
|
||||
out := make([]float64, len(x))
|
||||
for r := 0; r < len(x); r += 2 {
|
||||
inv := 1 / math.Sqrt((x[r]*x[r]+x[r+1]*x[r+1])/2+1e-6)
|
||||
out[r], out[r+1] = x[r]*inv, x[r+1]*inv
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func draftRotaryRotate(x []float64, start int) []float64 {
|
||||
out := slices.Clone(x)
|
||||
for r := 0; r < len(x); r += 2 {
|
||||
s, c := math.Sincos(float64(start + r/2))
|
||||
out[r], out[r+1] = x[r]*c-x[r+1]*s, x[r]*s+x[r+1]*c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *draftRotaryOracle) forward(context, block []float64, start int, noRoPE [2]bool) []float64 {
|
||||
contextValues := draftRotaryNorm(context)
|
||||
x := slices.Clone(block)
|
||||
for layer := range o.keys {
|
||||
contextKeys := draftRotaryNorm(contextValues)
|
||||
values := draftRotaryNorm(x)
|
||||
q := draftRotaryNorm(values)
|
||||
k := slices.Clone(q)
|
||||
if !noRoPE[layer] {
|
||||
contextKeys = draftRotaryRotate(contextKeys, start)
|
||||
q = draftRotaryRotate(q, start+len(context)/2)
|
||||
k = draftRotaryRotate(k, start+len(context)/2)
|
||||
}
|
||||
o.keys[layer] = append(o.keys[layer], contextKeys...)
|
||||
o.keys[layer] = append(o.keys[layer], k...)
|
||||
o.values[layer] = append(o.values[layer], contextValues...)
|
||||
o.values[layer] = append(o.values[layer], values...)
|
||||
for r := 0; r < len(x); r += 2 {
|
||||
scores := make([]float64, len(o.keys[layer])/2)
|
||||
for j := range scores {
|
||||
scores[j] = (q[r]*o.keys[layer][2*j] + q[r+1]*o.keys[layer][2*j+1]) / math.Sqrt(2)
|
||||
}
|
||||
maxScore := slices.Max(scores)
|
||||
total := 0.0
|
||||
for j, score := range scores {
|
||||
scores[j] = math.Exp(score - maxScore)
|
||||
total += scores[j]
|
||||
}
|
||||
for j, score := range scores {
|
||||
x[r] += score / total * o.values[layer][2*j]
|
||||
x[r+1] += score / total * o.values[layer][2*j+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
return draftRotaryNorm(x)
|
||||
}
|
||||
|
||||
// meanAttention is one head of zero queries over width-wide rows: every output
|
||||
// row is the mean of the value rows it may see, and values copy their input.
|
||||
func meanAttention(width int, window int32) (*Config, *Attention) {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
|
||||
"github.com/ollama/ollama/mlx"
|
||||
"github.com/ollama/ollama/mlxrunner/batch"
|
||||
|
|
@ -49,6 +50,7 @@ type Config struct {
|
|||
Scale float32
|
||||
SlidingWindow int32
|
||||
LayerTypes []string
|
||||
NoRoPELayers []int
|
||||
|
||||
BlockSize int
|
||||
MaskTokenID int32
|
||||
|
|
@ -143,6 +145,7 @@ type Attention struct {
|
|||
OProj nn.LinearLayer
|
||||
QNorm *nn.RMSNorm
|
||||
KNorm *nn.RMSNorm
|
||||
NoRoPE bool
|
||||
|
||||
// ctxInputNorm applies the layer's input norm to context rows (laguna).
|
||||
ctxInputNorm *nn.RMSNorm
|
||||
|
|
@ -183,6 +186,7 @@ func parseConfig(data []byte) (*Config, error) {
|
|||
VocabSize int32 `json:"vocab_size"`
|
||||
HasOwnLMHead bool `json:"has_own_lm_head"`
|
||||
LayerTypes []string `json:"layer_types"`
|
||||
NoRoPELayers []int `json:"no_rope_layers"`
|
||||
SlidingWindow int32 `json:"sliding_window"`
|
||||
RopeInterleaved *bool `json:"rope_interleaved"`
|
||||
}
|
||||
|
|
@ -200,6 +204,7 @@ func parseConfig(data []byte) (*Config, error) {
|
|||
RopeTheta: raw.RopeTheta,
|
||||
SlidingWindow: raw.SlidingWindow,
|
||||
LayerTypes: raw.LayerTypes,
|
||||
NoRoPELayers: raw.NoRoPELayers,
|
||||
BlockSize: raw.DFlashConfig.BlockSize,
|
||||
VocabSize: raw.VocabSize,
|
||||
HasOwnLMHead: raw.HasOwnLMHead,
|
||||
|
|
@ -217,6 +222,11 @@ func parseConfig(data []byte) (*Config, error) {
|
|||
if cfg.NumHiddenLayers > maxDraftLayers {
|
||||
return nil, fmt.Errorf("dflash num_hidden_layers %d exceeds supported maximum %d", cfg.NumHiddenLayers, maxDraftLayers)
|
||||
}
|
||||
for _, id := range cfg.NoRoPELayers {
|
||||
if id < 0 || id >= int(cfg.NumHiddenLayers) {
|
||||
return nil, fmt.Errorf("dflash no_rope_layers index %d out of range for %d layers", id, cfg.NumHiddenLayers)
|
||||
}
|
||||
}
|
||||
// The fused projection holds the query, key and value rows.
|
||||
if (int64(cfg.NumAttentionHeads)+2*int64(cfg.NumKeyValueHeads))*int64(cfg.HeadDim) > math.MaxInt32 {
|
||||
return nil, fmt.Errorf("dflash attention projection dimensions exceed int32")
|
||||
|
|
@ -391,8 +401,9 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {
|
|||
layer := &Layer{
|
||||
IsSliding: m.LayerTypes[i] == "sliding_attention",
|
||||
Attention: &Attention{
|
||||
GProj: linears.Make(layerPrefix + ".self_attn.g_proj"),
|
||||
OProj: linears.Make(layerPrefix + ".self_attn.o_proj"),
|
||||
NoRoPE: slices.Contains(m.NoRoPELayers, i),
|
||||
GProj: linears.Make(layerPrefix + ".self_attn.g_proj"),
|
||||
OProj: linears.Make(layerPrefix + ".self_attn.o_proj"),
|
||||
},
|
||||
MLP: &MLP{
|
||||
DownProj: linears.Make(layerPrefix + ".mlp.down_proj"),
|
||||
|
|
@ -583,9 +594,19 @@ func (m *Model) BlockParams() (int, int32) { return m.BlockSize, m.MaskTokenID }
|
|||
// NewCaches builds the per-layer context caches.
|
||||
func (m *Model) NewCaches() []cache.Cache {
|
||||
caches := make([]cache.Cache, len(m.Layers))
|
||||
draftWindow := 0
|
||||
if target, ok := m.target.(interface{ DraftWindow() int }); ok {
|
||||
draftWindow = target.DraftWindow()
|
||||
}
|
||||
for i, layer := range m.Layers {
|
||||
if layer.IsSliding {
|
||||
caches[i] = cache.NewRotatingKVCache(int(m.SlidingWindow))
|
||||
window := int(m.SlidingWindow)
|
||||
if draftWindow > 0 {
|
||||
window = min(window, draftWindow)
|
||||
}
|
||||
caches[i] = cache.NewRotatingKVCache(window)
|
||||
} else if draftWindow > 0 {
|
||||
caches[i] = cache.NewRotatingKVCache(draftWindow)
|
||||
} else {
|
||||
caches[i] = cache.NewKVCache()
|
||||
}
|
||||
|
|
@ -681,7 +702,9 @@ func (a *Attention) contextKV(hctx *mlx.Array, positions *mlx.Array, cfg *Config
|
|||
k = a.KNorm.Forward(k, cfg.RMSNormEps)
|
||||
k = mlx.Transpose(k, 0, 2, 1, 3)
|
||||
v = mlx.Transpose(v, 0, 2, 1, 3)
|
||||
k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
|
||||
if !a.NoRoPE {
|
||||
k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
|
||||
}
|
||||
return k, v
|
||||
}
|
||||
|
||||
|
|
@ -703,8 +726,10 @@ func (a *Attention) Forward(x, ctxK, ctxV *mlx.Array, c cache.Cache, bb *batch.B
|
|||
k = mlx.Transpose(k, 0, 2, 1, 3)
|
||||
v = mlx.Transpose(v, 0, 2, 1, 3)
|
||||
|
||||
q = mlx.RoPEWithBase(q, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
|
||||
k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
|
||||
if !a.NoRoPE {
|
||||
q = mlx.RoPEWithBase(q, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
|
||||
k = mlx.RoPEWithBase(k, int(cfg.HeadDim), cfg.RopeInterleaved, cfg.RopeTheta, 1.0, positions)
|
||||
}
|
||||
|
||||
if ctxK != nil {
|
||||
k = ctxK.Concatenate(2, k)
|
||||
|
|
|
|||
|
|
@ -241,6 +241,89 @@ func TestSelectorConditionalProbabilities(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestFourTapConvolutionOracle(t *testing.T) {
|
||||
for _, dtype := range []mlx.DType{mlx.DTypeFloat32, mlx.DTypeBFloat16} {
|
||||
for _, length := range []int{1, 2, 3, 4, 8, 16} {
|
||||
mlxtest.RunSubtest(t, fmt.Sprintf("%v length %d", dtype, length), func(t *mlxtest.T) {
|
||||
const width, group, taps = 32, 16, 4
|
||||
values := func(n int) []float32 {
|
||||
v := make([]float32, n)
|
||||
for i := range v {
|
||||
v[i] = float32(i%7-3) / 8
|
||||
}
|
||||
return v
|
||||
}
|
||||
h, d, s := values(length*width), values(length*taps*width/group), values(taps*width)
|
||||
want := convolutionOracle(h, d, s, 1, length, width, taps, width/group, func(v float32) float32 {
|
||||
return rounded(v, dtype == mlx.DTypeBFloat16)
|
||||
})
|
||||
inputs := []*mlx.Array{mlx.FromValues(h, 1, length, width).AsType(dtype), mlx.FromValues(d, 1, length, taps, width/group).AsType(dtype), mlx.FromValues(s, taps, width).AsType(dtype)}
|
||||
for _, got := range []*mlx.Array{convolveGraph(inputs[0], inputs[1], inputs[2]), convolve(inputs[0], inputs[1], inputs[2])} {
|
||||
got = mlx.Contiguous(got.AsType(mlx.DTypeFloat32), false)
|
||||
mlx.Eval(got)
|
||||
if !slices.Equal(got.Floats(), want) {
|
||||
t.Fatal("four-tap convolution differs from scalar oracle")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectorTop32Probabilities(t *testing.T) {
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
const vocab, rank, k, length = 40, 256, 32, 3
|
||||
pred, succ, projection := make([]float32, vocab*rank), make([]float32, vocab*rank), make([]float32, rank)
|
||||
for id := range vocab {
|
||||
for j := range rank {
|
||||
pred[id*rank+j] = float32(id%3-1) / 32
|
||||
succ[id*rank+j] = float32(id%5-2) / 64
|
||||
}
|
||||
}
|
||||
for j := range projection {
|
||||
projection[j] = 0.5
|
||||
}
|
||||
logits := make([]float32, length*vocab)
|
||||
for i := range logits {
|
||||
logits[i] = -float32(i%vocab) / 8
|
||||
}
|
||||
m := &model2{Selector: &candidateSelector{
|
||||
Projection: nn.NewLinear(mlx.FromValues(projection, rank, 1).AsType(mlx.DTypeBFloat16), nil),
|
||||
Predecessor: mlx.FromValues(pred, vocab, rank).AsType(mlx.DTypeBFloat16),
|
||||
Successor: mlx.FromValues(succ, vocab, rank).AsType(mlx.DTypeBFloat16), TopK: k,
|
||||
}}
|
||||
sampler := sample.New(16)
|
||||
defer sampler.Free()
|
||||
sampler.Add(0, sample.Options{Temperature: 1, UseSeed: true, Seed: 42}, nil)
|
||||
tokens, ids, probs := m.Select(mlx.FromValues([]float32{1, 1, 1}, 1, length, 1).AsType(mlx.DTypeBFloat16), mlx.FromValues(logits, 1, length, vocab).AsType(mlx.DTypeBFloat16), mlx.FromValues([]int32{0}, 1), func(ids, scores *mlx.Array) (*mlx.Array, *mlx.Array) {
|
||||
tokens, distribution := sampler.SampleCandidates(0, ids, scores)
|
||||
return tokens, distribution.Probs
|
||||
})
|
||||
mlx.Eval(tokens, ids, probs)
|
||||
path, support, q := tokens.Ints(), ids.Ints(), mlx.Contiguous(probs, false).Floats()
|
||||
previous := 0
|
||||
for row := range length {
|
||||
weights, total := make([]float64, k), float64(0)
|
||||
for id := range k {
|
||||
weights[id] = math.Exp(-float64(id)/8 + float64((previous%3-1)*(id%5-2))/16)
|
||||
total += weights[id]
|
||||
}
|
||||
for j, id := range support[row*k : (row+1)*k] {
|
||||
if id < 0 || id >= k {
|
||||
t.Fatalf("unexpected candidate %d", id)
|
||||
}
|
||||
if want := weights[id] / total; math.Abs(float64(q[row*k+j])-want) > 1e-6 {
|
||||
t.Fatalf("row %d candidate %d probability=%g want=%g", row, id, q[row*k+j], want)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(support[row*k:(row+1)*k], path[row]) {
|
||||
t.Fatal("selected token outside proposal support")
|
||||
}
|
||||
previous = int(path[row])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDFlash2ConfigValidation(t *testing.T) {
|
||||
valid := Config{HiddenSize: 5120, ConvGroupSize: 16, ConvKernelSize: 2, BlockSize: 8, SelectorRank: 256, SelectorTopK: 16, VocabSize: 248320}
|
||||
for _, tt := range []struct {
|
||||
|
|
|
|||
|
|
@ -2,9 +2,11 @@ package dflash
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -50,6 +52,50 @@ func TestDraftConfigBoundsAndCausality(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Keep the policy in JSON so these tests exercise parsing and loader wiring.
|
||||
func draftRotaryTestConfig(noRoPEField string) []byte {
|
||||
return []byte(fmt.Sprintf(`{
|
||||
"hidden_size": 2, "num_hidden_layers": 2,
|
||||
"num_attention_heads": 1, "num_key_value_heads": 1, "head_dim": 2,
|
||||
"rms_norm_eps": 0.000001, "rope_theta": 10000,
|
||||
"layer_types": ["sliding_attention", "full_attention"], "sliding_window": 64,
|
||||
"dflash_config": {"block_size": 4, "mask_token_id": 0, "target_layer_ids": [0], "causal": false}
|
||||
%s
|
||||
}`, noRoPEField))
|
||||
}
|
||||
|
||||
func TestDraftConfigNoRoPELayers(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, field string
|
||||
valid bool
|
||||
}{
|
||||
{"omitted", "", true},
|
||||
{"empty", `,"no_rope_layers":[]`, true},
|
||||
{"first", `,"no_rope_layers":[0]`, true},
|
||||
{"last", `,"no_rope_layers":[1]`, true},
|
||||
{"unordered", `,"no_rope_layers":[1,0]`, true},
|
||||
{"negative", `,"no_rope_layers":[-1]`, false},
|
||||
{"past_last", `,"no_rope_layers":[2]`, false},
|
||||
{"invalid_after_valid", `,"no_rope_layers":[0,2]`, false},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := parseConfig(draftRotaryTestConfig(tt.field))
|
||||
if tt.valid {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatalf("parseConfig accepted invalid %s", tt.field)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no_rope_layers") {
|
||||
t.Fatalf("error %q does not identify no_rope_layers", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzDraftConfig(f *testing.F) {
|
||||
f.Add([]byte(`{"hidden_size":32,"num_hidden_layers":2,"num_attention_heads":4,"num_key_value_heads":2,"head_dim":8,"is_causal":false,"vocab_size":64,"dflash_config":{"block_size":8,"mask_token_id":7,"target_layer_ids":[1,3],"conv_group_size":16,"conv_kernel_size":2,"selector_rank":4,"selector_top_k":2}}`))
|
||||
f.Add([]byte(`{"num_hidden_layers":-1}`))
|
||||
|
|
|
|||
156
mlxrunner/model/ripple/forward.go
Normal file
156
mlxrunner/model/ripple/forward.go
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
package ripple
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"github.com/ollama/ollama/mlx"
|
||||
"github.com/ollama/ollama/mlxrunner/batch"
|
||||
"github.com/ollama/ollama/mlxrunner/cache"
|
||||
"github.com/ollama/ollama/mlxrunner/nn"
|
||||
)
|
||||
|
||||
func (m *Model) Forward(b *batch.Batch, caches []cache.Cache) (hidden, auxHidden *mlx.Array) {
|
||||
if caches != nil && len(caches) < len(m.Layers) {
|
||||
panic("ripple: cache count must cover target layers")
|
||||
}
|
||||
if caches != nil && b.InputIDs.Dim(0) != 1 {
|
||||
panic("ripple: cached batches require one sequence")
|
||||
}
|
||||
h := scaleEmbedding(m.EmbedTokens.Forward(b.InputIDs), m.EmbedScale)
|
||||
seed := h
|
||||
positions := mlx.Add(mlx.Reshape(mlx.FromValues(b.SeqOffsets, len(b.SeqOffsets)), int32(len(b.SeqOffsets)), 1), mlx.Arange(0, float64(b.InputIDs.Dim(1)), 1, mlx.DTypeInt32))
|
||||
cos, sin := rotaryTables(positions, m.HeadDim, m.RopeTheta, h.DType())
|
||||
var features []*mlx.Array
|
||||
for i, layer := range m.Layers {
|
||||
var c cache.Cache
|
||||
if caches != nil {
|
||||
c = caches[i]
|
||||
}
|
||||
h = layer.Forward(h, seed, b, c, cos, sin, m.LayerTypes[i] == "sliding_attention", m.Config)
|
||||
if next := len(features); next < len(m.auxHiddenLayers) && i == m.auxHiddenLayers[next] {
|
||||
features = append(features, h)
|
||||
}
|
||||
}
|
||||
h = rmsNorm(h, m.Norm, m.RMSNormEps)
|
||||
if len(features) > 0 {
|
||||
return h, mlx.Concatenate(features, -1)
|
||||
}
|
||||
return h, h
|
||||
}
|
||||
|
||||
func (l *Layer) Forward(x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array {
|
||||
u := rmsNorm(x, l.InputNorm, cfg.RMSNormEps)
|
||||
a := l.Attention.Forward(u, seed, b, c, cos, sin, local, cfg)
|
||||
h := mlx.Add(x, rmsNorm(a, l.PostAttentionNorm, cfg.RMSNormEps))
|
||||
f := l.MLP.Forward(rmsNorm(h, l.PreFFNNorm, cfg.RMSNormEps), cfg.FFNClampThreshold)
|
||||
return mlx.Add(h, rmsNorm(f, l.PostFFNNorm, cfg.RMSNormEps))
|
||||
}
|
||||
|
||||
func (a *Attention) Forward(x, seed *mlx.Array, b *batch.Batch, c cache.Cache, cos, sin *mlx.Array, local bool, cfg *Config) *mlx.Array {
|
||||
B, L := int32(x.Dim(0)), int32(x.Dim(1))
|
||||
q := mlx.Reshape(a.QProj.Forward(x), B, L, cfg.NumAttentionHeads, cfg.HeadDim)
|
||||
k := mlx.Reshape(a.KProj.Forward(x), B, L, cfg.NumKeyValueHeads, cfg.HeadDim)
|
||||
v := mlx.Reshape(a.VProj.Forward(blendValues(seed, x, a.ValueResLambda)), B, L, cfg.NumKeyValueHeads, cfg.HeadDim)
|
||||
q = mlx.Transpose(rmsNorm(q, a.QNorm, cfg.RMSNormEps), 0, 2, 1, 3)
|
||||
k = mlx.Transpose(rmsNorm(k, a.KNorm, cfg.RMSNormEps), 0, 2, 1, 3)
|
||||
v = mlx.Transpose(v, 0, 2, 1, 3)
|
||||
if local {
|
||||
q, k = applyRotary(q, cos, sin), applyRotary(k, cos, sin)
|
||||
}
|
||||
mask := nn.CausalMask()
|
||||
var kv nn.SDPAOption
|
||||
if c != nil {
|
||||
kv = nn.WithKVHistory(c.(cache.Attention).Update(b, k, v))
|
||||
} else {
|
||||
kv = nn.WithKV(k, v, b.SeqQueryLens)
|
||||
if local {
|
||||
mask = mask.Intersect(nn.SlidingWindowMask(b, int(L), cfg.SlidingWindow, k.DType()))
|
||||
}
|
||||
}
|
||||
out := nn.ScaledDotProductAttention(b, q, cfg.Scale, kv, nn.WithMask(mask))
|
||||
out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim)
|
||||
return a.OProj.Forward(gatedAttention(out, a.GProj.Forward(x)))
|
||||
}
|
||||
|
||||
// MLP is the clipped SwiGLU feed-forward block.
|
||||
type MLP struct {
|
||||
GateProj, UpProj, DownProj nn.LinearLayer
|
||||
}
|
||||
|
||||
var ffnActivation = mlx.Compile2("SwiGLUF32", func(gate, up *mlx.Array) *mlx.Array {
|
||||
f := gate.AsType(mlx.DTypeFloat32)
|
||||
silu := mlx.Mul(f, mlx.Sigmoid(f)).AsType(gate.DType())
|
||||
return mlx.Mul(silu, up)
|
||||
}, mlx.Shapeless())
|
||||
|
||||
var clippedFFNActivation = mlx.Compile3("ClippedSwiGLUF32", func(gate, up, threshold *mlx.Array) *mlx.Array {
|
||||
gate = mlx.Minimum(mlx.Maximum(gate, mlx.FromValue(float32(math.Inf(-1)))), threshold).AsType(gate.DType())
|
||||
up = mlx.Minimum(mlx.Maximum(up, threshold.Negative()), threshold).AsType(up.DType())
|
||||
return ffnActivation(gate, up)
|
||||
}, mlx.Shapeless())
|
||||
|
||||
func (m *MLP) Forward(x *mlx.Array, threshold *float32) *mlx.Array {
|
||||
gate, up := m.GateProj.Forward(x), m.UpProj.Forward(x)
|
||||
var activated *mlx.Array
|
||||
if threshold != nil && *threshold > 0 {
|
||||
activated = clippedFFNActivation(gate, up, mlx.FromValue(*threshold))
|
||||
} else {
|
||||
activated = ffnActivation(gate, up)
|
||||
}
|
||||
return m.DownProj.Forward(activated)
|
||||
}
|
||||
|
||||
func rotaryTables(positions *mlx.Array, dim int32, theta float32, dtype mlx.DType) (*mlx.Array, *mlx.Array) {
|
||||
inv := make([]float32, dim/2)
|
||||
for i := range inv {
|
||||
inv[i] = 1 / float32(math.Pow(float64(theta), float64(float32(2*i)/float32(dim))))
|
||||
}
|
||||
freq := mlx.Mul(mlx.ExpandDims(positions.AsType(mlx.DTypeFloat32), -1), mlx.FromValues(inv, len(inv)))
|
||||
freq = mlx.Concatenate([]*mlx.Array{freq, freq}, -1)
|
||||
return mlx.Cos(freq).AsType(dtype), mlx.Sin(freq).AsType(dtype)
|
||||
}
|
||||
|
||||
func applyRotary(x, cos, sin *mlx.Array) *mlx.Array {
|
||||
f := x.AsType(mlx.DTypeFloat32)
|
||||
half := x.Dim(-1) / 2
|
||||
first := f.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(0, half))
|
||||
second := f.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(half, x.Dim(-1)))
|
||||
rotated := mlx.Concatenate([]*mlx.Array{second.Negative(), first}, -1)
|
||||
c := mlx.ExpandDims(cos.AsType(mlx.DTypeFloat32), 1)
|
||||
s := mlx.ExpandDims(sin.AsType(mlx.DTypeFloat32), 1)
|
||||
return mlx.Add(mlx.Mul(f, c), mlx.Mul(rotated, s)).AsType(x.DType())
|
||||
}
|
||||
|
||||
func rmsNorm(x, weight *mlx.Array, eps float32) *mlx.Array {
|
||||
// Round the normalized activation before multiplying the learned weight.
|
||||
return mlx.Mul(mlx.RMSNormFn(x, nil, eps), weight)
|
||||
}
|
||||
|
||||
func scaleEmbedding(x *mlx.Array, scale *float64) *mlx.Array {
|
||||
if scale == nil {
|
||||
return x
|
||||
}
|
||||
return mlx.MulScalar(x.AsType(mlx.DTypeFloat32), float32(*scale)).AsType(x.DType())
|
||||
}
|
||||
|
||||
func attentionGate(x *mlx.Array) *mlx.Array {
|
||||
return mlx.Sigmoid(x.AsType(mlx.DTypeFloat32)).AsType(x.DType())
|
||||
}
|
||||
|
||||
var gatedAttention = mlx.Compile2("AttentionOutputGate", func(out, gate *mlx.Array) *mlx.Array {
|
||||
return mlx.Mul(out, attentionGate(gate))
|
||||
}, mlx.Shapeless())
|
||||
|
||||
func blendValues(seed, x, lambda *mlx.Array) *mlx.Array {
|
||||
// Sigmoid rounds to the parameter dtype; lerp has one activation rounding.
|
||||
a := mlx.Sigmoid(lambda.AsType(mlx.DTypeFloat32)).AsType(lambda.DType()).AsType(x.DType()).AsType(mlx.DTypeFloat32)
|
||||
return interpolateValues(seed, x, a)
|
||||
}
|
||||
|
||||
var interpolateValues = mlx.Compile3("ValueBlend", func(seed, x, a *mlx.Array) *mlx.Array {
|
||||
start, end := seed.AsType(mlx.DTypeFloat32), x.AsType(mlx.DTypeFloat32)
|
||||
delta := mlx.Sub(end, start)
|
||||
low := mlx.Add(start, mlx.Mul(a, delta))
|
||||
high := mlx.Sub(end, mlx.Mul(mlx.Sub(mlx.FromValue(float32(1)), a), delta))
|
||||
return mlx.Where(a.Less(mlx.FromValue(float32(0.5))), low, high).AsType(x.DType())
|
||||
}, mlx.Shapeless())
|
||||
318
mlxrunner/model/ripple/ripple.go
Normal file
318
mlxrunner/model/ripple/ripple.go
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
// Package ripple implements the Ricursive transformer.
|
||||
package ripple
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/ollama/ollama/mlx"
|
||||
"github.com/ollama/ollama/mlx/quant"
|
||||
"github.com/ollama/ollama/mlxrunner/cache"
|
||||
"github.com/ollama/ollama/mlxrunner/nn"
|
||||
"github.com/ollama/ollama/mlxrunner/tokenizer"
|
||||
|
||||
"github.com/ollama/ollama/mlxrunner/model"
|
||||
)
|
||||
|
||||
func init() { model.Register("RicursiveForCausalLM", newModel) }
|
||||
|
||||
// Model retains all loaded arrays; value-residual seeds belong to Forward.
|
||||
type Model struct {
|
||||
*Config
|
||||
EmbedTokens nn.EmbeddingLayer
|
||||
Layers []*Layer
|
||||
Norm *mlx.Array
|
||||
LMHead nn.LinearLayer
|
||||
tok *tokenizer.Tokenizer
|
||||
auxHiddenLayers []int
|
||||
draftTensorPrefix string
|
||||
}
|
||||
|
||||
type Layer struct {
|
||||
Attention *Attention
|
||||
MLP *MLP
|
||||
InputNorm, PostAttentionNorm, PreFFNNorm, PostFFNNorm *mlx.Array
|
||||
}
|
||||
|
||||
type Attention struct {
|
||||
QProj, KProj, VProj, OProj, GProj nn.LinearLayer
|
||||
QNorm, KNorm, ValueResLambda *mlx.Array
|
||||
}
|
||||
|
||||
func newModel(root *model.Root) (model.Model, error) {
|
||||
cfg, err := loadConfig(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
configData, err := root.Manifest.ReadConfig("config.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tokData, err := root.Manifest.ReadConfig("tokenizer.json")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load tokenizer: %w", err)
|
||||
}
|
||||
tc := &tokenizer.TokenizerConfig{ConfigJSON: configData}
|
||||
tc.GenerationConfigJSON, _ = root.Manifest.ReadConfig("generation_config.json")
|
||||
tc.TokenizerConfigJSON, _ = root.Manifest.ReadConfig("tokenizer_config.json")
|
||||
tok, err := tokenizer.LoadFromBytesWithConfig(tokData, tc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse tokenizer: %w", err)
|
||||
}
|
||||
m := &Model{Config: cfg, Layers: make([]*Layer, cfg.NumHiddenLayers), tok: tok}
|
||||
if root.Draft != nil {
|
||||
m.draftTensorPrefix = root.Draft.TensorPrefix
|
||||
if m.draftTensorPrefix == "" {
|
||||
m.draftTensorPrefix = "draft."
|
||||
}
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) Tokenizer() *tokenizer.Tokenizer { return m.tok }
|
||||
func (m *Model) MaxContextLength() int { return int(m.MaxPositionEmbeddings) }
|
||||
func (m *Model) Unembed(x *mlx.Array) *mlx.Array { return m.LMHead.Forward(x) }
|
||||
func (m *Model) NumLayers() int { return len(m.Layers) }
|
||||
func (m *Model) SetAuxHiddenLayers(layers []int) { m.auxHiddenLayers = slices.Clone(layers) }
|
||||
func (m *Model) TokenEmbeddings(ids *mlx.Array) *mlx.Array { return m.EmbedTokens.Forward(ids) }
|
||||
func (m *Model) RawLogits(hidden *mlx.Array) *mlx.Array { return m.LMHead.Forward(hidden) }
|
||||
|
||||
// DraftWindow bounds draft attention to its training context.
|
||||
func (*Model) DraftWindow() int { return 16384 }
|
||||
|
||||
func (m *Model) NewCaches() []cache.Cache {
|
||||
out := make([]cache.Cache, len(m.LayerTypes))
|
||||
for i, kind := range m.LayerTypes {
|
||||
if kind == "sliding_attention" {
|
||||
out[i] = cache.NewRotatingKVCache(m.SlidingWindow)
|
||||
} else {
|
||||
out[i] = cache.NewKVCache()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Model) weightShapes() map[string][]int {
|
||||
h, d, f, v := int(m.HiddenSize), int(m.HeadDim), int(m.IntermediateSize), int(m.VocabSize)
|
||||
q, k := int(m.NumAttentionHeads)*d, int(m.NumKeyValueHeads)*d
|
||||
shapes := map[string][]int{"model.embed_tokens.weight": {v, h}, "lm_head.weight": {v, h}, "model.norm.weight": {h}}
|
||||
for i := range m.NumHiddenLayers {
|
||||
p := fmt.Sprintf("model.layers.%d.", i)
|
||||
for name, shape := range map[string][]int{
|
||||
"input_layernorm.weight": {h}, "post_attention_layernorm.weight": {h}, "pre_feedforward_layernorm.weight": {h}, "post_feedforward_layernorm.weight": {h},
|
||||
"self_attn.q_proj.weight": {q, h}, "self_attn.k_proj.weight": {k, h}, "self_attn.v_proj.weight": {k, h}, "self_attn.o_proj.weight": {h, q}, "self_attn.g_proj.weight": {q, h},
|
||||
"self_attn.q_norm.weight": {d}, "self_attn.k_norm.weight": {d}, "self_attn.value_res_lambda": {1},
|
||||
"mlp.gate_proj.weight": {f, h}, "mlp.up_proj.weight": {f, h}, "mlp.down_proj.weight": {h, f},
|
||||
} {
|
||||
shapes[p+name] = shape
|
||||
}
|
||||
}
|
||||
return shapes
|
||||
}
|
||||
|
||||
func floatTensor(a *mlx.Array) bool {
|
||||
return a.DType() == mlx.DTypeFloat32 || a.DType() == mlx.DTypeBFloat16 || a.DType() == mlx.DTypeFloat16
|
||||
}
|
||||
|
||||
func (m *Model) validateWeights(tensors map[string]*mlx.Array) error {
|
||||
allowed := make(map[string]bool)
|
||||
for name, shape := range m.weightShapes() {
|
||||
w := tensors[name]
|
||||
if w == nil {
|
||||
return fmt.Errorf("missing weight: %s", name)
|
||||
}
|
||||
allowed[name] = true
|
||||
scales := tensors[name+"_scale"]
|
||||
if scales == nil {
|
||||
if !floatTensor(w) || !slices.Equal(w.Dims(), shape) {
|
||||
return fmt.Errorf("invalid weight %s: dtype %v shape %v, want float %v", name, w.DType(), w.Dims(), shape)
|
||||
}
|
||||
if m.TensorQuant[name] != nil {
|
||||
return fmt.Errorf("missing scale for quantized weight %s", name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(shape) != 2 {
|
||||
return fmt.Errorf("cannot quantize %s", name)
|
||||
}
|
||||
if tq := m.TensorQuant[name]; tq != nil && quant.Canonical(tq.QuantType) == "" {
|
||||
return fmt.Errorf("unsupported quantization for %s: %q", name, tq.QuantType)
|
||||
}
|
||||
gs, bits, mode := model.ResolveLinearQuantParams(m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant, name, w, scales)
|
||||
if gs <= 0 || (bits != 4 && bits != 8) || (mode != "affine" && mode != "nvfp4" && mode != "mxfp4" && mode != "mxfp8") {
|
||||
return fmt.Errorf("invalid quantization for %s", name)
|
||||
}
|
||||
if (mode == "nvfp4" && (gs != 16 || bits != 4)) || (mode == "mxfp4" && (gs != 32 || bits != 4)) || (mode == "mxfp8" && (gs != 32 || bits != 8)) {
|
||||
return fmt.Errorf("invalid quantization group/bits for %s", name)
|
||||
}
|
||||
packed, ss := []int{shape[0], shape[1] * bits / 32}, []int{shape[0], shape[1] / gs}
|
||||
if shape[1]%gs != 0 || shape[1]*bits%32 != 0 || w.DType() != mlx.DTypeUint32 || !slices.Equal(w.Dims(), packed) || !slices.Equal(scales.Dims(), ss) {
|
||||
return fmt.Errorf("invalid quantized shape for %s: weight %v scale %v, want %v %v", name, w.Dims(), scales.Dims(), packed, ss)
|
||||
}
|
||||
allowed[name+"_scale"] = true
|
||||
if mode == "affine" {
|
||||
qb := tensors[name+"_qbias"]
|
||||
if !floatTensor(scales) || qb == nil || !floatTensor(qb) || !slices.Equal(qb.Dims(), ss) {
|
||||
return fmt.Errorf("invalid affine scale/bias for %s", name)
|
||||
}
|
||||
allowed[name+"_qbias"] = true
|
||||
} else if scales.DType() != mlx.DTypeUint8 {
|
||||
return fmt.Errorf("invalid packed scale dtype for %s", name)
|
||||
}
|
||||
for _, suffix := range []string{".global_scale", "_scale_2", ".input_scale", ".input_global_scale"} {
|
||||
if a := tensors[name+suffix]; a != nil {
|
||||
if mode != "nvfp4" || !floatTensor(a) || a.Size() != 1 {
|
||||
return fmt.Errorf("invalid %s%s", name, suffix)
|
||||
}
|
||||
allowed[name+suffix] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for name := range tensors {
|
||||
if !allowed[name] && !(m.draftTensorPrefix != "" && strings.HasPrefix(name, m.draftTensorPrefix)) {
|
||||
return fmt.Errorf("unexpected weight: %s", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {
|
||||
if err := m.validateWeights(tensors); err != nil {
|
||||
return err
|
||||
}
|
||||
linears := model.NewLinearFactory(tensors, m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant)
|
||||
m.EmbedTokens = model.MakeEmbeddingLayer(tensors, "model.embed_tokens", m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant)
|
||||
m.LMHead = linears.Make("lm_head")
|
||||
m.Norm = tensors["model.norm.weight"]
|
||||
for i := range m.Layers {
|
||||
p := fmt.Sprintf("model.layers.%d.", i)
|
||||
m.Layers[i] = &Layer{
|
||||
InputNorm: tensors[p+"input_layernorm.weight"], PostAttentionNorm: tensors[p+"post_attention_layernorm.weight"], PreFFNNorm: tensors[p+"pre_feedforward_layernorm.weight"], PostFFNNorm: tensors[p+"post_feedforward_layernorm.weight"],
|
||||
Attention: &Attention{QProj: linears.Make(p + "self_attn.q_proj"), KProj: linears.Make(p + "self_attn.k_proj"), VProj: linears.Make(p + "self_attn.v_proj"), OProj: linears.Make(p + "self_attn.o_proj"), GProj: linears.Make(p + "self_attn.g_proj"), QNorm: tensors[p+"self_attn.q_norm.weight"], KNorm: tensors[p+"self_attn.k_norm.weight"], ValueResLambda: tensors[p+"self_attn.value_res_lambda"]},
|
||||
MLP: &MLP{GateProj: linears.Make(p + "mlp.gate_proj"), UpProj: linears.Make(p + "mlp.up_proj"), DownProj: linears.Make(p + "mlp.down_proj")},
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
HiddenSize int32 `json:"hidden_size"`
|
||||
IntermediateSize int32 `json:"intermediate_size"`
|
||||
NumHiddenLayers int32 `json:"num_hidden_layers"`
|
||||
NumAttentionHeads int32 `json:"num_attention_heads"`
|
||||
NumKeyValueHeads int32 `json:"num_key_value_heads"`
|
||||
HeadDim int32 `json:"head_dim"`
|
||||
VocabSize int32 `json:"vocab_size"`
|
||||
MaxPositionEmbeddings int32 `json:"max_position_embeddings"`
|
||||
RMSNormEps float32 `json:"rms_norm_eps"`
|
||||
RopeTheta float32 `json:"rope_theta"`
|
||||
SlidingWindow int `json:"sliding_window"`
|
||||
LayerTypes []string `json:"layer_types"`
|
||||
HiddenAct string `json:"hidden_act"`
|
||||
UseGatedAttention bool `json:"use_gated_attention"`
|
||||
GateFullPrecision bool `json:"gate_full_precision"`
|
||||
UseValRes bool `json:"use_valres"`
|
||||
TieWordEmbeddings bool `json:"tie_word_embeddings"`
|
||||
AttentionBias bool `json:"attention_bias"`
|
||||
MLPBias bool `json:"mlp_bias"`
|
||||
FFNClampThreshold *float32 `json:"ffn_clamp_threshold"`
|
||||
EmbedScale *float64 `json:"embed_scale"`
|
||||
RopeScaling json.RawMessage `json:"rope_scaling"`
|
||||
RopeParameters json.RawMessage `json:"rope_parameters"`
|
||||
|
||||
QuantGroupSize int `json:"-"`
|
||||
QuantBits int `json:"-"`
|
||||
QuantMode string `json:"-"`
|
||||
TensorQuant map[string]*model.TensorQuantInfo `json:"-"`
|
||||
Scale float32 `json:"-"`
|
||||
}
|
||||
|
||||
func loadConfig(root *model.Root) (*Config, error) {
|
||||
data, err := root.Manifest.ReadConfig("config.json")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
cfg := &Config{HiddenAct: "silu", RMSNormEps: 1e-6, RopeTheta: 10000, SlidingWindow: 1024, UseGatedAttention: true, GateFullPrecision: true}
|
||||
if err := json.Unmarshal(data, cfg); err != nil {
|
||||
return nil, fmt.Errorf("parse config: %w", err)
|
||||
}
|
||||
for _, v := range []struct {
|
||||
name string
|
||||
value int32
|
||||
}{
|
||||
{"hidden_size", cfg.HiddenSize},
|
||||
{"intermediate_size", cfg.IntermediateSize},
|
||||
{"num_hidden_layers", cfg.NumHiddenLayers},
|
||||
{"num_attention_heads", cfg.NumAttentionHeads},
|
||||
{"num_key_value_heads", cfg.NumKeyValueHeads},
|
||||
{"head_dim", cfg.HeadDim},
|
||||
{"vocab_size", cfg.VocabSize},
|
||||
{"max_position_embeddings", cfg.MaxPositionEmbeddings},
|
||||
} {
|
||||
if v.value <= 0 {
|
||||
return nil, fmt.Errorf("invalid %s: %d", v.name, v.value)
|
||||
}
|
||||
}
|
||||
if cfg.HeadDim%2 != 0 {
|
||||
return nil, fmt.Errorf("head_dim must be even")
|
||||
}
|
||||
if cfg.NumAttentionHeads%cfg.NumKeyValueHeads != 0 {
|
||||
return nil, fmt.Errorf("num_attention_heads must be divisible by num_key_value_heads")
|
||||
}
|
||||
if int64(cfg.NumAttentionHeads)*int64(cfg.HeadDim) > math.MaxInt32 {
|
||||
return nil, fmt.Errorf("num_attention_heads * head_dim overflows")
|
||||
}
|
||||
if cfg.RMSNormEps <= 0 {
|
||||
return nil, fmt.Errorf("rms_norm_eps must be positive")
|
||||
}
|
||||
if cfg.RopeTheta <= 0 {
|
||||
return nil, fmt.Errorf("rope_theta must be positive")
|
||||
}
|
||||
if cfg.SlidingWindow <= 0 {
|
||||
return nil, fmt.Errorf("sliding_window must be positive")
|
||||
}
|
||||
if len(cfg.LayerTypes) != int(cfg.NumHiddenLayers) {
|
||||
return nil, fmt.Errorf("layer_types must have num_hidden_layers entries")
|
||||
}
|
||||
for i, kind := range cfg.LayerTypes {
|
||||
if kind != "sliding_attention" && kind != "full_attention" {
|
||||
return nil, fmt.Errorf("unsupported layer_types[%d]: %q", i, kind)
|
||||
}
|
||||
}
|
||||
if cfg.HiddenAct != "silu" {
|
||||
return nil, fmt.Errorf("unsupported hidden_act: %q", cfg.HiddenAct)
|
||||
}
|
||||
for _, v := range []struct {
|
||||
name string
|
||||
unsupported bool
|
||||
}{
|
||||
{"use_gated_attention", !cfg.UseGatedAttention},
|
||||
{"gate_full_precision", !cfg.GateFullPrecision},
|
||||
{"use_valres", !cfg.UseValRes},
|
||||
{"tie_word_embeddings", cfg.TieWordEmbeddings},
|
||||
{"attention_bias", cfg.AttentionBias},
|
||||
{"mlp_bias", cfg.MLPBias},
|
||||
{"rope_scaling", len(cfg.RopeScaling) > 0 && string(cfg.RopeScaling) != "null"},
|
||||
{"rope_parameters", len(cfg.RopeParameters) > 0 && string(cfg.RopeParameters) != "null"},
|
||||
} {
|
||||
if v.unsupported {
|
||||
return nil, fmt.Errorf("unsupported %s", v.name)
|
||||
}
|
||||
}
|
||||
if cfg.FFNClampThreshold != nil && *cfg.FFNClampThreshold < 0 {
|
||||
return nil, fmt.Errorf("ffn_clamp_threshold must be nonnegative")
|
||||
}
|
||||
if cfg.EmbedScale != nil && (math.IsInf(*cfg.EmbedScale, 0) || math.IsNaN(*cfg.EmbedScale) || math.Abs(*cfg.EmbedScale) > math.MaxFloat32) {
|
||||
return nil, fmt.Errorf("invalid embed_scale")
|
||||
}
|
||||
cfg.Scale = float32(1 / math.Sqrt(float64(cfg.HeadDim)))
|
||||
cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode = model.QuantizationParams(root.QuantType())
|
||||
if gs := root.GroupSize(); gs > 0 {
|
||||
cfg.QuantGroupSize = gs
|
||||
}
|
||||
cfg.TensorQuant = root.AllTensorQuant()
|
||||
return cfg, nil
|
||||
}
|
||||
1393
mlxrunner/model/ripple/ripple_test.go
Normal file
1393
mlxrunner/model/ripple/ripple_test.go
Normal file
File diff suppressed because it is too large
Load diff
681
mlxrunner/ripple_test.go
Normal file
681
mlxrunner/ripple_test.go
Normal file
|
|
@ -0,0 +1,681 @@
|
|||
package mlxrunner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ollama/ollama/api"
|
||||
"github.com/ollama/ollama/llm"
|
||||
"github.com/ollama/ollama/manifest"
|
||||
"github.com/ollama/ollama/mlx"
|
||||
"github.com/ollama/ollama/mlx/mlxtest"
|
||||
"github.com/ollama/ollama/mlxrunner/batch"
|
||||
"github.com/ollama/ollama/mlxrunner/cache"
|
||||
"github.com/ollama/ollama/mlxrunner/model"
|
||||
"github.com/ollama/ollama/mlxrunner/model/ripple"
|
||||
"github.com/ollama/ollama/mlxrunner/sample"
|
||||
modeltypes "github.com/ollama/ollama/types/model"
|
||||
)
|
||||
|
||||
type rippleRunnerTensor struct {
|
||||
data []float32
|
||||
shape []int
|
||||
}
|
||||
|
||||
func rippleRunnerFixture() (map[string]any, map[string]rippleRunnerTensor) {
|
||||
cfg := map[string]any{
|
||||
"architectures": []string{"RicursiveForCausalLM"},
|
||||
"vocab_size": 19, "hidden_size": 8, "intermediate_size": 12,
|
||||
"num_hidden_layers": 3, "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 4,
|
||||
"layer_types": []string{"sliding_attention", "full_attention", "sliding_attention"},
|
||||
"sliding_window": 3, "rms_norm_eps": 1e-6, "rope_theta": 10000.0,
|
||||
"use_valres": true, "use_gated_attention": true, "gate_full_precision": true,
|
||||
"ffn_clamp_threshold": 20.0, "embed_scale": 71.55417527999327,
|
||||
"max_position_embeddings": 64, "tie_word_embeddings": false,
|
||||
}
|
||||
shapes := map[string][]int{
|
||||
"model.embed_tokens.weight": {19, 8}, "lm_head.weight": {19, 8}, "model.norm.weight": {8},
|
||||
}
|
||||
for i := range 3 {
|
||||
prefix := fmt.Sprintf("model.layers.%d.", i)
|
||||
for name, shape := range map[string][]int{
|
||||
"input_layernorm.weight": {8}, "post_attention_layernorm.weight": {8},
|
||||
"pre_feedforward_layernorm.weight": {8}, "post_feedforward_layernorm.weight": {8},
|
||||
"self_attn.q_proj.weight": {16, 8}, "self_attn.k_proj.weight": {8, 8},
|
||||
"self_attn.v_proj.weight": {8, 8}, "self_attn.o_proj.weight": {8, 16},
|
||||
"self_attn.g_proj.weight": {16, 8}, "self_attn.q_norm.weight": {4},
|
||||
"self_attn.k_norm.weight": {4}, "self_attn.value_res_lambda": {1},
|
||||
"mlp.gate_proj.weight": {12, 8}, "mlp.up_proj.weight": {12, 8}, "mlp.down_proj.weight": {8, 12},
|
||||
} {
|
||||
shapes[prefix+name] = shape
|
||||
}
|
||||
}
|
||||
weights := make(map[string]rippleRunnerTensor)
|
||||
for name, shape := range shapes {
|
||||
size := 1
|
||||
for _, dim := range shape {
|
||||
size *= dim
|
||||
}
|
||||
values := make([]float32, size)
|
||||
hash := sha256.Sum256([]byte(name))
|
||||
phase := int(binary.BigEndian.Uint32(hash[:4]) % 101)
|
||||
if strings.HasSuffix(name, "value_res_lambda") {
|
||||
layer := strings.Split(name, ".")[2]
|
||||
values[0] = map[string]float32{"0": -1.25, "1": 0.75, "2": 1.5}[layer]
|
||||
} else {
|
||||
for i := range values {
|
||||
v := float32((i*37+phase)%101-50) / 50
|
||||
switch {
|
||||
case strings.Contains(name, "norm.weight"):
|
||||
scaled := v * 0.375
|
||||
v = 1 + scaled
|
||||
case strings.Contains(name, "embed_tokens"):
|
||||
v *= 0.045
|
||||
case strings.Contains(name, ".mlp.gate_proj") || strings.Contains(name, ".mlp.up_proj"):
|
||||
v *= 18
|
||||
default:
|
||||
v *= 0.3
|
||||
}
|
||||
values[i] = v
|
||||
}
|
||||
}
|
||||
weights[name] = rippleRunnerTensor{data: values, shape: shape}
|
||||
}
|
||||
return cfg, weights
|
||||
}
|
||||
|
||||
func writeRippleRunnerModel(t *testing.T) string {
|
||||
t.Helper()
|
||||
mlxtest.SkipIfUnavailable(t)
|
||||
dir := t.TempDir()
|
||||
t.Setenv("OLLAMA_MODELS", dir)
|
||||
fixtureConfig, fixtureWeights := rippleRunnerFixture()
|
||||
config, err := json.Marshal(fixtureConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vocab := make(map[string]int)
|
||||
for i := range fixtureConfig["vocab_size"].(int) {
|
||||
vocab[string(rune('a'+i))] = i
|
||||
}
|
||||
tokenizer, err := json.Marshal(map[string]any{
|
||||
"model": map[string]any{"type": "BPE", "vocab": vocab, "merges": []string{}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := "ripple-runner-test"
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
mlx.Scoped(func() {
|
||||
weights := make(map[string]*mlx.Array)
|
||||
for name, tensor := range fixtureWeights {
|
||||
weights[name] = mlx.FromValues(tensor.data, tensor.shape...)
|
||||
}
|
||||
path := filepath.Join(dir, "model.safetensors")
|
||||
if err := mlx.SaveSafetensors(path, weights); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
tensors, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tensors.Name = "model.safetensors"
|
||||
layers := []manifest.Layer{tensors}
|
||||
for name, data := range map[string][]byte{"config.json": config, "tokenizer.json": tokenizer} {
|
||||
layer, err := manifest.NewLayer(bytes.NewReader(data), manifest.MediaTypeImageJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
layer.Name = name
|
||||
layers = append(layers, layer)
|
||||
}
|
||||
meta, err := manifest.NewLayer(strings.NewReader(`{"model_format":"safetensors","model_family":"ripple"}`), manifest.MediaTypeImageConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), meta, layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
return name
|
||||
}
|
||||
|
||||
func loadRippleRunner(t *testing.T, name string) *Runner {
|
||||
t.Helper()
|
||||
r := &Runner{mlxThread: mlxtest.Worker(t)}
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
if err := r.Load(name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := r.Model.(*ripple.Model); !ok {
|
||||
t.Fatalf("loaded %T, want *ripple.Model", r.Model)
|
||||
}
|
||||
})
|
||||
t.Cleanup(func() {
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
r.cache.close()
|
||||
r.Close()
|
||||
})
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
func rippleRunnerMemory(t *testing.T) (active int) {
|
||||
t.Helper()
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
mlx.DefaultStream().Synchronize()
|
||||
mlx.ClearCache()
|
||||
active = mlx.ActiveMemory()
|
||||
})
|
||||
return active
|
||||
}
|
||||
|
||||
func rippleRunnerRequest(t *testing.T, r *Runner, prompt string, count int) Request {
|
||||
t.Helper()
|
||||
request := Request{
|
||||
CompletionRequest: CompletionRequest{Prompt: prompt, Options: api.Options{NumPredict: count}},
|
||||
Pipeline: r.TextGenerationPipeline,
|
||||
SamplerOpts: sample.Options{
|
||||
Temperature: 0, RepeatLastN: 8, RepeatPenalty: 1.1,
|
||||
Seed: 1, UseSeed: true, Logprobs: true, TopLogprobs: 3,
|
||||
},
|
||||
Responses: make(chan CompletionResponse, count+1),
|
||||
}
|
||||
if err := r.Prepare(&request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(request.Tokens) != len(prompt) {
|
||||
t.Fatalf("tokenized %q into %d tokens, want %d", prompt, len(request.Tokens), len(prompt))
|
||||
}
|
||||
return request
|
||||
}
|
||||
|
||||
func runRippleRunnerRequest(t *testing.T, r *Runner, prompt string) (string, []llm.Logprob, int) {
|
||||
t.Helper()
|
||||
request := rippleRunnerRequest(t, r, prompt, 4)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
request.Ctx = ctx
|
||||
if err := r.runRequest(request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
close(request.Responses)
|
||||
var text strings.Builder
|
||||
var logprobs []llm.Logprob
|
||||
var final *CompletionResponse
|
||||
for response := range request.Responses {
|
||||
if response.Error != nil {
|
||||
t.Fatal(response.Error)
|
||||
}
|
||||
if response.Done {
|
||||
if final != nil {
|
||||
t.Fatal("duplicate final response")
|
||||
}
|
||||
final = &response
|
||||
}
|
||||
text.WriteString(response.Content)
|
||||
logprobs = append(logprobs, response.Logprobs...)
|
||||
}
|
||||
if final == nil || final.EvalCount != 4 || final.DoneReason != 1 || final.PromptEvalCount != len(request.Tokens) || final.PromptEvalCachedCount == nil {
|
||||
t.Fatalf("invalid final response: %+v", final)
|
||||
}
|
||||
if text.Len() != 4 || len(logprobs) != 4 {
|
||||
t.Fatalf("empty or incomplete model output: text=%q logprobs=%d", text.String(), len(logprobs))
|
||||
}
|
||||
for _, lp := range logprobs {
|
||||
if math.IsNaN(lp.Logprob) || math.IsInf(lp.Logprob, 0) || lp.Logprob >= 0 || len(lp.TopLogprobs) != 3 {
|
||||
t.Fatalf("invalid model log probability: %+v", lp)
|
||||
}
|
||||
if lp.TopLogprobs[0].Logprob == lp.TopLogprobs[2].Logprob {
|
||||
t.Fatalf("model produced uniform logits: %+v", lp)
|
||||
}
|
||||
}
|
||||
return text.String(), logprobs, *final.PromptEvalCachedCount
|
||||
}
|
||||
|
||||
func TestRippleRunnerLoadClose(t *testing.T) {
|
||||
name := writeRippleRunnerModel(t)
|
||||
warm := loadRippleRunner(t, name)
|
||||
mlxtest.Run(t, func(t *mlxtest.T) { warm.Close() })
|
||||
baseline := rippleRunnerMemory(t)
|
||||
for range 3 {
|
||||
r := loadRippleRunner(t, name)
|
||||
if loaded := rippleRunnerMemory(t); loaded <= baseline {
|
||||
t.Fatalf("load allocated no weights: loaded=%d baseline=%d", loaded, baseline)
|
||||
}
|
||||
mlxtest.Run(t, func(t *mlxtest.T) { r.Close() })
|
||||
if got := rippleRunnerMemory(t); got != baseline {
|
||||
t.Fatalf("load/close retained %d bytes above warmed baseline %d", got-baseline, baseline)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleRunnerRequests(t *testing.T) {
|
||||
name := writeRippleRunnerModel(t)
|
||||
cold, reused := loadRippleRunner(t, name), loadRippleRunner(t, name)
|
||||
const prompt = "ihgfedcba"
|
||||
wantText, wantLogprobs, cached := runRippleRunnerRequest(t, cold, prompt)
|
||||
if cached != 0 {
|
||||
t.Fatalf("fresh runner reused %d tokens", cached)
|
||||
}
|
||||
runRippleRunnerRequest(t, reused, "abcdefghi")
|
||||
gotText, gotLogprobs, cached := runRippleRunnerRequest(t, reused, prompt)
|
||||
// These cold forwards have identical chunk schedules; cache-hit forwards do not.
|
||||
if cached != 0 || gotText != wantText || !reflect.DeepEqual(gotLogprobs, wantLogprobs) {
|
||||
t.Fatalf("independent request inherited state: cached=%d text=%q want=%q", cached, gotText, wantText)
|
||||
}
|
||||
wantText, wantLogprobs, cached = runRippleRunnerRequest(t, reused, prompt)
|
||||
if cached <= 0 || cached >= len(prompt) {
|
||||
t.Fatalf("repeat reused %d tokens, want a nonempty prefix with a decode seed", cached)
|
||||
}
|
||||
baseline := rippleRunnerMemory(t)
|
||||
// Settled allocator readings can vary by a few small buffers between requests.
|
||||
const memorySlack = 256
|
||||
for range 16 {
|
||||
gotText, gotLogprobs, gotCached := runRippleRunnerRequest(t, reused, prompt)
|
||||
if gotCached != cached || gotText != wantText || !reflect.DeepEqual(gotLogprobs, wantLogprobs) {
|
||||
t.Fatal("identical cache-hit schedule changed the response")
|
||||
}
|
||||
if got := rippleRunnerMemory(t); got > baseline+memorySlack {
|
||||
t.Fatalf("repeated request retained %d bytes above warmed baseline %d", got-baseline, baseline)
|
||||
}
|
||||
}
|
||||
|
||||
request := rippleRunnerRequest(t, reused, prompt, 32)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
request.Ctx = ctx
|
||||
request.Responses = make(chan CompletionResponse)
|
||||
responses := make(chan []CompletionResponse, 1)
|
||||
go func() {
|
||||
var received []CompletionResponse
|
||||
for response := range request.Responses {
|
||||
received = append(received, response)
|
||||
cancel()
|
||||
}
|
||||
responses <- received
|
||||
}()
|
||||
err := reused.runRequest(request)
|
||||
close(request.Responses)
|
||||
received := <-responses
|
||||
if !errors.Is(err, context.Canceled) || len(received) == 0 || received[0].Content == "" {
|
||||
t.Fatalf("cancel after real output: err=%v responses=%+v", err, received)
|
||||
}
|
||||
for _, response := range received {
|
||||
if response.Done {
|
||||
t.Fatal("cancelled generation emitted a successful final response")
|
||||
}
|
||||
}
|
||||
// Retry through the same worker, sampler, and prefix-cache owner.
|
||||
runRippleRunnerRequest(t, reused, prompt)
|
||||
runRippleRunnerRequest(t, reused, prompt)
|
||||
baseline = rippleRunnerMemory(t)
|
||||
runRippleRunnerRequest(t, reused, prompt)
|
||||
if got := rippleRunnerMemory(t); got > baseline+memorySlack {
|
||||
t.Fatalf("post-cancellation request retained %d bytes", got-baseline)
|
||||
}
|
||||
}
|
||||
|
||||
func writeRippleDFlash2RunnerModel(t *testing.T) string {
|
||||
t.Helper()
|
||||
name := writeRippleRunnerModel(t)
|
||||
m, err := manifest.ParseNamedManifest(modeltypes.ParseName(name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const draftConfig = `{
|
||||
"architectures": ["DFlash2DraftModel"],
|
||||
"hidden_size": 8, "num_hidden_layers": 2, "intermediate_size": 16,
|
||||
"num_attention_heads": 2, "num_key_value_heads": 1, "head_dim": 4,
|
||||
"rms_norm_eps": 0.000001, "rope_theta": 10000,
|
||||
"vocab_size": 19, "num_target_layers": 3, "block_size": 8,
|
||||
"layer_types": ["sliding_attention", "full_attention"],
|
||||
"sliding_window": 4, "no_rope_layers": [1], "is_causal": false,
|
||||
"dflash_config": {
|
||||
"mask_token_id": 18, "target_layer_ids": [0, 2],
|
||||
"conv_group_size": 2, "conv_kernel_size": 4,
|
||||
"selector_rank": 4, "selector_top_k": 4
|
||||
}
|
||||
}`
|
||||
config, err := manifest.NewLayer(strings.NewReader(draftConfig), manifest.MediaTypeImageJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config.Name = "draft/config.json"
|
||||
layers := append(m.Layers, config)
|
||||
path := filepath.Join(t.TempDir(), "draft.safetensors")
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
mlx.Scoped(func() {
|
||||
weights := make(map[string]*mlx.Array)
|
||||
add := func(name string, dims ...int) {
|
||||
n, phase := 1, 0
|
||||
for _, dim := range dims {
|
||||
n *= dim
|
||||
}
|
||||
for _, c := range name {
|
||||
phase += int(c)
|
||||
}
|
||||
values := make([]float32, n)
|
||||
for i := range values {
|
||||
values[i] = float32((i*17+phase)%31-15) / 128
|
||||
if len(dims) == 1 {
|
||||
values[i] = 1
|
||||
}
|
||||
}
|
||||
weights["draft."+name] = mlx.FromValues(values, dims...)
|
||||
}
|
||||
add("fc.weight", 8, 2*8)
|
||||
add("hidden_norm.weight", 8)
|
||||
add("norm.weight", 8)
|
||||
for i := range 2 {
|
||||
prefix := fmt.Sprintf("layers.%d.", i)
|
||||
add(prefix+"input_layernorm.weight", 8)
|
||||
add(prefix+"post_attention_layernorm.weight", 8)
|
||||
add(prefix+"self_attn.q_norm.weight", 4)
|
||||
add(prefix+"self_attn.k_norm.weight", 4)
|
||||
add(prefix+"self_attn.q_proj.weight", 8, 8)
|
||||
add(prefix+"self_attn.k_proj.weight", 4, 8)
|
||||
add(prefix+"self_attn.v_proj.weight", 4, 8)
|
||||
add(prefix+"self_attn.o_proj.weight", 8, 8)
|
||||
add(prefix+"mlp.gate_proj.weight", 16, 8)
|
||||
add(prefix+"mlp.up_proj.weight", 16, 8)
|
||||
add(prefix+"mlp.down_proj.weight", 8, 16)
|
||||
for _, conv := range []string{"attention_conv", "mlp_conv"} {
|
||||
add(prefix+conv+".base_kernel", 2, 4, 8)
|
||||
add(prefix+conv+".kernel_projection.weight", 2*4*8/2, 8)
|
||||
}
|
||||
}
|
||||
add("candidate_selector.predecessor_codebook", 19, 4)
|
||||
add("candidate_selector.successor_codebook", 19, 4)
|
||||
add("candidate_selector.hidden_projection.weight", 4, 8)
|
||||
if err := mlx.SaveSafetensors(path, weights); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
tensors, err := manifest.NewLayer(f, manifest.MediaTypeImageTensor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tensors.Name = "draft/model.safetensors"
|
||||
layers = append(layers, tensors)
|
||||
metadata, err := json.Marshal(modeltypes.ConfigV2{
|
||||
ModelFormat: "safetensors", ModelFamily: "ripple",
|
||||
Draft: &modeltypes.Draft{
|
||||
ModelFormat: "safetensors", Architecture: "DFlash2DraftModel",
|
||||
TensorPrefix: "draft.", Config: "draft/config.json",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := manifest.NewLayer(bytes.NewReader(metadata), manifest.MediaTypeImageConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name = "ripple-dflash2-runner-test"
|
||||
if err := manifest.WriteManifestWithMetadata(modeltypes.ParseName(name), meta, layers, manifest.RunnerMLX, manifest.FormatSafetensors); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
type rippleDraftForward struct {
|
||||
offset, contextRows, featureWidth, blockRows int
|
||||
}
|
||||
|
||||
// Observe shapes and calls only; every native operand and selector stays real.
|
||||
type rippleDFlash2Observer struct {
|
||||
model.BlockSelector
|
||||
forwards []rippleDraftForward
|
||||
selections int
|
||||
}
|
||||
|
||||
func (d *rippleDFlash2Observer) Forward(b *batch.Batch, targets, drafts []cache.Cache) (*mlx.Array, *mlx.Array) {
|
||||
call := rippleDraftForward{offset: int(b.SeqOffsets[0])}
|
||||
if b.Hidden != nil {
|
||||
call.contextRows, call.featureWidth = b.Hidden.Dim(1), b.Hidden.Dim(2)
|
||||
}
|
||||
if b.InputIDs != nil {
|
||||
call.blockRows = b.InputIDs.Dim(1)
|
||||
}
|
||||
d.forwards = append(d.forwards, call)
|
||||
return d.BlockSelector.Forward(b, targets, drafts)
|
||||
}
|
||||
|
||||
func (d *rippleDFlash2Observer) Select(hidden, logits, anchor *mlx.Array, choose func(ids, scores *mlx.Array) (*mlx.Array, *mlx.Array)) (*mlx.Array, *mlx.Array, *mlx.Array) {
|
||||
d.selections++
|
||||
return d.BlockSelector.Select(hidden, logits, anchor, choose)
|
||||
}
|
||||
|
||||
func loadRippleDFlash2Runner(t *testing.T, name string) (*Runner, *rippleDFlash2Observer) {
|
||||
t.Helper()
|
||||
r := loadRippleRunner(t, name)
|
||||
if r.spec == nil {
|
||||
t.Fatal("combined manifest did not attach a draft")
|
||||
}
|
||||
draft, ok := r.spec.draft.(model.BlockSelector)
|
||||
if !ok {
|
||||
t.Fatalf("loaded draft %T, want DFlash2 BlockSelector", r.spec.draft)
|
||||
}
|
||||
if block, mask := draft.BlockParams(); block != 8 || mask != 18 {
|
||||
t.Fatalf("draft block=%d mask=%d, want 8 and 18", block, mask)
|
||||
}
|
||||
if len(r.spec.targets) != 3 || len(r.spec.draftKV) != 2 || len(r.cache.caches) != 5 {
|
||||
t.Fatalf("cache partitions: target=%d draft=%d combined=%d", len(r.spec.targets), len(r.spec.draftKV), len(r.cache.caches))
|
||||
}
|
||||
if _, ok := r.spec.draftKV[0].(*cache.RotatingKVCache); !ok {
|
||||
t.Fatalf("local draft cache is %T", r.spec.draftKV[0])
|
||||
}
|
||||
if _, ok := r.spec.draftKV[1].(*cache.RotatingKVCache); !ok {
|
||||
t.Fatalf("full draft cache is %T", r.spec.draftKV[1])
|
||||
}
|
||||
observer := &rippleDFlash2Observer{BlockSelector: draft}
|
||||
r.spec.draft = observer
|
||||
return r, observer
|
||||
}
|
||||
|
||||
func checkRippleDFlash2Caches(t *testing.T, r *Runner, prompt string) {
|
||||
t.Helper()
|
||||
mlxtest.Run(t, func(t *mlxtest.T) {
|
||||
offset := r.spec.targets[0].Offset()
|
||||
if offset < len(prompt) {
|
||||
t.Fatalf("target stopped before prompt completion: offset=%d prompt=%d", offset, len(prompt))
|
||||
}
|
||||
for i, c := range r.cache.caches {
|
||||
if c.Offset() != offset {
|
||||
t.Fatalf("cache %d offset=%d, want settled target offset=%d", i, c.Offset(), offset)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type rippleDFlash2Response struct {
|
||||
text string
|
||||
cached int
|
||||
forwards []rippleDraftForward
|
||||
}
|
||||
|
||||
func runRippleDFlash2Request(t *testing.T, r *Runner, prompt string, count int) rippleDFlash2Response {
|
||||
t.Helper()
|
||||
d := r.spec.draft.(*rippleDFlash2Observer)
|
||||
d.forwards, d.selections = nil, 0
|
||||
request := rippleRunnerRequest(t, r, prompt, count)
|
||||
// The shared helper requests logprobs, which would silently park drafting.
|
||||
request.SamplerOpts.Logprobs, request.SamplerOpts.TopLogprobs = false, 0
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
request.Ctx = ctx
|
||||
if err := r.runRequest(request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
close(request.Responses)
|
||||
var text strings.Builder
|
||||
var final *CompletionResponse
|
||||
for response := range request.Responses {
|
||||
if response.Error != nil || len(response.Logprobs) != 0 {
|
||||
t.Fatalf("unexpected active response: %+v", response)
|
||||
}
|
||||
text.WriteString(response.Content)
|
||||
if response.Done {
|
||||
if final != nil {
|
||||
t.Fatal("duplicate final response")
|
||||
}
|
||||
final = &response
|
||||
}
|
||||
}
|
||||
if final == nil || final.EvalCount != count || final.DoneReason != 1 || final.PromptEvalCount != len(request.Tokens) || final.PromptEvalCachedCount == nil || text.Len() != count {
|
||||
t.Fatalf("invalid completion: text=%q final=%+v", text.String(), final)
|
||||
}
|
||||
proposals := 0
|
||||
for _, call := range d.forwards {
|
||||
if call.contextRows > 0 && call.featureWidth != 16 {
|
||||
t.Fatalf("draft did not receive both target taps: %+v", call)
|
||||
}
|
||||
if call.blockRows > 0 {
|
||||
proposals++
|
||||
if call.blockRows < 2 || call.blockRows > min(8, count) {
|
||||
t.Fatalf("proposal exceeded block/budget: %+v budget=%d", call, count)
|
||||
}
|
||||
}
|
||||
}
|
||||
if proposals != d.selections || (count > 1 && proposals == 0) || (count == 1 && proposals != 0) {
|
||||
t.Fatalf("active budget=%d: draft forwards=%d selector calls=%d", count, proposals, d.selections)
|
||||
}
|
||||
checkRippleDFlash2Caches(t, r, prompt)
|
||||
return rippleDFlash2Response{text: text.String(), cached: *final.PromptEvalCachedCount, forwards: d.forwards}
|
||||
}
|
||||
|
||||
func TestRippleRunnerDFlash2(t *testing.T) {
|
||||
name := writeRippleDFlash2RunnerModel(t)
|
||||
const prompt = "abcdefghi"
|
||||
|
||||
// New wiring: normal combined load, target prefill with trailing draft
|
||||
// caches, target taps, and a real convolution/selector proposal.
|
||||
t.Run("combined_manifest_prefill_proposal", func(t *testing.T) {
|
||||
r, d := loadRippleDFlash2Runner(t, name)
|
||||
got := runRippleDFlash2Request(t, r, prompt, 9)
|
||||
if got.cached != 0 || len(d.forwards) < 2 {
|
||||
t.Fatalf("cold request: cached=%d forwards=%+v", got.cached, d.forwards)
|
||||
}
|
||||
if want := (rippleDraftForward{contextRows: len(prompt) - 1, featureWidth: 16}); d.forwards[0] != want {
|
||||
t.Fatalf("prefill handoff=%+v, want %+v", d.forwards[0], want)
|
||||
}
|
||||
if want := (rippleDraftForward{offset: len(prompt) - 1, blockRows: 8}); d.forwards[1] != want {
|
||||
t.Fatalf("first proposal=%+v, want %+v", d.forwards[1], want)
|
||||
}
|
||||
})
|
||||
|
||||
// Existing runner lifecycle, characterized with the new target/draft pair.
|
||||
// Compare identities only after matching cache-hit and proposal schedules.
|
||||
t.Run("prefix_branch_reuse", func(t *testing.T) {
|
||||
r, _ := loadRippleDFlash2Runner(t, name)
|
||||
runRippleDFlash2Request(t, r, prompt, 9)
|
||||
runRippleDFlash2Request(t, r, prompt, 9) // warm the repeat schedule
|
||||
want := runRippleDFlash2Request(t, r, prompt, 9)
|
||||
if want.cached <= 0 || want.cached >= len(prompt) {
|
||||
t.Fatalf("repeat cached=%d, want prefix plus an unforwarded seed", want.cached)
|
||||
}
|
||||
if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("matched repeat changed: got=%+v want=%+v", got, want)
|
||||
}
|
||||
const branch = "abcdejklm"
|
||||
if got := runRippleDFlash2Request(t, r, branch, 9); got.cached != 5 {
|
||||
t.Fatalf("branch cached=%d, want shared five-token snapshot", got.cached)
|
||||
}
|
||||
if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("returning from branch changed matched schedule: got=%+v want=%+v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parked_to_enabled", func(t *testing.T) {
|
||||
r, d := loadRippleDFlash2Runner(t, name)
|
||||
runRippleRunnerRequest(t, r, prompt)
|
||||
if len(d.forwards) == 0 || d.selections != 0 {
|
||||
t.Fatalf("parked maintenance: forwards=%+v selections=%d", d.forwards, d.selections)
|
||||
}
|
||||
for _, call := range d.forwards {
|
||||
if call.blockRows != 0 || call.contextRows == 0 || call.featureWidth != 16 {
|
||||
t.Fatalf("logprobs must maintain context without proposals: %+v", call)
|
||||
}
|
||||
}
|
||||
checkRippleDFlash2Caches(t, r, prompt)
|
||||
if got := runRippleDFlash2Request(t, r, prompt, 9); got.cached <= 0 {
|
||||
t.Fatal("enabled request did not reuse parked context")
|
||||
}
|
||||
runRippleDFlash2Request(t, r, prompt, 9)
|
||||
want := runRippleDFlash2Request(t, r, prompt, 9)
|
||||
if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("enabled repeats changed: got=%+v want=%+v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("budget_tail", func(t *testing.T) {
|
||||
r, d := loadRippleDFlash2Runner(t, name)
|
||||
runRippleDFlash2Request(t, r, prompt, 3)
|
||||
if len(d.forwards) < 2 || d.forwards[1].blockRows != 3 {
|
||||
t.Fatalf("shorter-than-convolution proposal: %+v", d.forwards)
|
||||
}
|
||||
runRippleDFlash2Request(t, r, prompt, 1)
|
||||
})
|
||||
|
||||
t.Run("cancellation_recovery", func(t *testing.T) {
|
||||
r, d := loadRippleDFlash2Runner(t, name)
|
||||
request := rippleRunnerRequest(t, r, prompt, 16)
|
||||
request.SamplerOpts.Logprobs, request.SamplerOpts.TopLogprobs = false, 0
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
request.Ctx = ctx
|
||||
request.Responses = make(chan CompletionResponse)
|
||||
responses := make(chan []CompletionResponse, 1)
|
||||
go func() {
|
||||
var received []CompletionResponse
|
||||
for response := range request.Responses {
|
||||
received = append(received, response)
|
||||
cancel()
|
||||
}
|
||||
responses <- received
|
||||
}()
|
||||
err := r.runRequest(request)
|
||||
close(request.Responses)
|
||||
received := <-responses
|
||||
if !errors.Is(err, context.Canceled) || len(received) == 0 || received[0].Content == "" || d.selections == 0 {
|
||||
t.Fatalf("cancel after real speculative output: err=%v responses=%+v selections=%d", err, received, d.selections)
|
||||
}
|
||||
for _, response := range received {
|
||||
if response.Done || response.Error != nil {
|
||||
t.Fatalf("cancelled generation emitted terminal response: %+v", response)
|
||||
}
|
||||
}
|
||||
checkRippleDFlash2Caches(t, r, prompt)
|
||||
runRippleDFlash2Request(t, r, prompt, 9)
|
||||
runRippleDFlash2Request(t, r, prompt, 9)
|
||||
want := runRippleDFlash2Request(t, r, prompt, 9)
|
||||
if got := runRippleDFlash2Request(t, r, prompt, 9); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("recovery repeats changed: got=%+v want=%+v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
194
mlxrunner/testdata/generate_qwen35_yarn.py
vendored
194
mlxrunner/testdata/generate_qwen35_yarn.py
vendored
|
|
@ -1,194 +0,0 @@
|
|||
# /// script
|
||||
# requires-python = ">=3.12,<3.13"
|
||||
# dependencies = ["torch==2.6.0", "transformers==5.12.1"]
|
||||
# ///
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import transformers
|
||||
from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
|
||||
from transformers.models.qwen3_5.modeling_qwen3_5 import (
|
||||
Qwen3_5Attention,
|
||||
Qwen3_5TextRotaryEmbedding,
|
||||
apply_rotary_pos_emb,
|
||||
)
|
||||
|
||||
|
||||
def generate():
|
||||
torch.set_num_threads(1)
|
||||
defaults = {
|
||||
"rope_type": "default",
|
||||
"rope_theta": 10000000.0,
|
||||
"partial_rotary_factor": 0.25,
|
||||
"mrope_section": [11, 11, 10],
|
||||
}
|
||||
yarn = {
|
||||
**defaults,
|
||||
"rope_type": "yarn",
|
||||
"factor": 4.0,
|
||||
"original_max_position_embeddings": 262144,
|
||||
}
|
||||
cases = [
|
||||
("default", 262144, defaults, "float32"),
|
||||
("no_metadata", 262144, None, "float32"),
|
||||
("yarn4", 1048576, yarn, "float32"),
|
||||
("unextended_limit", 262144, yarn, "float32"),
|
||||
(
|
||||
"implicit_original",
|
||||
262144,
|
||||
{k: v for k, v in yarn.items() if k != "original_max_position_embeddings"},
|
||||
"float32",
|
||||
),
|
||||
("beta", 1048576, {**yarn, "beta_fast": 16.0, "beta_slow": 2.0}, "float32"),
|
||||
("fractional_range", 1048576, {**yarn, "truncate": False}, "float32"),
|
||||
("mscale", 1048576, {**yarn, "mscale": 2.0, "mscale_all_dim": 1.0}, "float32"),
|
||||
("attention_override", 1048576, {**yarn, "attention_factor": 1.2}, "float32"),
|
||||
("zero_attention", 1048576, {**yarn, "attention_factor": 0.0}, "float32"),
|
||||
("factor_one", 262144, {**yarn, "factor": 1.0}, "float32"),
|
||||
(
|
||||
"factor_one_attention",
|
||||
262144,
|
||||
{**yarn, "factor": 1.0, "attention_factor": 1.2},
|
||||
"float32",
|
||||
),
|
||||
("theta", 1048576, {**yarn, "rope_theta": 500000.0}, "float32"),
|
||||
(
|
||||
"half_rotary",
|
||||
1048576,
|
||||
{**yarn, "partial_rotary_factor": 0.5, "mrope_section": [22, 21, 21]},
|
||||
"float32",
|
||||
),
|
||||
("sections", 1048576, {**yarn, "mrope_section": [24, 4, 4]}, "float32"),
|
||||
("default_bfloat16", 262144, defaults, "bfloat16"),
|
||||
("yarn_bfloat16", 1048576, yarn, "bfloat16"),
|
||||
]
|
||||
sample_tokens = [0, 1, 7, 15]
|
||||
sample_dims = [0, 7, 15, 16, 23, 31, 32, 39, 47, 48, 55, 63, 64, 95, 127, 128, 255]
|
||||
indices = torch.arange(16)
|
||||
text_positions = indices.reshape(1, -1)
|
||||
media_positions = torch.stack(
|
||||
(32768 + 2 * indices, 7 + indices % 4, 11 + indices // 4)
|
||||
).unsqueeze(1)
|
||||
media_positions[:, 0, -1] = media_positions[:, :, :-1].max() + 1
|
||||
mask = torch.full((1, 1, 16, 16), float("-inf")).triu(1)
|
||||
results = []
|
||||
for name, limit, params, dtype_name in cases:
|
||||
raw_config = {
|
||||
"model_type": "qwen3_5_text",
|
||||
"hidden_size": 256,
|
||||
"num_hidden_layers": 1,
|
||||
"num_attention_heads": 1,
|
||||
"num_key_value_heads": 1,
|
||||
"head_dim": 256,
|
||||
"intermediate_size": 256,
|
||||
"rms_norm_eps": 1e-6,
|
||||
"linear_num_key_heads": 1,
|
||||
"linear_num_value_heads": 1,
|
||||
"linear_key_head_dim": 128,
|
||||
"linear_value_head_dim": 128,
|
||||
"layer_types": ["full_attention"],
|
||||
"max_position_embeddings": limit,
|
||||
"rope_theta": 10000000.0,
|
||||
"partial_rotary_factor": (params or defaults)["partial_rotary_factor"],
|
||||
"rope_parameters": params,
|
||||
}
|
||||
cfg = Qwen3_5TextConfig(
|
||||
**{
|
||||
k: v
|
||||
for k, v in raw_config.items()
|
||||
if k not in ("rope_theta", "rope_parameters")
|
||||
},
|
||||
rope_parameters=dict(params or defaults),
|
||||
)
|
||||
cfg._attn_implementation = "eager"
|
||||
dtype = getattr(torch, dtype_name)
|
||||
attention = Qwen3_5Attention(cfg, layer_idx=0).eval().to(dtype)
|
||||
rope = Qwen3_5TextRotaryEmbedding(cfg)
|
||||
assert rope.inv_freq.numel() * 2 == int(
|
||||
cfg.head_dim * (params or defaults)["partial_rotary_factor"]
|
||||
)
|
||||
with torch.no_grad():
|
||||
attention.q_proj.weight.zero_()
|
||||
attention.q_proj.weight[:256].copy_(torch.eye(256))
|
||||
for linear in (attention.k_proj, attention.v_proj, attention.o_proj):
|
||||
linear.weight.copy_(torch.eye(256))
|
||||
attention.q_norm.weight.fill_(-0.5)
|
||||
attention.k_norm.weight.fill_(-0.5)
|
||||
x = (
|
||||
(((torch.arange(16 * 256) % 37) - 18).float() / 16)
|
||||
.reshape(1, 16, 256)
|
||||
.to(dtype)
|
||||
)
|
||||
raw_keys = attention.k_norm(
|
||||
attention.k_proj(x).reshape(1, 16, 1, 256)
|
||||
).transpose(1, 2)
|
||||
entry = {
|
||||
"name": name,
|
||||
"dtype": dtype_name,
|
||||
"config": raw_config,
|
||||
"inv_freq": rope.inv_freq.tolist(),
|
||||
"attention_factor": rope.attention_scaling,
|
||||
}
|
||||
modes = (
|
||||
("text", text_positions),
|
||||
("mrope", media_positions),
|
||||
("long_text", text_positions + 32768),
|
||||
)
|
||||
if name == "sections":
|
||||
entry["modes"] = ["mrope"]
|
||||
modes = (("mrope", media_positions),)
|
||||
for mode, positions in modes:
|
||||
cos, sin = rope(x, positions)
|
||||
output, _ = attention(
|
||||
x, position_embeddings=(cos, sin), attention_mask=mask
|
||||
)
|
||||
entry[mode + "_output"] = [
|
||||
output[0, token, dim].float().item()
|
||||
for token in sample_tokens
|
||||
for dim in sample_dims
|
||||
]
|
||||
if mode != "long_text":
|
||||
_, keys = apply_rotary_pos_emb(raw_keys, raw_keys, cos, sin)
|
||||
entry[mode + "_keys"] = [
|
||||
keys[0, 0, token, dim].float().item()
|
||||
for token in sample_tokens
|
||||
for dim in sample_dims
|
||||
]
|
||||
results.append(entry)
|
||||
return {
|
||||
"source": f"transformers {transformers.__version__}; torch {torch.__version__}; CPU; Qwen3_5TextConfig/Qwen3_5TextRotaryEmbedding/Qwen3_5Attention",
|
||||
"sample_tokens": sample_tokens,
|
||||
"sample_dims": sample_dims,
|
||||
"mrope_positions": media_positions.flatten().tolist(),
|
||||
"cases": results,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("output", type=Path)
|
||||
args = parser.parse_args()
|
||||
fixture = generate()
|
||||
text = json.dumps(fixture, indent=2)
|
||||
for case in fixture["cases"]:
|
||||
for key in (
|
||||
"inv_freq",
|
||||
"text_output",
|
||||
"long_text_output",
|
||||
"text_keys",
|
||||
"mrope_output",
|
||||
"mrope_keys",
|
||||
):
|
||||
if key not in case:
|
||||
continue
|
||||
multiline = json.dumps(case[key], indent=2)
|
||||
body = "\n".join(" " + line for line in multiline.splitlines()[1:-1])
|
||||
text = text.replace("[\n" + body + "\n ]", json.dumps(case[key]))
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(text + "\n")
|
||||
print(
|
||||
f"Generated {len(fixture['cases'])} reference cases: {args.output} ({args.output.stat().st_size} bytes)"
|
||||
)
|
||||
|
|
@ -6,6 +6,12 @@ import (
|
|||
"github.com/ollama/ollama/mlx"
|
||||
)
|
||||
|
||||
func TestSupportsRippleArchitecture(t *testing.T) {
|
||||
if !SupportsArchitecture("RicursiveForCausalLM") {
|
||||
t.Fatal("missing model architecture")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRuntimeRequiresLoadedLibrary(t *testing.T) {
|
||||
_, loadErr := mlx.LoadedLibraryPath()
|
||||
err := CheckRuntime()
|
||||
|
|
|
|||
|
|
@ -53,6 +53,8 @@ func ParserForName(name string) Parser {
|
|||
var p Parser
|
||||
|
||||
switch name {
|
||||
case "ripple":
|
||||
p = &RippleParser{}
|
||||
case "qwen3":
|
||||
p = &Qwen3Parser{hasThinkingSupport: false, defaultThinking: false}
|
||||
case "qwen3-thinking":
|
||||
|
|
|
|||
184
model/parsers/ripple.go
Normal file
184
model/parsers/ripple.go
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
package parsers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/ollama/ollama/api"
|
||||
)
|
||||
|
||||
type rippleState int
|
||||
|
||||
const (
|
||||
rippleStart rippleState = iota
|
||||
rippleThinkLead
|
||||
rippleThinking
|
||||
rippleThinkEnd
|
||||
rippleContent
|
||||
rippleTool
|
||||
)
|
||||
|
||||
const (
|
||||
rippleThinkOpen = "<think>"
|
||||
rippleThinkClose = "</think>"
|
||||
rippleToolOpen = "<tool_call>"
|
||||
rippleToolClose = "</tool_call>"
|
||||
)
|
||||
|
||||
type RippleParser struct {
|
||||
state rippleState
|
||||
buffer string
|
||||
callIndex int
|
||||
toolScan int
|
||||
toolString bool
|
||||
toolEscape bool
|
||||
}
|
||||
|
||||
func (p *RippleParser) Init(tools []api.Tool, _ *api.Message, _ *api.ThinkValue) []api.Tool {
|
||||
*p = RippleParser{}
|
||||
return tools
|
||||
}
|
||||
|
||||
func (p *RippleParser) HasToolSupport() bool { return true }
|
||||
func (p *RippleParser) HasThinkingSupport() bool { return true }
|
||||
|
||||
func (p *RippleParser) ThinkingClose() []string {
|
||||
if p.state == rippleThinkLead || p.state == rippleThinking {
|
||||
return []string{rippleThinkClose}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *RippleParser) PreservedTokens() []string {
|
||||
return []string{rippleThinkOpen, rippleThinkClose, rippleToolOpen, rippleToolClose}
|
||||
}
|
||||
|
||||
func (p *RippleParser) Add(s string, done bool) (content string, thinking string, calls []api.ToolCall, err error) {
|
||||
p.buffer += s
|
||||
parse:
|
||||
for p.buffer != "" {
|
||||
switch p.state {
|
||||
case rippleStart:
|
||||
if strings.HasPrefix(p.buffer, rippleThinkOpen) {
|
||||
p.buffer = p.buffer[len(rippleThinkOpen):]
|
||||
p.state = rippleThinkLead
|
||||
} else if !done && strings.HasPrefix(rippleThinkOpen, p.buffer) {
|
||||
return content, thinking, calls, nil
|
||||
} else {
|
||||
p.state = rippleContent
|
||||
}
|
||||
case rippleThinkLead:
|
||||
p.buffer = strings.TrimPrefix(p.buffer, "\n")
|
||||
p.state = rippleThinking
|
||||
case rippleThinking:
|
||||
if i := strings.Index(p.buffer, rippleThinkClose); i >= 0 {
|
||||
thinking += strings.TrimSuffix(p.buffer[:i], "\n")
|
||||
p.buffer = p.buffer[i+len(rippleThinkClose):]
|
||||
p.state = rippleThinkEnd
|
||||
} else {
|
||||
n := len(p.buffer)
|
||||
if !done {
|
||||
// The newline before a closing tag belongs to the template.
|
||||
n -= max(overlap(p.buffer, rippleThinkClose), overlap(p.buffer, "\n"+rippleThinkClose))
|
||||
}
|
||||
thinking += p.buffer[:n]
|
||||
p.buffer = p.buffer[n:]
|
||||
return content, thinking, calls, nil
|
||||
}
|
||||
case rippleThinkEnd:
|
||||
p.buffer = strings.TrimPrefix(p.buffer, "\n")
|
||||
p.state = rippleContent
|
||||
case rippleContent:
|
||||
if i := strings.Index(p.buffer, rippleToolOpen); i >= 0 {
|
||||
if p.callIndex == 0 {
|
||||
content += strings.TrimSuffix(p.buffer[:i], "\n")
|
||||
}
|
||||
p.buffer = p.buffer[i+len(rippleToolOpen):]
|
||||
p.state = rippleTool
|
||||
p.toolScan, p.toolString, p.toolEscape = 0, false, false
|
||||
} else {
|
||||
n := len(p.buffer)
|
||||
if !done {
|
||||
n -= max(overlap(p.buffer, rippleToolOpen), overlap(p.buffer, "\n"+rippleToolOpen))
|
||||
}
|
||||
if p.callIndex == 0 {
|
||||
content += p.buffer[:n]
|
||||
}
|
||||
p.buffer = p.buffer[n:]
|
||||
return content, thinking, calls, nil
|
||||
}
|
||||
case rippleTool:
|
||||
if i := p.toolEnd(); i >= 0 {
|
||||
call, parseErr := parseRippleTool(p.buffer[:i])
|
||||
p.buffer = p.buffer[i+len(rippleToolClose):]
|
||||
p.state = rippleContent
|
||||
if parseErr != nil {
|
||||
return content, thinking, calls, parseErr
|
||||
}
|
||||
call.Function.Index = p.callIndex
|
||||
p.callIndex++
|
||||
calls = append(calls, call)
|
||||
} else if !done {
|
||||
return content, thinking, calls, nil
|
||||
} else {
|
||||
break parse
|
||||
}
|
||||
}
|
||||
}
|
||||
if done && p.state == rippleTool {
|
||||
p.buffer = ""
|
||||
p.state = rippleContent
|
||||
return content, thinking, calls, fmt.Errorf("ripple: unterminated tool call")
|
||||
}
|
||||
return content, thinking, calls, nil
|
||||
}
|
||||
|
||||
// toolEnd ignores closing tags inside JSON strings, including escaped quotes.
|
||||
func (p *RippleParser) toolEnd() int {
|
||||
for p.toolScan < len(p.buffer) {
|
||||
i := p.toolScan
|
||||
c := p.buffer[i]
|
||||
if p.toolString {
|
||||
if p.toolEscape {
|
||||
p.toolEscape = false
|
||||
} else if c == '\\' {
|
||||
p.toolEscape = true
|
||||
} else if c == '"' {
|
||||
p.toolString = false
|
||||
}
|
||||
} else if c == '"' {
|
||||
p.toolString = true
|
||||
} else if c == '<' {
|
||||
if strings.HasPrefix(p.buffer[i:], rippleToolClose) {
|
||||
return i
|
||||
}
|
||||
if strings.HasPrefix(rippleToolClose, p.buffer[i:]) {
|
||||
return -1
|
||||
}
|
||||
}
|
||||
p.toolScan++
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func parseRippleTool(raw string) (api.ToolCall, error) {
|
||||
var value struct {
|
||||
Name string `json:"name"`
|
||||
Arguments json.RawMessage `json:"arguments"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &value); err != nil {
|
||||
return api.ToolCall{}, fmt.Errorf("ripple: invalid tool call JSON: %w", err)
|
||||
}
|
||||
if value.Name == "" {
|
||||
return api.ToolCall{}, fmt.Errorf("ripple: empty function name")
|
||||
}
|
||||
if !strings.HasPrefix(strings.TrimSpace(string(value.Arguments)), "{") {
|
||||
return api.ToolCall{}, fmt.Errorf("ripple: tool arguments must be a JSON object")
|
||||
}
|
||||
var arguments api.ToolCallFunctionArguments
|
||||
if err := json.Unmarshal(value.Arguments, &arguments); err != nil {
|
||||
return api.ToolCall{}, fmt.Errorf("ripple: invalid tool arguments: %w", err)
|
||||
}
|
||||
return api.ToolCall{Function: api.ToolCallFunction{Name: value.Name, Arguments: arguments}}, nil
|
||||
}
|
||||
228
model/parsers/ripple_test.go
Normal file
228
model/parsers/ripple_test.go
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
package parsers
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ollama/ollama/api"
|
||||
)
|
||||
|
||||
func rippleParseChunks(t *testing.T, chunks []string) (string, string, []api.ToolCall, error) {
|
||||
t.Helper()
|
||||
p := ParserForName("ripple")
|
||||
if p == nil {
|
||||
t.Fatal("missing ripple parser")
|
||||
}
|
||||
p.Init(nil, nil, nil)
|
||||
var content, thinking strings.Builder
|
||||
var calls []api.ToolCall
|
||||
for _, chunk := range append(chunks, "") {
|
||||
c, r, tc, err := p.Add(chunk, false)
|
||||
content.WriteString(c)
|
||||
thinking.WriteString(r)
|
||||
calls = append(calls, tc...)
|
||||
if err != nil {
|
||||
return content.String(), thinking.String(), calls, err
|
||||
}
|
||||
}
|
||||
c, r, tc, err := p.Add("", true)
|
||||
content.WriteString(c)
|
||||
thinking.WriteString(r)
|
||||
calls = append(calls, tc...)
|
||||
return content.String(), thinking.String(), calls, err
|
||||
}
|
||||
|
||||
func rippleChunkings(text string) [][]string {
|
||||
chunks := [][]string{{text}}
|
||||
for i := 0; i <= len(text); i++ {
|
||||
chunks = append(chunks, []string{text[:i], text[i:]})
|
||||
}
|
||||
bytes := make([]string, len(text))
|
||||
for i := range len(text) {
|
||||
bytes[i] = text[i : i+1]
|
||||
}
|
||||
return append(chunks, bytes)
|
||||
}
|
||||
|
||||
func TestRippleParserThinking(t *testing.T) {
|
||||
for _, tt := range []struct{ name, input, content, thinking string }{
|
||||
{"plain", " \nhello\n", " \nhello\n", ""},
|
||||
{"thinking", "<think>\nreason\n</think>\nanswer", "answer", "reason"},
|
||||
{"no structural newlines", "<think>reason</think>answer", "answer", "reason"},
|
||||
{"one newline only", "<think>\n</think>\n", "", ""},
|
||||
{"meaningful whitespace", "<think>\n\n \treason \n\n</think>\n\n answer \n", "\n answer \n", "\n \treason \n"},
|
||||
{"CRLF is not a structural newline", "<think>\r\nreason\r\n</think>\r\nanswer", "\r\nanswer", "\r\nreason\r"},
|
||||
{"nonleading opener", " \n<think>reason</think>\nanswer", " \n<think>reason</think>\nanswer", ""},
|
||||
{"no opener", "reason\n</think>\nanswer", "reason\n</think>\nanswer", ""},
|
||||
{"later tags literal", "<think>r</think>c<think>x</think>", "c<think>x</think>", "r"},
|
||||
{"tool inside thinking stays literal", "<think>\n<tool_call>{}</tool_call>\n</think>\nanswer", "answer", "<tool_call>{}</tool_call>"},
|
||||
{"truncated opener", "<thin", "<thin", ""},
|
||||
{"truncated reasoning", "<think>\nreason\n</thi", "", "reason\n</thi"},
|
||||
{"trailing newline drained", "<think>\nreason\n", "", "reason\n"},
|
||||
{"opener only", "<think>", "", ""},
|
||||
{"opener and newline", "<think>\n", "", ""},
|
||||
{"close only", "<think>\nreason\n</think>", "", "reason"},
|
||||
{"partial tool text", "text\n<tool_", "text\n<tool_", ""},
|
||||
{"unicode", "<think>\n日本語🙂\n</think>\né\n", "é\n", "日本語🙂"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
for _, chunks := range rippleChunkings(tt.input) {
|
||||
c, r, calls, err := rippleParseChunks(t, chunks)
|
||||
if err != nil || c != tt.content || r != tt.thinking || len(calls) != 0 {
|
||||
t.Fatalf("chunks %q: got (%q, %q, %v, %v), want (%q, %q)", chunks, c, r, calls, err, tt.content, tt.thinking)
|
||||
}
|
||||
}
|
||||
p := ParserForName("ripple")
|
||||
p.Init(nil, nil, nil)
|
||||
c, r, calls, err := p.Add(tt.input, true)
|
||||
if err != nil || c != tt.content || r != tt.thinking || len(calls) != 0 {
|
||||
t.Fatalf("final chunk: (%q, %q, %v, %v)", c, r, calls, err)
|
||||
}
|
||||
c, r, calls, err = p.Add("", true)
|
||||
if err != nil || c != "" || r != "" || len(calls) != 0 {
|
||||
t.Fatal("final drain repeated output")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleParserThinkingTransitions(t *testing.T) {
|
||||
p := ParserForName("ripple")
|
||||
p.Init(nil, nil, nil)
|
||||
for _, tt := range []struct {
|
||||
input, content, thinking string
|
||||
close bool
|
||||
}{
|
||||
{"<thi", "", "", false},
|
||||
{"nk>", "", "", true},
|
||||
{"\nreason\n", "", "reason", true},
|
||||
{"</thi", "", "", true},
|
||||
{"nk>", "", "", false},
|
||||
{"\nanswer", "answer", "", false},
|
||||
} {
|
||||
c, r, _, err := p.Add(tt.input, false)
|
||||
if c != tt.content || r != tt.thinking || err != nil {
|
||||
t.Fatalf("%q: (%q, %q, %v)", tt.input, c, r, err)
|
||||
}
|
||||
if close := p.ThinkingClose(); (len(close) != 0) != tt.close || tt.close && close[0] != "</think>" {
|
||||
t.Fatalf("%q: close %v", tt.input, close)
|
||||
}
|
||||
}
|
||||
p.Init(nil, nil, nil)
|
||||
c, r, _, err := p.Add("<think>\nfresh\n</think>\nnew", true)
|
||||
if c != "new" || r != "fresh" || err != nil {
|
||||
t.Fatalf("reset: (%q, %q, %v)", c, r, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleParserTools(t *testing.T) {
|
||||
one := "<tool_call>\n" + `{"name": "one", "arguments": {"z": "a\n\"b\"\\<&é", "n": 3, "nested": {"x": true}}}` + "\n</tool_call>"
|
||||
two := "<tool_call>\n" + `{"name": "two", "arguments": {}}` + "\n</tool_call>"
|
||||
for _, tt := range []struct {
|
||||
name, input, content, thinking string
|
||||
calls []api.ToolCall
|
||||
}{
|
||||
{"one", one, "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "one", Arguments: args(`{"z":"a\n\"b\"\\<&é","n":3,"nested":{"x":true}}`)}}}},
|
||||
{"multiple", " \n before \t\n\n" + one + "\n" + two, " \n before \t\n", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "one", Arguments: args(`{"z":"a\n\"b\"\\<&é","n":3,"nested":{"x":true}}`)}}, {Function: api.ToolCallFunction{Index: 1, Name: "two", Arguments: args(`{}`)}}}},
|
||||
{"reasoning and tools", "<think>\n R \n</think>\n\n" + two, "", " R ", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}}},
|
||||
{"preserve spaces before call", " before \t" + two, " before \t", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}}},
|
||||
{"drop post call text", two + "ignored \n<think>also text</think>\n" + two + "more ignored", "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{}`)}}, {Function: api.ToolCallFunction{Index: 1, Name: "two", Arguments: args(`{}`)}}}},
|
||||
{"delimiter inside string", `<tool_call>{"name":"two","arguments":{"text":"</tool_call> <tool_call> \"quoted\""}}</tool_call>`, "", "", []api.ToolCall{{Function: api.ToolCallFunction{Name: "two", Arguments: args(`{"text":"</tool_call> <tool_call> \"quoted\""}`)}}}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
for _, chunks := range rippleChunkings(tt.input) {
|
||||
c, r, calls, err := rippleParseChunks(t, chunks)
|
||||
if err != nil || c != tt.content || r != tt.thinking || len(calls) != len(tt.calls) {
|
||||
t.Fatalf("chunks %q: (%q, %q, %v, %v)", chunks, c, r, calls, err)
|
||||
}
|
||||
for i := range calls {
|
||||
if !toolCallEqual(calls[i], tt.calls[i]) {
|
||||
t.Fatalf("call %d = %+v, want %+v", i, calls[i], tt.calls[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleParserMalformedCalls(t *testing.T) {
|
||||
for _, tt := range []struct{ name, input, errorText string }{
|
||||
{"invalid JSON", "<tool_call>{bad}</tool_call>", "JSON"},
|
||||
{"missing name", `<tool_call>{"arguments":{}}</tool_call>`, "name"},
|
||||
{"empty name", `<tool_call>{"name":"","arguments":{}}</tool_call>`, "name"},
|
||||
{"missing arguments", `<tool_call>{"name":"f"}</tool_call>`, "object"},
|
||||
{"string arguments", `<tool_call>{"name":"f","arguments":"{}"}</tool_call>`, "object"},
|
||||
{"array arguments", `<tool_call>{"name":"f","arguments":[]}</tool_call>`, "object"},
|
||||
{"null arguments", `<tool_call>{"name":"f","arguments":null}</tool_call>`, "object"},
|
||||
{"JSON array", `<tool_call>[{"name":"f","arguments":{}}]</tool_call>`, "JSON"},
|
||||
{"multiple objects", `<tool_call>{"name":"f","arguments":{}}{}</tool_call>`, "JSON"},
|
||||
{"truncated call", `<tool_call>{"name":"f","arguments":{}`, "unterminated"},
|
||||
{"truncated closer", `<tool_call>{"name":"f","arguments":{}}</tool_`, "unterminated"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
for _, chunks := range rippleChunkings(tt.input) {
|
||||
_, _, calls, err := rippleParseChunks(t, chunks)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.errorText) || len(calls) != 0 {
|
||||
t.Fatalf("chunks %q: calls %v, error %v; want %s", chunks, calls, err, tt.errorText)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleParserToolNewlineHoldback(t *testing.T) {
|
||||
p := ParserForName("ripple")
|
||||
p.Init(nil, nil, nil)
|
||||
c, _, _, err := p.Add("before\n\n", false)
|
||||
if c != "before\n" || err != nil {
|
||||
t.Fatalf("first chunk: %q, %v", c, err)
|
||||
}
|
||||
c, _, _, err = p.Add("<tool_", false)
|
||||
if c != "" || err != nil {
|
||||
t.Fatalf("partial delimiter: %q, %v", c, err)
|
||||
}
|
||||
c, _, calls, err := p.Add(`call>{"name":"f","arguments":{}}</tool_call>`, true)
|
||||
if c != "" || err != nil || len(calls) != 1 {
|
||||
t.Fatalf("call: %q, %v, %v", c, calls, err)
|
||||
}
|
||||
p.Init(nil, nil, nil)
|
||||
_, _, _, _ = p.Add("before\n", false)
|
||||
c, _, _, err = p.Add("<tool_", true)
|
||||
if c != "\n<tool_" || err != nil {
|
||||
t.Fatalf("drain: %q, %v", c, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleParserPlain(t *testing.T) {
|
||||
for _, text := range []string{"", "Hello", " \nanswer\n\n", "Reason without opener</think>\nAnswer", " <think>literal</think>", "Text <think>literal</think>", "<thinking>literal", "é 日本語 🙂"} {
|
||||
t.Run(text, func(t *testing.T) {
|
||||
content, thinking, calls, err := rippleParseChunks(t, []string{text})
|
||||
if err != nil || content != text || thinking != "" || len(calls) != 0 {
|
||||
t.Fatalf("got (%q, %q, %v, %v), want %q", content, thinking, calls, err, text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleParserCapabilities(t *testing.T) {
|
||||
p := ParserForName("ripple")
|
||||
if p == nil {
|
||||
t.Fatal("missing ripple parser")
|
||||
}
|
||||
if !p.HasToolSupport() || !p.HasThinkingSupport() {
|
||||
t.Fatal("missing capabilities")
|
||||
}
|
||||
if got := p.PreservedTokens(); !reflect.DeepEqual(got, []string{"<think>", "</think>", "<tool_call>", "</tool_call>"}) {
|
||||
t.Fatalf("preserved = %v", got)
|
||||
}
|
||||
tools := []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "f"}}}
|
||||
for _, think := range []*api.ThinkValue{nil, {Value: true}, {Value: false}, {Value: "high"}} {
|
||||
if got := p.Init(tools, &api.Message{Role: "assistant", Thinking: "old", Content: "old"}, think); !reflect.DeepEqual(got, tools) {
|
||||
t.Fatalf("changed tools: %v", got)
|
||||
}
|
||||
if got := p.ThinkingClose(); len(got) != 0 {
|
||||
t.Fatalf("generation prompt does not open reasoning: %v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -59,6 +59,8 @@ func rendererForName(name string) Renderer {
|
|||
return constructor()
|
||||
}
|
||||
switch name {
|
||||
case "ripple":
|
||||
return &RippleRenderer{}
|
||||
case "qwen3-coder":
|
||||
renderer := &Qwen3CoderRenderer{}
|
||||
return renderer
|
||||
|
|
|
|||
88
model/renderers/ripple.go
Normal file
88
model/renderers/ripple.go
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
package renderers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/ollama/ollama/api"
|
||||
"github.com/ollama/ollama/types/model"
|
||||
)
|
||||
|
||||
type RippleRenderer struct{}
|
||||
|
||||
func (r *RippleRenderer) LeadingBOS() string { return "" }
|
||||
|
||||
// Ripple can emit reasoning but has no template control to disable or force it.
|
||||
func (r *RippleRenderer) Thinking() *model.Thinking { return nil }
|
||||
|
||||
func (r *RippleRenderer) Render(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) {
|
||||
if len(messages) == 0 {
|
||||
return "", fmt.Errorf("ripple: expected at least one message")
|
||||
}
|
||||
for _, message := range messages {
|
||||
if len(message.Images) > 0 {
|
||||
return "", fmt.Errorf("ripple: text-only messages required")
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
if len(tools) > 0 {
|
||||
b.WriteString("<|im_start|>system\n")
|
||||
if messages[0].Role == "system" {
|
||||
b.WriteString(messages[0].Content + "\n\n")
|
||||
}
|
||||
b.WriteString("# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>")
|
||||
for _, tool := range tools {
|
||||
encoded, err := PythonJSON(tool, 0)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.WriteString("\n" + encoded)
|
||||
}
|
||||
b.WriteString("\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n")
|
||||
}
|
||||
for i, message := range messages {
|
||||
if i == 0 && message.Role == "system" && len(tools) > 0 {
|
||||
continue
|
||||
}
|
||||
content := message.Content
|
||||
if message.Role == "assistant" && message.Thinking != "" && !strings.HasPrefix(strings.TrimLeftFunc(content, func(r rune) bool {
|
||||
return unicode.IsSpace(r) || r >= '\x1c' && r <= '\x1f'
|
||||
}), "<think>") {
|
||||
content = "<think>\n" + message.Thinking + "\n</think>\n" + content
|
||||
}
|
||||
switch message.Role {
|
||||
case "assistant":
|
||||
b.WriteString("<|im_start|>assistant")
|
||||
if content != "" || len(message.ToolCalls) == 0 {
|
||||
b.WriteString("\n" + content)
|
||||
}
|
||||
for _, call := range message.ToolCalls {
|
||||
name, err := PythonJSON(call.Function.Name, 0)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
arguments, err := PythonJSON(call.Function.Arguments, 0)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.WriteString("\n<tool_call>\n{\"name\": " + name + ", \"arguments\": " + arguments + "}\n</tool_call>")
|
||||
}
|
||||
b.WriteString("<|im_end|>\n")
|
||||
case "tool":
|
||||
if i == 0 || messages[i-1].Role != "tool" {
|
||||
b.WriteString("<|im_start|>user")
|
||||
}
|
||||
b.WriteString("\n<tool_response>\n" + content + "\n</tool_response>")
|
||||
if i == len(messages)-1 || messages[i+1].Role != "tool" {
|
||||
b.WriteString("<|im_end|>\n")
|
||||
}
|
||||
case "system", "user":
|
||||
b.WriteString("<|im_start|>" + message.Role + "\n")
|
||||
b.WriteString(content)
|
||||
b.WriteString("<|im_end|>\n")
|
||||
}
|
||||
}
|
||||
b.WriteString("<|im_start|>assistant\n")
|
||||
return b.String(), nil
|
||||
}
|
||||
292
model/renderers/ripple_test.go
Normal file
292
model/renderers/ripple_test.go
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
package renderers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ollama/ollama/api"
|
||||
"github.com/ollama/ollama/model/parsers"
|
||||
)
|
||||
|
||||
const rippleTemplate = "testdata/ripple_chat_template.jinja"
|
||||
|
||||
func TestRipplePublisherTemplate(t *testing.T) {
|
||||
verifyLFM3TemplateFixture(t, rippleTemplate, "877c2dbf365243dda14738ca250b31398059b35ffd91d4ed0ec68be45ca6ebd3")
|
||||
}
|
||||
|
||||
func TestRippleRendererTools(t *testing.T) {
|
||||
call := api.ToolCall{Function: api.ToolCallFunction{Name: "f\"<&é", Arguments: args(`{"z":"line\n\"quote\"\\<&é","a":[true,null,3]}`)}}
|
||||
tests := []struct {
|
||||
name string
|
||||
messages []api.Message
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"call alone",
|
||||
[]api.Message{{Role: "assistant", ToolCalls: []api.ToolCall{call}}},
|
||||
"<|im_start|>assistant\n<tool_call>\n" + `{"name": "f\"<&é", "arguments": {"z": "line\n\"quote\"\\<&é", "a": [true, null, 3]}}` + "\n</tool_call><|im_end|>\n<|im_start|>assistant\n",
|
||||
},
|
||||
{
|
||||
"multiple calls with reasoning",
|
||||
[]api.Message{{Role: "assistant", Thinking: "R", Content: " C \n", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "one"}}, {Function: api.ToolCallFunction{Name: "two"}}}}},
|
||||
"<|im_start|>assistant\n<think>\nR\n</think>\n C \n\n<tool_call>\n{\"name\": \"one\", \"arguments\": {}}\n</tool_call>\n<tool_call>\n{\"name\": \"two\", \"arguments\": {}}\n</tool_call><|im_end|>\n<|im_start|>assistant\n",
|
||||
},
|
||||
{
|
||||
"grouped results",
|
||||
[]api.Message{{Role: "tool", Content: " A\n"}, {Role: "tool", Content: "B"}, {Role: "user", Content: "Next"}, {Role: "tool", Content: "C"}},
|
||||
"<|im_start|>user\n<tool_response>\n A\n\n</tool_response>\n<tool_response>\nB\n</tool_response><|im_end|>\n<|im_start|>user\nNext<|im_end|>\n<|im_start|>user\n<tool_response>\nC\n</tool_response><|im_end|>\n<|im_start|>assistant\n",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := RenderWithRenderer("ripple", tt.messages, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("got %q\nwant %q", got, tt.want)
|
||||
}
|
||||
if os.Getenv("VERIFY_JINJA2") != "" {
|
||||
want := renderQwenJinja(t, rippleTemplate, tt.messages, nil, nil, "")
|
||||
if got != want {
|
||||
t.Fatalf("Go %q\nPython %q", got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleRendererToolInstructions(t *testing.T) {
|
||||
tool := api.Tool{Type: "function", Function: api.ToolFunction{Name: "execute_bash", Parameters: api.ToolFunctionParameters{Type: "object"}}}
|
||||
for _, system := range []string{"", "Be precise."} {
|
||||
messages := []api.Message{{Role: "system", Content: system}, {Role: "user", Content: "pwd"}}
|
||||
got, err := RenderWithRenderer("ripple", messages, []api.Tool{tool}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "<|im_start|>system\n" + system + "\n\n" + `# Tools
|
||||
|
||||
You may call one or more functions to assist with the user query.
|
||||
|
||||
You are provided with function signatures within <tools> XML tags:
|
||||
<tools>
|
||||
{"type": "function", "function": {"name": "execute_bash", "parameters": {"type": "object", "properties": null}}}
|
||||
</tools>
|
||||
|
||||
For each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:
|
||||
<tool_call>
|
||||
{"name": "execute_bash", "arguments": {"command": "pwd"}}
|
||||
</tool_call><|im_end|>
|
||||
<|im_start|>user
|
||||
pwd<|im_end|>
|
||||
<|im_start|>assistant
|
||||
`
|
||||
if got != want {
|
||||
t.Fatalf("got %q\nwant %q", got, want)
|
||||
}
|
||||
if os.Getenv("VERIFY_JINJA2") != "" {
|
||||
want := renderQwenJinja(t, rippleTemplate, messages, []api.Tool{tool}, nil, "")
|
||||
if got != want {
|
||||
t.Fatalf("Go %q\nPython %q", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleRendererInvalidInput(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
messages []api.Message
|
||||
tools []api.Tool
|
||||
errorText string
|
||||
}{
|
||||
{name: "empty transcript", errorText: "at least one message"},
|
||||
{name: "empty transcript with tools", tools: []api.Tool{{Type: "function"}}, errorText: "at least one message"},
|
||||
{name: "image", messages: []api.Message{{Role: "user", Content: "describe", Images: []api.ImageData{{1}}}}, errorText: "text-only"},
|
||||
{name: "invalid tool schema", messages: []api.Message{{Role: "user"}}, tools: []api.Tool{{Items: make(chan int)}}, errorText: "unsupported type"},
|
||||
{name: "invalid tool arguments", messages: []api.Message{{Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "f", Arguments: testArgs(map[string]any{"x": make(chan int)})}}}}}, errorText: "unsupported type"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := RenderWithRenderer("ripple", tt.messages, tt.tools, nil)
|
||||
if got != "" || err == nil || !strings.Contains(err.Error(), tt.errorText) {
|
||||
t.Fatalf("got (%q, %v), want %s", got, err, tt.errorText)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleRendererToolResultContinuation(t *testing.T) {
|
||||
messages := []api.Message{
|
||||
{Role: "user", Content: "Run the program and report what it prints."},
|
||||
{Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "execute_bash", Arguments: args(`{"command":"go run ./cmd/demo"}`)}}}},
|
||||
{Role: "tool", Content: `{"exit_code":0,"stdout":"fmt.Println(\"done\")","stderr":""}`},
|
||||
{Role: "assistant", Thinking: "The command succeeded; I can report its output.", Content: "The program prints `done`."},
|
||||
}
|
||||
tools := []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "execute_bash"}}}
|
||||
want := "<|im_start|>system\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>\n{\"type\": \"function\", \"function\": {\"name\": \"execute_bash\", \"parameters\": {\"type\": \"\", \"properties\": null}}}\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n<|im_start|>user\nRun the program and report what it prints.<|im_end|>\n<|im_start|>assistant\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"go run ./cmd/demo\"}}\n</tool_call><|im_end|>\n<|im_start|>user\n<tool_response>\n{\"exit_code\":0,\"stdout\":\"fmt.Println(\\\"done\\\")\",\"stderr\":\"\"}\n</tool_response><|im_end|>\n<|im_start|>assistant\n<think>\nThe command succeeded; I can report its output.\n</think>\nThe program prints `done`.<|im_end|>\n<|im_start|>assistant\n"
|
||||
got, err := (&RippleRenderer{}).Render(messages, tools, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("render mismatch:\ngot:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleRendererJinjaParity(t *testing.T) {
|
||||
if os.Getenv("VERIFY_JINJA2") == "" {
|
||||
t.Skip("set VERIFY_JINJA2=1 and JINJA_PYTHON for publisher template parity")
|
||||
}
|
||||
values := api.NewToolCallFunctionArguments()
|
||||
values.Set("z", json.Number("3.0"))
|
||||
values.Set("a", json.RawMessage(`{"z": 1e-7, "a": ["\u2028\u2029<&é", true, null]}`))
|
||||
call := api.ToolCall{Function: api.ToolCallFunction{Name: "f\"\\\n<&é", Arguments: values}}
|
||||
tool := api.Tool{Type: "function", Function: api.ToolFunction{Name: "f", Description: "Use \"quoted\" <&é\n", Parameters: api.ToolFunctionParameters{Type: "object", Properties: propsMap(`{"z":{"type":"integer"},"a":{"type":"string"}}`), Required: []string{"z"}}}}
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
messages []api.Message
|
||||
tools []api.Tool
|
||||
think *api.ThinkValue
|
||||
}{
|
||||
{name: "plain", messages: []api.Message{{Role: "user", Content: " \nHi\t "}}},
|
||||
{name: "empty user", messages: []api.Message{{Role: "user"}}},
|
||||
{name: "initial and later systems", messages: []api.Message{{Role: "system", Content: "A"}, {Role: "system", Content: "B"}, {Role: "user", Content: "C"}, {Role: "system", Content: "D"}}},
|
||||
{name: "tools without system", messages: []api.Message{{Role: "user", Content: "Call"}}, tools: []api.Tool{tool, tool}},
|
||||
{name: "old reasoning retained", messages: []api.Message{{Role: "assistant", Thinking: "old", Content: "first"}, {Role: "user", Content: "next"}, {Role: "assistant", Thinking: "new", Content: "second"}, {Role: "user", Content: "again"}}},
|
||||
{name: "no duplicate wrapper", messages: []api.Message{{Role: "assistant", Thinking: "ignored", Content: "\x1c\t\u2003<think>old</think>answer"}}},
|
||||
{name: "nonleading tag", messages: []api.Message{{Role: "assistant", Thinking: "new", Content: "words<think>literal</think>"}}},
|
||||
{name: "thinking false has no invented prefix", messages: []api.Message{{Role: "assistant", Thinking: "retain", Content: "answer"}}, think: &api.ThinkValue{Value: false}},
|
||||
{name: "thinking true no forced opening", messages: []api.Message{{Role: "user", Content: "Hi"}}, think: &api.ThinkValue{Value: true}},
|
||||
{name: "tools JSON numbers escaping and order", messages: []api.Message{{Role: "assistant", Thinking: "Reason", Content: " \n ", ToolCalls: []api.ToolCall{call}}, {Role: "tool", Content: " result \n"}, {Role: "tool", Content: "other"}}, tools: []api.Tool{tool}},
|
||||
{name: "tools reasoning only", messages: []api.Message{{Role: "assistant", Thinking: "Reason", ToolCalls: []api.ToolCall{call}}}},
|
||||
{name: "group boundary ignores unknown role", messages: []api.Message{{Role: "tool", Content: "one"}, {Role: "ignored"}, {Role: "tool", Content: "two"}}},
|
||||
{
|
||||
name: "tool result followed by assistant decision",
|
||||
tools: []api.Tool{{Type: "function", Function: api.ToolFunction{Name: "execute_bash"}}},
|
||||
messages: []api.Message{
|
||||
{Role: "user", Content: "Run the program and report what it prints."},
|
||||
{Role: "assistant", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "execute_bash", Arguments: args(`{"command":"go run ./cmd/demo"}`)}}}},
|
||||
{Role: "tool", Content: `{"exit_code":0,"stdout":"fmt.Println(\"done\")","stderr":""}`},
|
||||
{Role: "assistant", Thinking: "The command succeeded; I can report its output.", Content: "The program prints `done`."},
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := RenderWithRenderer("ripple", tt.messages, tt.tools, tt.think)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := renderQwenJinja(t, rippleTemplate, tt.messages, tt.tools, tt.think, "")
|
||||
if got != want {
|
||||
t.Fatalf("Go %q\nPython %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
python := os.Getenv("JINJA_PYTHON")
|
||||
if python == "" {
|
||||
t.Fatal("set JINJA_PYTHON for negative publisher-template checks")
|
||||
}
|
||||
for _, messages := range []any{[]any{}, []any{map[string]any{"role": "user", "content": []any{map[string]any{"type": "image"}}}}} {
|
||||
output, err := lfm3RunJinja2(t, python, rippleTemplate, map[string]any{"messages": messages, "add_generation_prompt": true})
|
||||
if err == nil {
|
||||
t.Fatalf("publisher accepted unsupported transcript: %q", output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleRendererParserRoundTrip(t *testing.T) {
|
||||
for _, content := range []string{"", " ", "\n", "\n\n", " \t\n ", "answer\n", "日本語\n"} {
|
||||
for _, thinking := range []string{"", "R", " \n", "\n\nR\t\n"} {
|
||||
for _, calls := range [][]api.ToolCall{nil, {{Function: api.ToolCallFunction{Name: "f", Arguments: args(`{"text":"<&é\n\"quoted\""}`)}}}, {{Function: api.ToolCallFunction{Name: "f"}}, {Function: api.ToolCallFunction{Name: "g"}}}} {
|
||||
message := api.Message{Role: "assistant", Content: content, Thinking: thinking, ToolCalls: calls}
|
||||
original, err := RenderWithRenderer("ripple", []api.Message{message}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := strings.TrimSuffix(strings.TrimPrefix(original, "<|im_start|>assistant\n"), "<|im_end|>\n<|im_start|>assistant\n")
|
||||
p := parsers.ParserForName("ripple")
|
||||
p.Init(nil, nil, nil)
|
||||
parsed := api.Message{Role: "assistant"}
|
||||
for i := 0; i <= len(body); i++ {
|
||||
chunk := ""
|
||||
if i < len(body) {
|
||||
chunk = body[i : i+1]
|
||||
}
|
||||
c, r, tc, err := p.Add(chunk, i == len(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parsed.Content += c
|
||||
parsed.Thinking += r
|
||||
parsed.ToolCalls = append(parsed.ToolCalls, tc...)
|
||||
}
|
||||
got, err := RenderWithRenderer("ripple", []api.Message{parsed}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != original {
|
||||
t.Fatalf("roundtrip got %q\nwant %q", got, original)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleRendererReasoningHistory(t *testing.T) {
|
||||
for _, tt := range []struct{ name, content, reasoning, want string }{
|
||||
{"separate", " Answer \n", " Reason \n", "<think>\n Reason \n\n</think>\n Answer \n"},
|
||||
{"already tagged", "<think>\noriginal\n</think>\nAnswer", "ignored", "<think>\noriginal\n</think>\nAnswer"},
|
||||
{"leading whitespace already tagged", "\x1c\t\u2003<think>original</think>Answer", "ignored", "\x1c\t\u2003<think>original</think>Answer"},
|
||||
{"nonleading tag", "Text <think>literal</think>", "Reason", "<think>\nReason\n</think>\nText <think>literal</think>"},
|
||||
{"empty reasoning", "Answer", "", "Answer"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
for _, think := range []*api.ThinkValue{nil, {Value: true}, {Value: false}, {Value: "high"}} {
|
||||
got, err := RenderWithRenderer("ripple", []api.Message{{Role: "assistant", Content: tt.content, Thinking: tt.reasoning}, {Role: "user", Content: "Next"}}, nil, think)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "<|im_start|>assistant\n" + tt.want + "<|im_end|>\n<|im_start|>user\nNext<|im_end|>\n<|im_start|>assistant\n"
|
||||
if got != want {
|
||||
t.Fatalf("got %q\nwant %q", got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRippleRendererMessages(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
messages []api.Message
|
||||
want string
|
||||
}{
|
||||
{"plain", []api.Message{{Role: "user", Content: " Hi\n"}}, "<|im_start|>user\n Hi\n<|im_end|>\n<|im_start|>assistant\n"},
|
||||
{"system", []api.Message{{Role: "system", Content: "Be precise."}, {Role: "user", Content: "Hi"}}, "<|im_start|>system\nBe precise.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n"},
|
||||
{"empty system", []api.Message{{Role: "system"}}, "<|im_start|>system\n<|im_end|>\n<|im_start|>assistant\n"},
|
||||
{"later system", []api.Message{{Role: "user", Content: "Hi"}, {Role: "system", Content: "Now"}}, "<|im_start|>user\nHi<|im_end|>\n<|im_start|>system\nNow<|im_end|>\n<|im_start|>assistant\n"},
|
||||
{"assistant is closed not prefilled", []api.Message{{Role: "assistant", Content: "Part"}}, "<|im_start|>assistant\nPart<|im_end|>\n<|im_start|>assistant\n"},
|
||||
{"unknown role ignored", []api.Message{{Role: "developer", Content: "ignored"}}, "<|im_start|>assistant\n"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := RenderWithRenderer("ripple", tt.messages, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("got %q\nwant %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if rendererForName("ripple") == nil {
|
||||
t.Fatal("missing ripple renderer")
|
||||
}
|
||||
if got := LeadingBOSForRenderer("ripple"); got != "" {
|
||||
t.Fatalf("BOS = %q", got)
|
||||
}
|
||||
if got := ThinkingForRenderer("ripple"); got != nil {
|
||||
t.Fatalf("unexpected thinking control: %+v", got)
|
||||
}
|
||||
}
|
||||
52
model/renderers/testdata/ripple_chat_template.jinja
vendored
Normal file
52
model/renderers/testdata/ripple_chat_template.jinja
vendored
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- set content = message.content or '' %}
|
||||
{%- if message.role == 'assistant' and message.reasoning_content and not (clear_thinking | default(false)) and not content.lstrip().startswith('<think>') %}
|
||||
{%- set content = '<think>\n' + message.reasoning_content + '\n</think>\n' + content %}
|
||||
{%- endif %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if content %}
|
||||
{{- '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": ' }}
|
||||
{{- tool_call.name | tojson }}
|
||||
{{- ', "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' + message.content + '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
Loading…
Add table
Add a link
Reference in a new issue