mirror of
https://github.com/ollama/ollama.git
synced 2026-09-21 22:04:25 +00:00
laguna: wire the DFlash target side
Add what a DFlash draft borrows from its target: the tapped layer
outputs, the raw embedding lookup, and the undecorated lm_head
projection. The laguna draft architecture (DFlashLagunaForCausalLM) is
registered here, alongside the only wired target.
Matched nvfp4 target+draft pairs, M5 Max, temp 0.8, repeat_penalty 1.1,
adaptive depth; decode tok/s:
prose code edit
laguna-xs plain 139.4 139.7 137.3
DFlash 142.3 139.2 145.1
laguna-s plain 75.4 70.0 72.4
DFlash 74.6 80.8 115.3
This commit is contained in:
parent
cf129bbb11
commit
b880b76c43
2 changed files with 31 additions and 0 deletions
|
|
@ -20,6 +20,9 @@ func init() {
|
|||
base.RegisterDraft("DFlashDraftModel", func(root *model.Root, target base.Model) (base.DraftModel, error) {
|
||||
return newModel(root, target, false)
|
||||
})
|
||||
base.RegisterDraft("DFlashLagunaForCausalLM", func(root *model.Root, target base.Model) (base.DraftModel, error) {
|
||||
return newModel(root, target, true)
|
||||
})
|
||||
}
|
||||
|
||||
var _ base.BlockDraft = (*Model)(nil)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/ollama/ollama/x/mlxrunner/batch"
|
||||
|
|
@ -83,6 +84,9 @@ type Model struct {
|
|||
Norm *nn.RMSNorm
|
||||
LMHead nn.LinearLayer
|
||||
|
||||
// auxHiddenLayers are the tapped layers; empty means the final hidden.
|
||||
auxHiddenLayers []int
|
||||
|
||||
tok *tokenizer.Tokenizer
|
||||
*Config
|
||||
}
|
||||
|
|
@ -1395,6 +1399,7 @@ func (m *Model) Forward(b *batch.Batch, caches []cache.Cache) (hidden, auxHidden
|
|||
|
||||
func (m *Model) forward(b *batch.Batch, caches []cache.Cache, B, L int32) (hidden, auxHidden *mlx.Array) {
|
||||
positions := mlx.FromValues(b.SeqOffsets, len(b.SeqOffsets))
|
||||
var features []*mlx.Array
|
||||
h := m.EmbedTokens.Forward(b.InputIDs)
|
||||
for i, layer := range m.Layers {
|
||||
var c cache.Cache
|
||||
|
|
@ -1402,11 +1407,34 @@ func (m *Model) forward(b *batch.Batch, caches []cache.Cache, B, L int32) (hidde
|
|||
c = caches[i]
|
||||
}
|
||||
h = layer.Forward(h, b, c, positions, B, L, m.Config)
|
||||
if slices.Contains(m.auxHiddenLayers, i) {
|
||||
features = append(features, h)
|
||||
}
|
||||
}
|
||||
out := m.Norm.Forward(h, m.RMSNormEps)
|
||||
if features != nil {
|
||||
return out, mlx.Concatenate(features, -1)
|
||||
}
|
||||
return out, out
|
||||
}
|
||||
|
||||
// SetAuxHiddenLayers taps the listed layers' outputs, which Forward then
|
||||
// returns as the draft-conditioning state in place of the final hidden.
|
||||
func (m *Model) SetAuxHiddenLayers(layers []int) {
|
||||
m.auxHiddenLayers = layers
|
||||
}
|
||||
|
||||
// TokenEmbeddings is the raw lookup, for a draft that embeds with the
|
||||
// target's table.
|
||||
func (m *Model) TokenEmbeddings(ids *mlx.Array) *mlx.Array {
|
||||
return m.EmbedTokens.Forward(ids)
|
||||
}
|
||||
|
||||
// RawLogits matches Unembed: this head applies no output decoration.
|
||||
func (m *Model) RawLogits(hidden *mlx.Array) *mlx.Array {
|
||||
return m.LMHead.Forward(hidden)
|
||||
}
|
||||
|
||||
func (m *Model) Unembed(x *mlx.Array) *mlx.Array {
|
||||
return m.LMHead.Forward(x)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue