mirror of
https://github.com/ollama/ollama.git
synced 2026-08-04 14:56:15 +00:00
add MTP wiring for llama-server and create with GGUFs
This commit is contained in:
Daniel Hiltgen
parent
e6015135ca
commit
da36fff9f7
21 changed files with 804 additions and 95 deletions
30
api/types.go
30
api/types.go
|
|
@ -600,12 +600,13 @@ type Options struct {
|
|||
|
||||
// Runner options which must be set when the model is loaded into memory
|
||||
type Runner struct {
|
||||
NumCtx int `json:"num_ctx,omitempty"`
|
||||
NumBatch int `json:"num_batch,omitempty"`
|
||||
NumGPU int `json:"num_gpu,omitempty"`
|
||||
MainGPU *int `json:"main_gpu,omitempty"`
|
||||
UseMMap *bool `json:"use_mmap,omitempty"`
|
||||
NumThread int `json:"num_thread,omitempty"`
|
||||
NumCtx int `json:"num_ctx,omitempty"`
|
||||
NumBatch int `json:"num_batch,omitempty"`
|
||||
NumGPU int `json:"num_gpu,omitempty"`
|
||||
MainGPU *int `json:"main_gpu,omitempty"`
|
||||
UseMMap *bool `json:"use_mmap,omitempty"`
|
||||
NumThread int `json:"num_thread,omitempty"`
|
||||
DraftNumPredict int `json:"draft_num_predict,omitempty"`
|
||||
}
|
||||
|
||||
// EmbedRequest is the request passed to [Client.Embed].
|
||||
|
|
@ -672,6 +673,9 @@ type CreateRequest struct {
|
|||
// Quantize is the quantization format for the model; leave blank to not change the quantization level.
|
||||
Quantize string `json:"quantize,omitempty"`
|
||||
|
||||
// DraftQuantize is the quantization format for the draft model.
|
||||
DraftQuantize string `json:"draft_quantize,omitempty"`
|
||||
|
||||
// From is the name of the model or file to use as the source.
|
||||
From string `json:"from,omitempty"`
|
||||
|
||||
|
|
@ -681,6 +685,9 @@ type CreateRequest struct {
|
|||
// Files is a map of files include when creating the model.
|
||||
Files map[string]string `json:"files,omitempty"`
|
||||
|
||||
// DraftFiles is a map of draft model files to include when creating the model.
|
||||
DraftFiles map[string]string `json:"draft_files,omitempty"`
|
||||
|
||||
// Adapters is a map of LoRA adapters to include when creating the model.
|
||||
Adapters map[string]string `json:"adapters,omitempty"`
|
||||
|
||||
|
|
@ -1100,11 +1107,12 @@ func DefaultOptions() Options {
|
|||
|
||||
Runner: Runner{
|
||||
// options set when the model is loaded
|
||||
NumCtx: int(envconfig.ContextLength()),
|
||||
NumBatch: 512,
|
||||
NumGPU: -1, // -1 here indicates that NumGPU should be set dynamically
|
||||
NumThread: 0, // let the runtime decide
|
||||
UseMMap: nil,
|
||||
NumCtx: int(envconfig.ContextLength()),
|
||||
NumBatch: 512,
|
||||
NumGPU: -1, // -1 here indicates that NumGPU should be set dynamically
|
||||
NumThread: 0, // let the runtime decide
|
||||
DraftNumPredict: 4,
|
||||
UseMMap: nil,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
26
cmd/cmd.go
26
cmd/cmd.go
|
|
@ -234,9 +234,6 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
|
|||
// This gates both safetensors LLM and imagegen model creation
|
||||
experimental, _ := cmd.Flags().GetBool("experimental")
|
||||
draftQuantize, _ := cmd.Flags().GetString("draft-quantize")
|
||||
if draftQuantize != "" && !experimental {
|
||||
return errors.New("--draft-quantize requires --experimental")
|
||||
}
|
||||
if experimental {
|
||||
if !isLocalhost() {
|
||||
return errors.New("remote safetensor model creation not yet supported")
|
||||
|
|
@ -331,6 +328,12 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
|
|||
if quantize != "" {
|
||||
req.Quantize = quantize
|
||||
}
|
||||
if draftQuantize != "" {
|
||||
if len(req.DraftFiles) == 0 {
|
||||
return errors.New("--draft-quantize requires a DRAFT model")
|
||||
}
|
||||
req.DraftQuantize = draftQuantize
|
||||
}
|
||||
|
||||
client, err := api.ClientFromEnvironment()
|
||||
if err != nil {
|
||||
|
|
@ -366,12 +369,26 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
|
|||
})
|
||||
}
|
||||
|
||||
draftFiles := syncmap.NewSyncMap[string, string]()
|
||||
draftFileNames := createRequestFileNames(req.DraftFiles)
|
||||
for f, digest := range req.DraftFiles {
|
||||
g.Go(func() error {
|
||||
if _, err := createBlob(cmd, client, f, digest, p); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
draftFiles.Store(draftFileNames[f], digest)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Files = files.Items()
|
||||
req.Adapters = adapters.Items()
|
||||
req.DraftFiles = draftFiles.Items()
|
||||
|
||||
bars := make(map[string]*progress.Bar)
|
||||
fn := func(resp api.ProgressResponse) error {
|
||||
|
|
@ -2336,9 +2353,6 @@ func NewCLI() *cobra.Command {
|
|||
if experimental, _ := cmd.Flags().GetBool("experimental"); experimental {
|
||||
return nil
|
||||
}
|
||||
if draftQuantize, _ := cmd.Flags().GetString("draft-quantize"); draftQuantize != "" {
|
||||
return errors.New("--draft-quantize requires --experimental")
|
||||
}
|
||||
return checkServerHeartbeat(cmd, args)
|
||||
},
|
||||
RunE: CreateHandler,
|
||||
|
|
|
|||
|
|
@ -1568,34 +1568,22 @@ func TestCreateRequestFileNamesPreservesRelativeModelDirectoryLayout(t *testing.
|
|||
}
|
||||
}
|
||||
|
||||
func TestCreateHandlerDraftQuantizeRequiresExperimental(t *testing.T) {
|
||||
cmd := &cobra.Command{}
|
||||
cmd.Flags().Bool("experimental", false, "")
|
||||
cmd.Flags().String("draft-quantize", "mxfp8", "")
|
||||
cmd.SetContext(t.Context())
|
||||
|
||||
err := CreateHandler(cmd, []string{"test-model"})
|
||||
if err == nil || !strings.Contains(err.Error(), "--draft-quantize requires --experimental") {
|
||||
t.Fatalf("error = %v, want draft-quantize requires experimental", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateHandlerDraftRequiresExperimental(t *testing.T) {
|
||||
func TestCreateHandlerDraftQuantizeRequiresDraft(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
modelfile := filepath.Join(dir, "Modelfile")
|
||||
if err := os.WriteFile(modelfile, []byte("FROM base\nDRAFT ./assistant\n"), 0o644); err != nil {
|
||||
if err := os.WriteFile(modelfile, []byte("FROM base\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cmd := &cobra.Command{}
|
||||
cmd.Flags().Bool("experimental", false, "")
|
||||
cmd.Flags().String("draft-quantize", "", "")
|
||||
cmd.Flags().String("file", modelfile, "")
|
||||
cmd.Flags().String("draft-quantize", "mxfp8", "")
|
||||
cmd.SetContext(t.Context())
|
||||
|
||||
err := CreateHandler(cmd, []string{"test-model"})
|
||||
if err == nil || !strings.Contains(err.Error(), "DRAFT requires --experimental") {
|
||||
t.Fatalf("error = %v, want DRAFT requires --experimental", err)
|
||||
if err == nil || !strings.Contains(err.Error(), "--draft-quantize requires a DRAFT model") {
|
||||
t.Fatalf("error = %v, want draft-quantize requires DRAFT", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,8 +8,11 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"math"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/d4l3k/go-bfloat16"
|
||||
|
|
@ -38,6 +41,8 @@ type qwen3NextTextConfig struct {
|
|||
MaxPositionEmbeddings uint32 `json:"max_position_embeddings"`
|
||||
HiddenSize uint32 `json:"hidden_size"`
|
||||
NumHiddenLayers uint32 `json:"num_hidden_layers"`
|
||||
NumNextNPredictLayers uint32 `json:"num_nextn_predict_layers"`
|
||||
MTPNumHiddenLayers uint32 `json:"mtp_num_hidden_layers"`
|
||||
IntermediateSize uint32 `json:"intermediate_size"`
|
||||
NumAttentionHeads uint32 `json:"num_attention_heads"`
|
||||
NumKeyValueHeads uint32 `json:"num_key_value_heads"`
|
||||
|
|
@ -114,6 +119,16 @@ func (q *qwen3NextModel) parseMore(fsys fs.FS) error {
|
|||
if q.TextConfig != nil {
|
||||
q.qwen3NextTextConfig = *q.TextConfig
|
||||
}
|
||||
if q.NumNextNPredictLayers == 0 {
|
||||
q.NumNextNPredictLayers = q.MTPNumHiddenLayers
|
||||
}
|
||||
if q.NumNextNPredictLayers == 0 {
|
||||
nextn, err := qwen3NextInferNextNPredictLayers(fsys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q.NumNextNPredictLayers = nextn
|
||||
}
|
||||
|
||||
if q.RopeTheta == 0 {
|
||||
q.RopeTheta = q.RopeParameters.RopeTheta
|
||||
|
|
@ -194,6 +209,150 @@ func (q *qwen3NextModel) parseMore(fsys fs.FS) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func qwen3NextInferNextNPredictLayers(fsys fs.FS) (uint32, error) {
|
||||
paths, err := fs.Glob(fsys, "*.safetensors")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
maxLayer := -1
|
||||
hasMTP := false
|
||||
for _, p := range paths {
|
||||
f, err := fsys.Open(p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var n int64
|
||||
if err := binary.Read(f, binary.LittleEndian, &n); err != nil {
|
||||
f.Close()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
b := bytes.NewBuffer(make([]byte, 0, n))
|
||||
if _, err = io.CopyN(b, f, n); err != nil {
|
||||
f.Close()
|
||||
return 0, err
|
||||
}
|
||||
f.Close()
|
||||
|
||||
var headers map[string]safetensorMetadata
|
||||
if err := json.NewDecoder(b).Decode(&headers); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
for name, value := range headers {
|
||||
if value.Type == "" || !strings.HasPrefix(name, "mtp.") {
|
||||
continue
|
||||
}
|
||||
hasMTP = true
|
||||
rest := strings.TrimPrefix(name, "mtp.layers.")
|
||||
layer, suffix, ok := strings.Cut(rest, ".")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(layer)
|
||||
if err == nil && n > maxLayer && suffix != "" {
|
||||
maxLayer = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if maxLayer >= 0 {
|
||||
return uint32(maxLayer + 1), nil
|
||||
}
|
||||
if hasMTP {
|
||||
return 1, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func ConvertQwen35MTPDraft(fsys fs.FS, f *os.File, baseKV ggml.KV, baseTensors []*ggml.Tensor) error {
|
||||
arch := baseKV.Architecture()
|
||||
if arch != "qwen35" && arch != "qwen35moe" {
|
||||
return fmt.Errorf("MTP draft safetensors require a qwen3.5 base model, got %q", arch)
|
||||
}
|
||||
|
||||
baseBlocks := baseKV.Uint("block_count")
|
||||
if baseBlocks == 0 {
|
||||
return fmt.Errorf("MTP draft safetensors require a base model with block_count")
|
||||
}
|
||||
if baseKV.Uint("nextn_predict_layers") > 0 {
|
||||
return fmt.Errorf("MTP draft safetensors require a base model without embedded MTP layers")
|
||||
}
|
||||
|
||||
nextn, err := qwen3NextInferNextNPredictLayers(fsys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if nextn == 0 {
|
||||
return fmt.Errorf("MTP draft safetensors did not contain mtp tensors")
|
||||
}
|
||||
|
||||
q := &qwen3NextModel{
|
||||
qwen3NextTextConfig: qwen3NextTextConfig{
|
||||
NumHiddenLayers: baseBlocks,
|
||||
NumNextNPredictLayers: nextn,
|
||||
},
|
||||
}
|
||||
ts, err := parseTensors(fsys, strings.NewReplacer(q.Replacements()...))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureUniqueTensorNames(ts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mtpTensors := q.Tensors(ts)
|
||||
if len(mtpTensors) == 0 {
|
||||
return fmt.Errorf("MTP draft safetensors did not produce GGUF tensors")
|
||||
}
|
||||
for _, tensor := range mtpTensors {
|
||||
if !qwen35MTPDraftTensorName(tensor.Name, baseBlocks, nextn) {
|
||||
return fmt.Errorf("MTP draft safetensors produced unexpected tensor %q", tensor.Name)
|
||||
}
|
||||
tensor.Shape = slices.Clone(tensor.Shape)
|
||||
slices.Reverse(tensor.Shape)
|
||||
}
|
||||
|
||||
kv := maps.Clone(baseKV)
|
||||
qwen35RemoveSplitMetadata(kv, arch)
|
||||
kv[arch+".block_count"] = baseBlocks + nextn
|
||||
kv[arch+".nextn_predict_layers"] = nextn
|
||||
|
||||
tensors := make([]*ggml.Tensor, 0, len(baseTensors)+len(mtpTensors))
|
||||
tensors = append(tensors, baseTensors...)
|
||||
tensors = append(tensors, mtpTensors...)
|
||||
|
||||
var parameters uint64
|
||||
for _, tensor := range tensors {
|
||||
parameters += tensor.Elements()
|
||||
}
|
||||
kv["general.parameter_count"] = parameters
|
||||
|
||||
return ggml.WriteGGUF(f, kv, tensors)
|
||||
}
|
||||
|
||||
func qwen35RemoveSplitMetadata(kv ggml.KV, arch string) {
|
||||
for _, key := range []string{
|
||||
"split.no",
|
||||
"split.count",
|
||||
"split.tensors.count",
|
||||
} {
|
||||
delete(kv, key)
|
||||
delete(kv, arch+"."+key)
|
||||
}
|
||||
}
|
||||
|
||||
func qwen35MTPDraftTensorName(name string, base, nextn uint32) bool {
|
||||
for i := range nextn {
|
||||
if strings.HasPrefix(name, fmt.Sprintf("blk.%d.", base+i)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (q *qwen3NextModel) kvHeadCounts() ([]uint32, error) {
|
||||
if len(q.LayerTypes) > 0 {
|
||||
kv := make([]uint32, q.NumHiddenLayers)
|
||||
|
|
@ -271,7 +430,10 @@ func (q *qwen3NextModel) KV(t *Tokenizer) KV {
|
|||
}
|
||||
kv["general.architecture"] = arch
|
||||
kv["tokenizer.ggml.pre"] = "qwen35"
|
||||
kv["block_count"] = q.NumHiddenLayers
|
||||
kv["block_count"] = q.NumHiddenLayers + q.NumNextNPredictLayers
|
||||
if q.NumNextNPredictLayers > 0 {
|
||||
kv["nextn_predict_layers"] = q.NumNextNPredictLayers
|
||||
}
|
||||
kv["context_length"] = q.MaxPositionEmbeddings
|
||||
kv["embedding_length"] = q.HiddenSize
|
||||
kv["feed_forward_length"] = q.IntermediateSize
|
||||
|
|
@ -488,7 +650,7 @@ func (q *qwen3NextModel) ProjectorKV(*Tokenizer) KV {
|
|||
func (q *qwen3NextModel) TextTensors(ts []Tensor, _ *Tokenizer) []*ggml.Tensor {
|
||||
var text []Tensor
|
||||
for _, t := range ts {
|
||||
if qwen3NextVisionTensor(t.Name()) || strings.HasPrefix(t.Name(), "mtp.") {
|
||||
if qwen3NextVisionTensor(t.Name()) {
|
||||
continue
|
||||
}
|
||||
text = append(text, t)
|
||||
|
|
@ -795,6 +957,13 @@ func (q *qwen3NextModel) Tensors(ts []Tensor) []*ggml.Tensor {
|
|||
name := t.Name()
|
||||
shape := t.Shape()
|
||||
|
||||
if names := q.mtpTensorNames(name); len(names) > 0 {
|
||||
for _, name := range names {
|
||||
out = q.appendDirectTensor(out, t, name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasSuffix(name, ".ssm_in.weight") {
|
||||
if qkv, gate, ok := q.splitQKVZTensor(t); ok {
|
||||
out = append(out, qkv, gate)
|
||||
|
|
@ -896,6 +1065,73 @@ func (q *qwen3NextModel) Tensors(ts []Tensor) []*ggml.Tensor {
|
|||
return out
|
||||
}
|
||||
|
||||
func (q *qwen3NextModel) appendDirectTensor(out []*ggml.Tensor, t Tensor, name string) []*ggml.Tensor {
|
||||
if qwen3NextShouldShiftNorm(name) {
|
||||
t = t.Clone()
|
||||
t.SetRepacker(q.addOne)
|
||||
}
|
||||
return append(out, &ggml.Tensor{Name: name, Kind: t.Kind(), Shape: slices.Clone(t.Shape()), WriterTo: t})
|
||||
}
|
||||
|
||||
func qwen3NextShouldShiftNorm(name string) bool {
|
||||
if strings.HasSuffix(name, ".ssm_norm.weight") {
|
||||
return false
|
||||
}
|
||||
return strings.HasSuffix(name, "_norm.weight") ||
|
||||
strings.HasSuffix(name, ".nextn.enorm.weight") ||
|
||||
strings.HasSuffix(name, ".nextn.hnorm.weight")
|
||||
}
|
||||
|
||||
func (q *qwen3NextModel) mtpTensorNames(name string) []string {
|
||||
if !strings.HasPrefix(name, "mtp.") {
|
||||
return nil
|
||||
}
|
||||
|
||||
base := q.NumHiddenLayers
|
||||
nextn := q.NumNextNPredictLayers
|
||||
if nextn == 0 {
|
||||
nextn = 1
|
||||
}
|
||||
|
||||
if rest := strings.TrimPrefix(name, "mtp.layers."); rest != name {
|
||||
layer, suffix, ok := strings.Cut(rest, ".")
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
idx, err := strconv.ParseUint(layer, 10, 32)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return []string{fmt.Sprintf("blk.%d.%s", base+uint32(idx), suffix)}
|
||||
}
|
||||
|
||||
var suffix string
|
||||
switch name {
|
||||
case "mtp.fc.weight":
|
||||
suffix = "nextn.eh_proj.weight"
|
||||
case "mtp.pre_fc_norm_embedding.weight":
|
||||
suffix = "nextn.enorm.weight"
|
||||
case "mtp.pre_fc_norm_hidden.weight":
|
||||
suffix = "nextn.hnorm.weight"
|
||||
case "mtp.norm.weight":
|
||||
suffix = "nextn.shared_head_norm.weight"
|
||||
case "mtp.embed_tokens.weight":
|
||||
suffix = "nextn.embed_tokens.weight"
|
||||
case "mtp.shared_head.head.weight":
|
||||
suffix = "nextn.shared_head_head.weight"
|
||||
case "mtp.shared_head.norm.weight":
|
||||
suffix = "nextn.shared_head_norm.weight"
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
names := make([]string, 0, nextn)
|
||||
for i := range nextn {
|
||||
names = append(names, fmt.Sprintf("blk.%d.%s", base+i, suffix))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func (q *qwen3NextModel) repackReorderDim(dim, headDim int) Repacker {
|
||||
return func(_ string, data []float32, shape []uint64) ([]float32, error) {
|
||||
if !q.shouldReorderVHeads() {
|
||||
|
|
|
|||
|
|
@ -259,6 +259,63 @@ func TestQwen35KVFromTextConfig(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestQwen35MTPTensors(t *testing.T) {
|
||||
m := &qwen3NextModel{
|
||||
ModelParameters: ModelParameters{
|
||||
ModelType: "qwen3_5",
|
||||
},
|
||||
qwen3NextTextConfig: qwen3NextTextConfig{
|
||||
NumHiddenLayers: 32,
|
||||
NumNextNPredictLayers: 1,
|
||||
},
|
||||
}
|
||||
|
||||
kv := m.KV(&Tokenizer{Vocabulary: &Vocabulary{}})
|
||||
if got, want := kv["block_count"], uint32(33); got != want {
|
||||
t.Fatalf("unexpected block_count: got %v want %v", got, want)
|
||||
}
|
||||
if got, want := kv["nextn_predict_layers"], uint32(1); got != want {
|
||||
t.Fatalf("unexpected nextn_predict_layers: got %v want %v", got, want)
|
||||
}
|
||||
|
||||
tensors := m.Tensors([]Tensor{
|
||||
&fakeTensor{name: "mtp.fc.weight", shape: []uint64{2, 2}, data: make([]float32, 4)},
|
||||
&fakeTensor{name: "mtp.pre_fc_norm_embedding.weight", shape: []uint64{2}, data: []float32{0, 1}},
|
||||
&fakeTensor{name: "mtp.pre_fc_norm_hidden.weight", shape: []uint64{2}, data: []float32{0, 1}},
|
||||
&fakeTensor{name: "mtp.norm.weight", shape: []uint64{2}, data: []float32{0, 1}},
|
||||
&fakeTensor{name: "mtp.layers.0.attn_q.weight", shape: []uint64{2, 2}, data: make([]float32, 4)},
|
||||
&fakeTensor{name: "mtp.layers.0.ffn_down.weight", shape: []uint64{2, 2}, data: make([]float32, 4)},
|
||||
})
|
||||
|
||||
byName := map[string]*ggml.Tensor{}
|
||||
for _, tensor := range tensors {
|
||||
byName[tensor.Name] = tensor
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"blk.32.nextn.eh_proj.weight",
|
||||
"blk.32.nextn.enorm.weight",
|
||||
"blk.32.nextn.hnorm.weight",
|
||||
"blk.32.nextn.shared_head_norm.weight",
|
||||
"blk.32.attn_q.weight",
|
||||
"blk.32.ffn_down.weight",
|
||||
} {
|
||||
if _, ok := byName[name]; !ok {
|
||||
t.Fatalf("missing MTP tensor %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"blk.32.nextn.enorm.weight",
|
||||
"blk.32.nextn.hnorm.weight",
|
||||
"blk.32.nextn.shared_head_norm.weight",
|
||||
} {
|
||||
if got, want := readTensorData(t, byName[name]), []float32{1, 2}; !slices.Equal(got, want) {
|
||||
t.Fatalf("unexpected shifted norm values for %s: got %v want %v", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQwen35NativeSplitKV(t *testing.T) {
|
||||
m := &qwen3NextModel{
|
||||
ModelParameters: ModelParameters{
|
||||
|
|
|
|||
|
|
@ -398,6 +398,7 @@ curl http://localhost:11434/api/generate -d '{
|
|||
"num_keep": 5,
|
||||
"seed": 42,
|
||||
"num_predict": 100,
|
||||
"draft_num_predict": 4,
|
||||
"top_k": 20,
|
||||
"top_p": 0.9,
|
||||
"min_p": 0.0,
|
||||
|
|
|
|||
|
|
@ -157,6 +157,7 @@ PARAMETER <parameter> <parametervalue>
|
|||
| seed | Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. (Default: 0) | int | seed 42 |
|
||||
| stop | Sets the stop sequences to use. When this pattern is encountered the LLM will stop generating text and return. Multiple stop patterns may be set by specifying multiple separate `stop` parameters in a modelfile. | string | stop "AI assistant:" |
|
||||
| num_predict | Maximum number of tokens to predict when generating text. (Default: -1, infinite generation) | int | num_predict 42 |
|
||||
| draft_num_predict | Maximum number of speculative draft tokens to predict per step when a draft model is available. Set to 0 to disable speculative drafting. (Default: 4) | int | draft_num_predict 4 |
|
||||
| top_k | Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40) | int | top_k 40 |
|
||||
| top_p | Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9) | float | top_p 0.9 |
|
||||
| min_p | Alternative to the top*p, and aims to ensure a balance of quality and variety. The parameter \_p* represents the minimum probability for a token to be considered, relative to the probability of the most likely token. For example, with _p_=0.05 and the most likely token having a probability of 0.9, logits with a value less than 0.045 are filtered out. (Default: 0.0) | float | min_p 0.05 |
|
||||
|
|
|
|||
|
|
@ -303,6 +303,7 @@ func startLlamaServer(
|
|||
params = appendJinjaArgs(params, config)
|
||||
|
||||
params = appendMMProjArgs(params, modelPath, projectors, opts, gpus, modelLayers)
|
||||
params = appendMTPDraftArgs(params, config, opts)
|
||||
|
||||
params = append(params, qwenVLServerArgs(modelArch)...)
|
||||
|
||||
|
|
@ -607,6 +608,26 @@ func appendJinjaArgs(params []string, config LlamaServerConfig) []string {
|
|||
return params
|
||||
}
|
||||
|
||||
func appendMTPDraftArgs(params []string, config LlamaServerConfig, opts api.Options) []string {
|
||||
if !config.EnableMTP && config.DraftModelPath == "" {
|
||||
return params
|
||||
}
|
||||
if opts.DraftNumPredict <= 0 {
|
||||
return params
|
||||
}
|
||||
|
||||
params = append(params, "--spec-type", "draft-mtp")
|
||||
params = append(params, "--spec-draft-n-max", strconv.Itoa(opts.DraftNumPredict))
|
||||
if config.DraftModelPath != "" {
|
||||
params = append(params, "--spec-draft-model", config.DraftModelPath)
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
func hasMTPDraft(f *ggml.GGML) bool {
|
||||
return f.KV().Uint("nextn_predict_layers") > 0
|
||||
}
|
||||
|
||||
// NewLlamaServerRunner creates a new llama-server runner that wraps the upstream llama-server binary.
|
||||
func NewLlamaServerRunner(
|
||||
gpus []ml.DeviceInfo,
|
||||
|
|
@ -651,6 +672,9 @@ func NewLlamaServerRunner(
|
|||
compatClipArches[arch] {
|
||||
projectors = []string{modelPath}
|
||||
}
|
||||
if config.DraftModelPath == "" && hasMTPDraft(f) {
|
||||
config.EnableMTP = true
|
||||
}
|
||||
|
||||
gpuLibs := ml.LibraryPaths(gpus)
|
||||
status := NewStatusWriter(os.Stderr)
|
||||
|
|
@ -2163,8 +2187,10 @@ var deviceFreeRegex = regexp.MustCompile(`using device (\S+)\s+\(.*\)\s+-\s+(\d+
|
|||
// component so repeated fit/probe values can be replaced by the final load.
|
||||
var bufferSizeRegex = regexp.MustCompile(`(?m)(?:^|\n)[^\n:]*?([A-Za-z_][A-Za-z0-9_]*):\s+(\S+)\s+(model|KV|compute|output|RS)\s+buffer size\s*=\s*([\d.]+)\s*MiB`)
|
||||
|
||||
var offloadedLayersRegex = regexp.MustCompile(`offloaded\s+(\d+)/(\d+)\s+layers to GPU`)
|
||||
var fitOverflowingLayersRegex = regexp.MustCompile(`common_params_fit_impl:\s+-\s+.+:\s+\d+\s+layers\s+\(\s*(\d+)\s+overflowing\)`)
|
||||
var (
|
||||
offloadedLayersRegex = regexp.MustCompile(`offloaded\s+(\d+)/(\d+)\s+layers to GPU`)
|
||||
fitOverflowingLayersRegex = regexp.MustCompile(`common_params_fit_impl:\s+-\s+.+:\s+\d+\s+layers\s+\(\s*(\d+)\s+overflowing\)`)
|
||||
)
|
||||
|
||||
// isGPUBuffer returns true if the backend buffer name represents GPU memory.
|
||||
// CPU, BLAS, and host-pinned buffers (*_Host) are not GPU memory.
|
||||
|
|
|
|||
|
|
@ -1447,6 +1447,48 @@ func TestAppendJinjaArgs(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestAppendMTPDraftArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config LlamaServerConfig
|
||||
opts api.Options
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "no draft model leaves speculative decoding disabled",
|
||||
opts: api.Options{Runner: api.Runner{DraftNumPredict: 4}},
|
||||
want: []string{"base"},
|
||||
},
|
||||
{
|
||||
name: "embedded draft uses configured draft depth",
|
||||
config: LlamaServerConfig{EnableMTP: true},
|
||||
opts: api.Options{Runner: api.Runner{DraftNumPredict: 4}},
|
||||
want: []string{"base", "--spec-type", "draft-mtp", "--spec-draft-n-max", "4"},
|
||||
},
|
||||
{
|
||||
name: "separate draft model uses configured draft depth",
|
||||
config: LlamaServerConfig{DraftModelPath: "draft.gguf"},
|
||||
opts: api.Options{Runner: api.Runner{DraftNumPredict: 8}},
|
||||
want: []string{"base", "--spec-type", "draft-mtp", "--spec-draft-n-max", "8", "--spec-draft-model", "draft.gguf"},
|
||||
},
|
||||
{
|
||||
name: "zero draft depth disables speculative decoding",
|
||||
config: LlamaServerConfig{EnableMTP: true, DraftModelPath: "draft.gguf"},
|
||||
opts: api.Options{Runner: api.Runner{DraftNumPredict: 0}},
|
||||
want: []string{"base"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := appendMTPDraftArgs([]string{"base"}, tt.config, tt.opts)
|
||||
if !slices.Equal(got, tt.want) {
|
||||
t.Fatalf("appendMTPDraftArgs = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testIntPtr(v int) *int {
|
||||
return &v
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,9 @@ type LlamaServer interface {
|
|||
}
|
||||
|
||||
type LlamaServerConfig struct {
|
||||
DisableJinja bool
|
||||
DisableJinja bool
|
||||
EnableMTP bool
|
||||
DraftModelPath string
|
||||
}
|
||||
|
||||
// LoadModel will load a model from disk. The model must be in the GGML format.
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ type Layer struct {
|
|||
|
||||
const (
|
||||
MediaTypeImageTensor = "application/vnd.ollama.image.tensor"
|
||||
MediaTypeImageDraft = "application/vnd.ollama.image.draft"
|
||||
)
|
||||
|
||||
func NewLayer(r io.Reader, mediatype string) (Layer, error) {
|
||||
|
|
|
|||
|
|
@ -84,7 +84,23 @@ func (f Modelfile) CreateRequest(relativeDir string) (*api.CreateRequest, error)
|
|||
}
|
||||
}
|
||||
case "draft":
|
||||
return nil, errors.New("DRAFT requires --experimental")
|
||||
path, err := expandPath(c.Args, relativeDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
digestMap, err := fileDigestMap(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if req.DraftFiles == nil {
|
||||
req.DraftFiles = digestMap
|
||||
} else {
|
||||
for k, v := range digestMap {
|
||||
req.DraftFiles[k] = v
|
||||
}
|
||||
}
|
||||
case "adapter":
|
||||
path, err := expandPath(c.Args, relativeDir)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -73,15 +73,21 @@ DRAFT ./assistant
|
|||
assert.Contains(t, modelfile.String(), "DRAFT ./assistant")
|
||||
}
|
||||
|
||||
func TestCreateRequestDraftRequiresExperimental(t *testing.T) {
|
||||
func TestCreateRequestDraftFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
draft := filepath.Join(dir, "draft.gguf")
|
||||
require.NoError(t, os.WriteFile(draft, []byte("draft"), 0o644))
|
||||
|
||||
modelfile, err := ParseFile(strings.NewReader(`
|
||||
FROM base
|
||||
DRAFT ./assistant
|
||||
DRAFT ./draft.gguf
|
||||
`))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = modelfile.CreateRequest("")
|
||||
require.ErrorContains(t, err, "DRAFT requires --experimental")
|
||||
req, err := modelfile.CreateRequest(dir)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, req.DraftFiles, 1)
|
||||
assert.Contains(t, req.DraftFiles, draft)
|
||||
}
|
||||
|
||||
func TestParseFileTrimSpace(t *testing.T) {
|
||||
|
|
|
|||
315
server/create.go
315
server/create.go
|
|
@ -10,6 +10,7 @@ import (
|
|||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -78,6 +79,21 @@ func (s *Server) CreateHandler(c *gin.Context) {
|
|||
}
|
||||
}
|
||||
|
||||
for v, digest := range r.DraftFiles {
|
||||
if !fs.ValidPath(v) {
|
||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": errFilePath.Error()})
|
||||
return
|
||||
}
|
||||
if digest == "" {
|
||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": manifest.ErrInvalidDigestFormat.Error()})
|
||||
return
|
||||
}
|
||||
}
|
||||
if r.DraftQuantize != "" && len(r.DraftFiles) == 0 {
|
||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "--draft-quantize requires a DRAFT model"})
|
||||
return
|
||||
}
|
||||
|
||||
for _, digest := range r.Adapters {
|
||||
if digest == "" {
|
||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": manifest.ErrInvalidDigestFormat.Error()})
|
||||
|
|
@ -190,6 +206,21 @@ func (s *Server) CreateHandler(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
var draftLayers []*layerGGML
|
||||
if !remote && r.DraftFiles != nil {
|
||||
draftLayers, err = convertDraftModelFromFiles(r.DraftFiles, baseLayers, fn)
|
||||
if err != nil {
|
||||
for _, badReq := range []error{errNoFilesProvided, errOnlyGGUFSupported, errUnknownType, errFilePath} {
|
||||
if errors.Is(err, badReq) {
|
||||
ch <- gin.H{"error": err.Error(), "status": http.StatusBadRequest}
|
||||
return
|
||||
}
|
||||
}
|
||||
ch <- gin.H{"error": err.Error(), "status": http.StatusBadRequest}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var adapterLayers []*layerGGML
|
||||
if !remote && r.Adapters != nil {
|
||||
adapterLayers, err = convertModelFromFiles(r.Adapters, baseLayers, true, fn)
|
||||
|
|
@ -208,6 +239,9 @@ func (s *Server) CreateHandler(c *gin.Context) {
|
|||
if len(adapterLayers) > 0 {
|
||||
baseLayers = append(baseLayers, adapterLayers...)
|
||||
}
|
||||
if len(draftLayers) > 0 {
|
||||
baseLayers = append(baseLayers, draftLayers...)
|
||||
}
|
||||
|
||||
// Info is not currently exposed by Modelfiles, but allows overriding various
|
||||
// config values
|
||||
|
|
@ -332,9 +366,17 @@ func remoteURL(raw string) (string, error) {
|
|||
}
|
||||
|
||||
func convertModelFromFiles(files map[string]string, baseLayers []*layerGGML, isAdapter bool, fn func(resp api.ProgressResponse)) ([]*layerGGML, error) {
|
||||
return convertModelFromFilesWithMediaType(files, baseLayers, isAdapter, "", true, fn)
|
||||
}
|
||||
|
||||
func convertDraftModelFromFiles(files map[string]string, baseLayers []*layerGGML, fn func(resp api.ProgressResponse)) ([]*layerGGML, error) {
|
||||
return convertModelFromFilesWithMediaType(files, baseLayers, false, manifest.MediaTypeImageDraft, false, fn)
|
||||
}
|
||||
|
||||
func convertModelFromFilesWithMediaType(files map[string]string, baseLayers []*layerGGML, isAdapter bool, mediaType string, detectTemplate bool, fn func(resp api.ProgressResponse)) ([]*layerGGML, error) {
|
||||
switch detectModelTypeFromFiles(files) {
|
||||
case "safetensors":
|
||||
layers, err := convertFromSafetensors(files, baseLayers, isAdapter, fn)
|
||||
layers, err := convertFromSafetensors(files, baseLayers, isAdapter, mediaType, detectTemplate, fn)
|
||||
if err != nil {
|
||||
slog.Error("error converting from safetensors", "error", err)
|
||||
return nil, err
|
||||
|
|
@ -357,7 +399,7 @@ func convertModelFromFiles(files map[string]string, baseLayers []*layerGGML, isA
|
|||
var splitGroupKeys []string
|
||||
splitGroups := map[string][]*layerGGML{}
|
||||
for _, filePath := range filePaths {
|
||||
layers, err := ggufLayers(files[filePath], filePath, fn)
|
||||
layers, err := ggufLayersWithMediaType(files[filePath], filePath, mediaType, fn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -383,7 +425,10 @@ func convertModelFromFiles(files map[string]string, baseLayers []*layerGGML, isA
|
|||
allLayers = append(allLayers, layer)
|
||||
}
|
||||
|
||||
return detectChatTemplate(allLayers)
|
||||
if detectTemplate {
|
||||
return detectChatTemplate(allLayers)
|
||||
}
|
||||
return allLayers, nil
|
||||
default:
|
||||
return nil, errUnknownType
|
||||
}
|
||||
|
|
@ -427,7 +472,7 @@ func detectModelTypeFromFiles(files map[string]string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
func convertFromSafetensors(files map[string]string, baseLayers []*layerGGML, isAdapter bool, fn func(resp api.ProgressResponse)) ([]*layerGGML, error) {
|
||||
func convertFromSafetensors(files map[string]string, baseLayers []*layerGGML, isAdapter bool, mediaType string, detectTemplate bool, fn func(resp api.ProgressResponse)) ([]*layerGGML, error) {
|
||||
tmpDir, err := os.MkdirTemp(envconfig.Models(), "ollama-safetensors")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -473,12 +518,17 @@ func convertFromSafetensors(files map[string]string, baseLayers []*layerGGML, is
|
|||
defer projFile.Close()
|
||||
}
|
||||
|
||||
var mediaType string
|
||||
if !isAdapter {
|
||||
fn(api.ProgressResponse{Status: "converting model"})
|
||||
mediaType = "application/vnd.ollama.image.model"
|
||||
if err := convert.ConvertModel(os.DirFS(tmpDir), t, projFile); err != nil {
|
||||
return nil, err
|
||||
mediaType = cmp.Or(mediaType, "application/vnd.ollama.image.model")
|
||||
if mediaType == manifest.MediaTypeImageDraft {
|
||||
if err := convertMTPDraftFromSafetensors(os.DirFS(tmpDir), t, baseLayers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err := convert.ConvertModel(os.DirFS(tmpDir), t, projFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
kv, err := kvFromLayers(baseLayers)
|
||||
|
|
@ -544,11 +594,85 @@ func convertFromSafetensors(files map[string]string, baseLayers []*layerGGML, is
|
|||
}
|
||||
layers = append(layers, projectorLayer)
|
||||
}
|
||||
return detectChatTemplate(layers)
|
||||
if detectTemplate {
|
||||
return detectChatTemplate(layers)
|
||||
}
|
||||
}
|
||||
return layers, nil
|
||||
}
|
||||
|
||||
func convertMTPDraftFromSafetensors(fsys fs.FS, out *os.File, baseLayers []*layerGGML) error {
|
||||
baseLayer, err := baseModelLayer(baseLayers)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tensors, cleanup, err := baseLayerTensors(baseLayer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
return convert.ConvertQwen35MTPDraft(fsys, out, baseLayer.GGML.KV(), tensors)
|
||||
}
|
||||
|
||||
func baseLayerTensors(layer *layerGGML) ([]*ggml.Tensor, func(), error) {
|
||||
if len(layer.splitParts) == 0 {
|
||||
blobPath, err := manifest.BlobsPath(layer.Digest)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
blob, err := os.Open(blobPath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
tensors := tensorsFromGGUFFile(blob, layer.GGML)
|
||||
return tensors, func() { blob.Close() }, nil
|
||||
}
|
||||
|
||||
var files []*os.File
|
||||
tensors := make([]*ggml.Tensor, 0, len(layer.GGML.Tensors().Items()))
|
||||
cleanup := func() {
|
||||
for _, f := range files {
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
for _, part := range layer.splitParts {
|
||||
blobPath, err := manifest.BlobsPath(part.Digest)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return nil, nil, err
|
||||
}
|
||||
blob, err := os.Open(blobPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return nil, nil, err
|
||||
}
|
||||
files = append(files, blob)
|
||||
tensors = append(tensors, tensorsFromGGUFFile(blob, part.GGML)...)
|
||||
}
|
||||
|
||||
return tensors, cleanup, nil
|
||||
}
|
||||
|
||||
func tensorsFromGGUFFile(file *os.File, f *ggml.GGML) []*ggml.Tensor {
|
||||
tensors := make([]*ggml.Tensor, 0, len(f.Tensors().Items()))
|
||||
for _, tensor := range f.Tensors().Items() {
|
||||
tensors = append(tensors, tensorFromFile(file, f.Tensors().Offset+tensor.Offset, tensor))
|
||||
}
|
||||
return tensors
|
||||
}
|
||||
|
||||
func baseModelLayer(layers []*layerGGML) (*layerGGML, error) {
|
||||
for _, layer := range layers {
|
||||
if layer.GGML != nil && layer.MediaType == "application/vnd.ollama.image.model" {
|
||||
return layer, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("no base model was found")
|
||||
}
|
||||
|
||||
func kvFromLayers(baseLayers []*layerGGML) (ofs.Config, error) {
|
||||
for _, l := range baseLayers {
|
||||
if l.GGML != nil {
|
||||
|
|
@ -562,19 +686,34 @@ func createModel(r api.CreateRequest, name model.Name, baseLayers []*layerGGML,
|
|||
var layers []manifest.Layer
|
||||
for _, layer := range baseLayers {
|
||||
if layer.GGML != nil {
|
||||
quantType := strings.ToUpper(cmp.Or(r.Quantize, r.Quantization))
|
||||
if layer.rewriteForCreate && layer.GGML.Name() == "gguf" && len(layer.splitParts) > 0 && layerHasEmbeddedCompatibilityTensors(layer) {
|
||||
var err error
|
||||
layer, err = copySplitLayerPreservingTensors(layer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
quantType := ""
|
||||
if layer.MediaType == "application/vnd.ollama.image.model" {
|
||||
quantType = strings.ToUpper(cmp.Or(r.Quantize, r.Quantization))
|
||||
} else if layer.MediaType == manifest.MediaTypeImageDraft {
|
||||
quantType = strings.ToUpper(r.DraftQuantize)
|
||||
}
|
||||
ft := layer.GGML.KV().FileType()
|
||||
rewroteLayer := false
|
||||
if quantType == "" && hasSourceFP8Tensors(layer.GGML.KV()) && layer.GGML.Name() == "gguf" && layer.MediaType == "application/vnd.ollama.image.model" && slices.Contains([]string{"F16", "BF16", "F32"}, ft.String()) {
|
||||
quantType = "Q8_0"
|
||||
}
|
||||
if quantType != "" && layer.GGML.Name() == "gguf" && layer.MediaType == "application/vnd.ollama.image.model" {
|
||||
if quantType != "" && layer.GGML.Name() == "gguf" && slices.Contains([]string{"application/vnd.ollama.image.model", manifest.MediaTypeImageDraft}, layer.MediaType) {
|
||||
want, err := ggml.ParseFileType(quantType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !slices.Contains([]string{"F16", "BF16", "F32"}, ft.String()) {
|
||||
if layer.MediaType == manifest.MediaTypeImageDraft && ft.ToTensorType().IsQuantized() {
|
||||
return fmt.Errorf("draft quantization requires an unquantized draft model, got %s", ft)
|
||||
} else if !slices.Contains([]string{"F16", "BF16", "F32"}, ft.String()) {
|
||||
return errors.New("quantization is only supported for F16, BF16 and F32 models")
|
||||
} else if ft != want {
|
||||
layer, err = quantizeLayer(layer, quantType, fn)
|
||||
|
|
@ -591,6 +730,13 @@ func createModel(r api.CreateRequest, name model.Name, baseLayers []*layerGGML,
|
|||
return err
|
||||
}
|
||||
}
|
||||
if !rewroteLayer && layer.rewriteForCreate && layer.GGML.Name() == "gguf" && layer.MediaType == manifest.MediaTypeImageDraft && len(layer.splitParts) > 0 {
|
||||
var err error
|
||||
layer, err = copyLayerWithLlamaQuantize(layer, fn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if layer.rewriteForCreate && layer.GGML.Name() == "gguf" && layer.MediaType == "application/vnd.ollama.image.projector" && needsDefaultLlavaProjectorType(layer.GGML) {
|
||||
var err error
|
||||
fn(api.ProgressResponse{Status: "updating GGUF projector metadata"})
|
||||
|
|
@ -599,33 +745,41 @@ func createModel(r api.CreateRequest, name model.Name, baseLayers []*layerGGML,
|
|||
return err
|
||||
}
|
||||
}
|
||||
config.ModelFormat = cmp.Or(config.ModelFormat, layer.GGML.Name())
|
||||
config.ModelFamily = cmp.Or(config.ModelFamily, layer.GGML.KV().Architecture())
|
||||
config.ModelType = cmp.Or(config.ModelType, format.HumanNumber(layer.GGML.KV().ParameterCount()))
|
||||
config.FileType = cmp.Or(config.FileType, layer.GGML.KV().FileType().String())
|
||||
config.ModelFamilies = append(config.ModelFamilies, layer.GGML.KV().Architecture())
|
||||
switch layer.MediaType {
|
||||
case "application/vnd.ollama.image.model":
|
||||
config.ModelFormat = cmp.Or(config.ModelFormat, layer.GGML.Name())
|
||||
config.ModelFamily = cmp.Or(config.ModelFamily, layer.GGML.KV().Architecture())
|
||||
config.ModelType = cmp.Or(config.ModelType, format.HumanNumber(layer.GGML.KV().ParameterCount()))
|
||||
config.FileType = cmp.Or(config.FileType, layer.GGML.KV().FileType().String())
|
||||
config.ModelFamilies = append(config.ModelFamilies, layer.GGML.KV().Architecture())
|
||||
|
||||
// Auto-detect renderer, parser, and stop tokens from GGUF architecture.
|
||||
// TODO: abstract this into a registry/lookup table when multiple models
|
||||
// need architecture-based renderer/parser/stop defaults.
|
||||
if config.Renderer == "" || config.Parser == "" {
|
||||
arch := layer.GGML.KV().Architecture()
|
||||
switch arch {
|
||||
case "gemma4":
|
||||
config.Renderer = cmp.Or(config.Renderer, gemma4RendererLegacy)
|
||||
config.Parser = cmp.Or(config.Parser, "gemma4")
|
||||
if _, ok := r.Parameters["stop"]; !ok {
|
||||
if r.Parameters == nil {
|
||||
r.Parameters = make(map[string]any)
|
||||
// Auto-detect renderer, parser, and stop tokens from GGUF architecture.
|
||||
// TODO: abstract this into a registry/lookup table when multiple models
|
||||
// need architecture-based renderer/parser/stop defaults.
|
||||
if config.Renderer == "" || config.Parser == "" {
|
||||
arch := layer.GGML.KV().Architecture()
|
||||
switch arch {
|
||||
case "gemma4":
|
||||
config.Renderer = cmp.Or(config.Renderer, gemma4RendererLegacy)
|
||||
config.Parser = cmp.Or(config.Parser, "gemma4")
|
||||
if _, ok := r.Parameters["stop"]; !ok {
|
||||
if r.Parameters == nil {
|
||||
r.Parameters = make(map[string]any)
|
||||
}
|
||||
r.Parameters["stop"] = []string{"<turn|>"}
|
||||
}
|
||||
r.Parameters["stop"] = []string{"<turn|>"}
|
||||
case "laguna":
|
||||
config.Renderer = cmp.Or(config.Renderer, "laguna")
|
||||
config.Parser = cmp.Or(config.Parser, "laguna")
|
||||
case "nemotron_h", "nemotron_h_moe", "nemotron_h_omni":
|
||||
config.Renderer = cmp.Or(config.Renderer, "nemotron-3-nano")
|
||||
config.Parser = cmp.Or(config.Parser, "nemotron-3-nano")
|
||||
}
|
||||
case "laguna":
|
||||
config.Renderer = cmp.Or(config.Renderer, "laguna")
|
||||
config.Parser = cmp.Or(config.Parser, "laguna")
|
||||
case "nemotron_h", "nemotron_h_moe", "nemotron_h_omni":
|
||||
config.Renderer = cmp.Or(config.Renderer, "nemotron-3-nano")
|
||||
config.Parser = cmp.Or(config.Parser, "nemotron-3-nano")
|
||||
}
|
||||
case manifest.MediaTypeImageDraft:
|
||||
config.Draft = &model.Draft{
|
||||
ModelFormat: layer.GGML.Name(),
|
||||
Architecture: layer.GGML.KV().Architecture(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -714,6 +868,18 @@ func hasEmbeddedCompatibilityTensors(f *ggml.GGML) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
func layerHasEmbeddedCompatibilityTensors(layer *layerGGML) bool {
|
||||
if hasEmbeddedCompatibilityTensors(layer.GGML) {
|
||||
return true
|
||||
}
|
||||
for _, part := range layer.splitParts {
|
||||
if part.GGML != nil && hasEmbeddedCompatibilityTensors(part.GGML) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isEmbeddedCompatibilityTensor(name string) bool {
|
||||
for _, prefix := range []string{"a.", "mm.", "mtp.", "s.", "v."} {
|
||||
if strings.HasPrefix(name, prefix) {
|
||||
|
|
@ -744,6 +910,61 @@ func copyLayerWithLlamaQuantize(layer *layerGGML, fn func(resp api.ProgressRespo
|
|||
return newLayer, nil
|
||||
}
|
||||
|
||||
func copySplitLayerPreservingTensors(layer *layerGGML) (*layerGGML, error) {
|
||||
blob, err := manifest.BlobsPath(layer.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tensors, cleanup, err := baseLayerTensors(layer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
kv := maps.Clone(layer.GGML.KV())
|
||||
removeSplitMetadata(kv, layer.GGML.KV().Architecture())
|
||||
|
||||
temp, err := os.CreateTemp(filepath.Dir(blob), "split-copy")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(temp.Name())
|
||||
defer temp.Close()
|
||||
|
||||
if err := ggml.WriteGGUF(temp, kv, tensors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := temp.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
newLayer, err := manifest.NewLayer(temp, layer.MediaType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := temp.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f, err := ggml.Decode(temp, 1024)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &layerGGML{Layer: newLayer, GGML: f}, nil
|
||||
}
|
||||
|
||||
func removeSplitMetadata(kv ggml.KV, arch string) {
|
||||
for _, key := range []string{
|
||||
"split.no",
|
||||
"split.count",
|
||||
"split.tensors.count",
|
||||
} {
|
||||
delete(kv, key)
|
||||
delete(kv, arch+"."+key)
|
||||
}
|
||||
}
|
||||
|
||||
func rewriteLayerWithLlamaQuantize(layer *layerGGML, typeName string, fn func(resp api.ProgressResponse), rewrite func(in, out *os.File, progressFn func(uint64)) error) (*layerGGML, error) {
|
||||
ft := layer.GGML.KV().FileType()
|
||||
var doneBytes atomic.Uint64
|
||||
|
|
@ -930,7 +1151,7 @@ func mergeSplitGGUFLayers(layers []*layerGGML) (*layerGGML, error) {
|
|||
if layer == nil {
|
||||
return nil, fmt.Errorf("split GGUF %q is missing shard %d", primary.From, i)
|
||||
}
|
||||
primary.splitParts = append(primary.splitParts, splitGGUFPart{Digest: layer.Digest, Name: layer.From})
|
||||
primary.splitParts = append(primary.splitParts, splitGGUFPart{Digest: layer.Digest, Name: layer.From, GGML: layer.GGML})
|
||||
}
|
||||
|
||||
return primary, nil
|
||||
|
|
@ -967,6 +1188,10 @@ func splitGGUFUint(kv ggml.KV, key string) (uint16, bool) {
|
|||
}
|
||||
|
||||
func ggufLayers(digest, sourceName string, fn func(resp api.ProgressResponse)) ([]*layerGGML, error) {
|
||||
return ggufLayersWithMediaType(digest, sourceName, "", fn)
|
||||
}
|
||||
|
||||
func ggufLayersWithMediaType(digest, sourceName, mediaType string, fn func(resp api.ProgressResponse)) ([]*layerGGML, error) {
|
||||
var layers []*layerGGML
|
||||
|
||||
fn(api.ProgressResponse{Status: "parsing GGUF"})
|
||||
|
|
@ -997,15 +1222,17 @@ func ggufLayers(digest, sourceName string, fn func(resp api.ProgressResponse)) (
|
|||
return nil, err
|
||||
}
|
||||
|
||||
mediatype := "application/vnd.ollama.image.model"
|
||||
if f.KV().Kind() == "adapter" {
|
||||
mediatype = "application/vnd.ollama.image.adapter"
|
||||
} else if (f.KV().Uint("block_count") == 0 && f.KV().Uint("vision.block_count") > 0) || f.KV().Kind() == "projector" {
|
||||
// if a model has vision.block_count but not block_count, it is a standalone vision model
|
||||
mediatype = "application/vnd.ollama.image.projector"
|
||||
if mediaType == "" {
|
||||
mediaType = "application/vnd.ollama.image.model"
|
||||
if f.KV().Kind() == "adapter" {
|
||||
mediaType = "application/vnd.ollama.image.adapter"
|
||||
} else if (f.KV().Uint("block_count") == 0 && f.KV().Uint("vision.block_count") > 0) || f.KV().Kind() == "projector" {
|
||||
// if a model has vision.block_count but not block_count, it is a standalone vision model
|
||||
mediaType = "application/vnd.ollama.image.projector"
|
||||
}
|
||||
}
|
||||
|
||||
layer, err := manifest.NewLayerFromLayer(digest, mediatype, sourceName)
|
||||
layer, err := manifest.NewLayerFromLayer(digest, mediaType, sourceName)
|
||||
if err != nil {
|
||||
slog.Debug("could not create new layer from layer", "error", err)
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ func TestConvertFromSafetensors(t *testing.T) {
|
|||
"tokenizer.json": tokenizer,
|
||||
}
|
||||
|
||||
_, err := convertFromSafetensors(files, nil, false, func(resp api.ProgressResponse) {})
|
||||
_, err := convertFromSafetensors(files, nil, false, "", true, func(resp api.ProgressResponse) {})
|
||||
|
||||
if (tt.wantErr == nil && err != nil) ||
|
||||
(tt.wantErr != nil && err == nil) ||
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ type Model struct {
|
|||
Config model.ConfigV2
|
||||
ShortName string
|
||||
ModelPath string
|
||||
DraftPath string
|
||||
ParentModel string
|
||||
HasChatTemplate bool
|
||||
HasLegacyTemplate bool
|
||||
|
|
@ -353,6 +354,13 @@ func (m *Model) String() string {
|
|||
})
|
||||
}
|
||||
|
||||
if m.DraftPath != "" {
|
||||
modelfile.Commands = append(modelfile.Commands, parser.Command{
|
||||
Name: "draft",
|
||||
Args: m.DraftPath,
|
||||
})
|
||||
}
|
||||
|
||||
for _, projector := range m.ProjectorPaths {
|
||||
modelfile.Commands = append(modelfile.Commands, parser.Command{
|
||||
Name: "model",
|
||||
|
|
@ -474,6 +482,8 @@ func GetModel(name string) (*Model, error) {
|
|||
modelHasPooling = f.KeyValue("pooling_type").Valid()
|
||||
f.Close()
|
||||
}
|
||||
case manifest.MediaTypeImageDraft:
|
||||
m.DraftPath = filename
|
||||
case "application/vnd.ollama.image.embed":
|
||||
// Deprecated in versions > 0.1.2
|
||||
// TODO: remove this warning in a future version
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ type layerGGML struct {
|
|||
type splitGGUFPart struct {
|
||||
Digest string
|
||||
Name string
|
||||
GGML *ggml.GGML
|
||||
}
|
||||
|
||||
func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
|
||||
|
|
@ -62,7 +63,8 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe
|
|||
switch layer.MediaType {
|
||||
case "application/vnd.ollama.image.model",
|
||||
"application/vnd.ollama.image.projector",
|
||||
"application/vnd.ollama.image.adapter":
|
||||
"application/vnd.ollama.image.adapter",
|
||||
manifest.MediaTypeImageDraft:
|
||||
blobpath, err := manifest.BlobsPath(layer.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -2321,7 +2321,8 @@ func chatModeForModel(m *Model) chatExecutionMode {
|
|||
|
||||
func llamaServerConfigForModel(m *Model) llm.LlamaServerConfig {
|
||||
return llm.LlamaServerConfig{
|
||||
DisableJinja: usesOllamaRenderedChat(m),
|
||||
DisableJinja: usesOllamaRenderedChat(m),
|
||||
DraftModelPath: m.DraftPath,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -329,6 +329,74 @@ func TestCreateModelValidatesSplitGGUFWithOriginalShardNames(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestBaseLayerTensorsReadsAllSplitGGUFShards(t *testing.T) {
|
||||
t.Setenv("OLLAMA_MODELS", t.TempDir())
|
||||
firstData := []byte{1, 2, 3, 4}
|
||||
secondData := []byte{5, 6, 7, 8}
|
||||
|
||||
_, firstDigest := createBinFile(t, ggml.KV{
|
||||
"general.architecture": "qwen35",
|
||||
"general.file_type": uint32(ggml.FileTypeF32),
|
||||
"qwen35.split.no": uint32(0),
|
||||
"qwen35.split.count": uint32(2),
|
||||
"qwen35.split.tensors.count": int32(2),
|
||||
}, []*ggml.Tensor{
|
||||
{
|
||||
Name: "blk.0.attn_q.weight",
|
||||
Kind: uint32(ggml.TensorTypeF32),
|
||||
Shape: []uint64{1},
|
||||
WriterTo: bytes.NewReader(firstData),
|
||||
},
|
||||
})
|
||||
_, secondDigest := createBinFile(t, ggml.KV{
|
||||
"general.architecture": "qwen35",
|
||||
"general.file_type": uint32(ggml.FileTypeF32),
|
||||
"qwen35.split.no": uint32(1),
|
||||
"qwen35.split.count": uint32(2),
|
||||
"qwen35.split.tensors.count": int32(2),
|
||||
}, []*ggml.Tensor{
|
||||
{
|
||||
Name: "blk.1.attn_q.weight",
|
||||
Kind: uint32(ggml.TensorTypeF32),
|
||||
Shape: []uint64{1},
|
||||
WriterTo: bytes.NewReader(secondData),
|
||||
},
|
||||
})
|
||||
|
||||
baseLayers, err := convertModelFromFiles(map[string]string{
|
||||
"model-00001-of-00002.gguf": firstDigest,
|
||||
"model-00002-of-00002.gguf": secondDigest,
|
||||
}, nil, false, func(api.ProgressResponse) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := len(baseLayers), 1; got != want {
|
||||
t.Fatalf("base layers = %d, want %d", got, want)
|
||||
}
|
||||
|
||||
tensors, cleanup, err := baseLayerTensors(baseLayers[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
byName := map[string][]byte{}
|
||||
for _, tensor := range tensors {
|
||||
var b bytes.Buffer
|
||||
if _, err := tensor.WriteTo(&b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byName[tensor.Name] = b.Bytes()
|
||||
}
|
||||
|
||||
if got := byName["blk.0.attn_q.weight"]; !bytes.Equal(got, firstData) {
|
||||
t.Fatalf("first shard tensor data = %v, want %v", got, firstData)
|
||||
}
|
||||
if got := byName["blk.1.attn_q.weight"]; !bytes.Equal(got, secondData) {
|
||||
t.Fatalf("second shard tensor data = %v, want %v", got, secondData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateModelAddsDefaultLlavaProjectorType(t *testing.T) {
|
||||
t.Setenv("OLLAMA_MODELS", t.TempDir())
|
||||
|
||||
|
|
|
|||
|
|
@ -1570,8 +1570,8 @@ func TestCreateSafetensorsModel_Qwen35Transforms(t *testing.T) {
|
|||
t.Fatalf("CreateSafetensorsModel failed: %v", err)
|
||||
}
|
||||
|
||||
if _, ok := calls["mtp.layers.0.foo.weight"]; ok {
|
||||
t.Fatal("mtp tensor should have been dropped")
|
||||
if _, ok := calls["mtp.layers.0.foo.weight"]; !ok {
|
||||
t.Fatal("mtp tensor should have been preserved")
|
||||
}
|
||||
|
||||
layerNorm := calls["language_model.model.layers.0.input_layernorm.weight"]
|
||||
|
|
|
|||
|
|
@ -58,7 +58,6 @@ func qwen35InspectSource(modelDir string) (qwen35SourceInfo, error) {
|
|||
info.hasPrequantizedWeights = true
|
||||
return info, nil
|
||||
}
|
||||
// This should change when MTP is supported
|
||||
if strings.Contains(name, "mtp.") {
|
||||
info.shouldShiftNormWeights = true
|
||||
continue
|
||||
|
|
@ -84,7 +83,7 @@ func qwen35InspectSource(modelDir string) (qwen35SourceInfo, error) {
|
|||
}
|
||||
|
||||
func (t qwen35ImportTransform) skipTensor(name string) bool {
|
||||
return strings.Contains(name, "mtp.")
|
||||
return false
|
||||
}
|
||||
|
||||
func qwen35ShouldKeepBF16ForDirectNonAffine(name string) bool {
|
||||
|
|
@ -254,6 +253,8 @@ func (t qwen35ImportTransform) canonicalTensorName(name string) string {
|
|||
return "vision_tower." + strings.TrimPrefix(name, "model.visual.")
|
||||
case strings.HasPrefix(name, "vision_tower."):
|
||||
return name
|
||||
case strings.HasPrefix(name, "mtp."):
|
||||
return name
|
||||
}
|
||||
|
||||
// Language model tensors: normalize to language_model.model.* prefix
|
||||
|
|
@ -274,6 +275,8 @@ func qwen35ShouldShiftNormKey(key string) bool {
|
|||
for _, suffix := range []string{
|
||||
".input_layernorm.weight",
|
||||
".post_attention_layernorm.weight",
|
||||
"mtp.pre_fc_norm_embedding.weight",
|
||||
"mtp.pre_fc_norm_hidden.weight",
|
||||
"model.norm.weight",
|
||||
".q_norm.weight",
|
||||
".k_norm.weight",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue