CERBERUS/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2026-10-14 19:34:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

server: extract GGUF metadata and unify capabilities (#17858)

Browse source 
Loading GGUF metadata is an expensive operation. Two caches had evolved to
mitigate this, and the two capability implementations produced inconsistent
results for some models.

This PR now extracts the metadata once per blob into a file at
<OLLAMA_MODELS>/metadata/sha256-<hex>.json. Only arrays over 4096 elements and
non-finite floats are left out.

Direct Capabilities() discovery now costs us instead of ms.  /api/tags can
build directly from manifests and the extracted metadata.
This commit is contained in:
Daniel Hiltgen 2026-09-10 11:16:37 -07:00 • committed by GitHub
parent c951dabec4
commit ea8d65004a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 1550 additions and 1679 deletions
Download patch file Download diff file Expand all files Collapse all files

38
fs/gguf/gguf.go
View file

@ -51,12 +51,19 @@ type File struct {
bts []byte
}
func Open(path string) (f *File, err error) {
f = &File{bts: make([]byte, 4096)}
func Open(path string) (_ *File, err error) {
f := &File{bts: make([]byte, 4096)}
f.file, err = os.Open(path)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
if closeErr := f.close(); closeErr != nil {
err = errors.Join(err, closeErr)
}
}
}()
f.reader = newBufferedReader(f.file, 32<<10)
@ -273,14 +280,14 @@ func readArrayData[T any](f *File, n uint64) (s []T, err error) {
return nil, err
}
s = make([]T, size)
for i := range size {
s = make([]T, 0, min(size, 4096))
for range size {
e, err := read[T](f)
if err != nil {
return nil, err
}
s[i] = e
s = append(s, e)
}
return s, nil
@ -292,14 +299,14 @@ func readArrayString(f *File, n uint64) (s []string, err error) {
return nil, err
}
s = make([]string, size)
for i := range size {
s = make([]string, 0, min(size, 4096))
for range size {
e, err := readString(f)
if err != nil {
return nil, err
}
s[i] = e
s = append(s, e)
}
return s, nil
@ -324,8 +331,19 @@ func maxInt64() uint64 {
}
func (f *File) Close() error {
f.keyValues.stop()
f.tensors.stop()
return f.close()
}
func (f *File) close() error {
if f.keyValues != nil {
f.keyValues.stop()
}
if f.tensors != nil {
f.tensors.stop()
}
if f.file == nil {
return nil
}
return f.file.Close()
}

5
fs/gguf/keyvalue.go
View file

@ -18,6 +18,11 @@ type Value struct {
value any
}
// Any returns Value as stored, without conversion. If it is not set, it returns nil.
func (v Value) Any() any {
return v.value
}
func value[T any](v Value, kinds ...reflect.Kind) (t T) {
vv := reflect.ValueOf(v.value)
if slices.Contains(kinds, vv.Kind()) {

15
manifest/manifest.go
View file

@ -71,10 +71,13 @@ func (m *Manifest) Remove() error {
return PruneDirectory(manifests)
}
func (m *Manifest) RemoveLayers() error {
// RemoveLayers deletes the layers no other manifest references. The digests it
// removed are returned even alongside an error, so callers can clean up
// anything derived from them.
func (m *Manifest) RemoveLayers() ([]string, error) {
ms, err := Manifests(true)
if err != nil {
return err
return nil, err
}
// Build set of digests still in use by other manifests
@ -88,6 +91,7 @@ func (m *Manifest) RemoveLayers() error {
}
// Remove layers not used by any other manifest
var removed []string
for _, layer := range append(m.Layers, m.Config) {
if layer.Digest == "" {
continue
@ -97,16 +101,17 @@ func (m *Manifest) RemoveLayers() error {
}
blob, err := BlobsPath(layer.Digest)
if err != nil {
return err
return removed, err
}
if err := os.Remove(blob); os.IsNotExist(err) {
slog.Debug("layer does not exist", "digest", layer.Digest)
} else if err != nil {
return err
return removed, err
}
removed = append(removed, layer.Digest)
}
return nil
return removed, nil
}
func ParseNamedManifest(n model.Name) (*Manifest, error) {

23
manifest/paths.go
View file

@ -14,6 +14,18 @@ import (
var ErrInvalidDigestFormat = errors.New("invalid digest format")
// a manifest spells a digest with ":", a filename with "-"
var digestPattern = regexp.MustCompile(`^sha256[:-][0-9a-fA-F]{64}$`)
// ValidateDigest reports whether digest names a blob.
func ValidateDigest(digest string) error {
if !digestPattern.MatchString(digest) {
return ErrInvalidDigestFormat
}
return nil
}
func Path() (string, error) {
path := filepath.Join(envconfig.Models(), "manifests")
if err := os.MkdirAll(path, 0o755); err != nil {
@ -38,12 +50,11 @@ func PathForName(n model.Name) (string, error) {
}
func BlobsPath(digest string) (string, error) {
// only accept actual sha256 digests
pattern := "^sha256[:-][0-9a-fA-F]{64}$"
re := regexp.MustCompile(pattern)
if digest != "" && !re.MatchString(digest) {
return "", ErrInvalidDigestFormat
// the empty digest names the blobs directory
if digest != "" {
if err := ValidateDigest(digest); err != nil {
return "", err
}
}
digest = strings.ReplaceAll(digest, ":", "-")

6
server/create.go
View file

@ -337,13 +337,13 @@ func (s *Server) CreateHandler(c *gin.Context) {
}
if !envconfig.NoPrune() && oldManifest != nil {
if err := oldManifest.RemoveLayers(); err != nil {
removed, err := oldManifest.RemoveLayers()
removeGGUFMetadata(removed...)
if err != nil {
ch <- gin.H{"error": err.Error()}
}
}
s.refreshModelListCache(name)
ch <- api.ProgressResponse{Status: "success"}
}()

338
server/gguf_metadata.go Normal file
View file

@ -0,0 +1,338 @@
package server
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"log/slog"
"math"
"os"
"path/filepath"
"strings"
"github.com/ollama/ollama/envconfig"
fsgguf "github.com/ollama/ollama/fs/gguf"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/version"
)
// A blob's metadata block is extracted once into a file beside the model
// store, keyed by blob digest. Values are held verbatim rather than derived, so
// reading them differently later needs no invalidation, and values left out are
// recorded by key so an absent key stays distinct from an uncopied one.
// Comfortably above per-layer arrays, which scale with block count, and far
// below any tokenizer vocabulary.
const ggufMetadataMaxArray = 4096
type ggufMetadata struct {
OllamaVersion string `json:"ollama_version,omitempty"`
// Values keyed exactly as they appear in the file. Never omitempty: absent
// on load means the file is not ours, and an all-omitted file is still
// usable.
KV map[string]any `json:"kv"`
// Keys present in the file whose values were too large to copy.
Omitted []string `json:"omitted,omitempty"`
}
// Valid reports whether the key is present. Keys are architecture qualified the
// same way fs/gguf does it.
func (m ggufMetadata) Valid(key string) bool {
_, ok := m.lookup(key)
return ok
}
func (m ggufMetadata) String(key string) string {
v, _ := m.lookup(key)
s, _ := v.(string)
return s
}
func (m ggufMetadata) Int(key string) int64 {
n, ok := m.number(key)
if !ok {
return 0
}
i, err := n.Int64()
if err != nil {
return 0
}
return i
}
func (m ggufMetadata) number(key string) (json.Number, bool) {
v, _ := m.lookup(key)
n, ok := v.(json.Number)
return n, ok
}
// Keys returns every key the file carried, including omitted ones.
func (m ggufMetadata) Keys() []string {
keys := make([]string, 0, len(m.KV)+len(m.Omitted))
for k := range m.KV {
keys = append(keys, k)
}
return append(keys, m.Omitted...)
}
func (m ggufMetadata) lookup(key string) (any, bool) {
if !strings.HasPrefix(key, "general.") && !strings.HasPrefix(key, "tokenizer.") {
arch, _ := m.KV["general.architecture"].(string)
key = arch + "." + key
}
v, ok := m.KV[key]
return v, ok
}
func ggufMetadataPath(digest string) (string, error) {
if err := manifest.ValidateDigest(digest); err != nil {
return "", fmt.Errorf("%w: %q", err, digest)
}
return filepath.Join(ggufMetadataDir(), strings.ReplaceAll(digest, ":", "-")+".json"), nil
}
func ggufMetadataDir() string {
return filepath.Join(envconfig.Models(), "metadata")
}
// readGGUFMetadata extracts the blob when there is no usable metadata file. Best
// effort throughout: a bad file is re-extracted, a failed write is dropped.
func readGGUFMetadata(digest string) (ggufMetadata, error) {
path, err := ggufMetadataPath(digest)
if err != nil {
return ggufMetadata{}, err
}
if md, ok := loadGGUFMetadata(path); ok {
return md, nil
}
blob, err := manifest.BlobsPath(digest)
if err != nil {
return ggufMetadata{}, err
}
md, err := extractGGUFMetadata(blob)
if err != nil {
return ggufMetadata{}, err
}
writeGGUFMetadata(path, blob, md)
return md, nil
}
func loadGGUFMetadata(path string) (ggufMetadata, bool) {
data, err := os.ReadFile(path)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
slog.Debug("could not read gguf metadata file", "path", path, "error", err)
}
return ggufMetadata{}, false
}
md, err := decodeGGUFMetadata(data)
if err != nil {
slog.Debug("ignoring unusable gguf metadata file", "path", path, "error", err)
return ggufMetadata{}, false
}
return md, true
}
func writeGGUFMetadata(path, blob string, md ggufMetadata) {
data, err := json.Marshal(md)
if err != nil {
slog.Debug("could not encode gguf metadata", "path", path, "error", err)
return
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
slog.Debug("could not create gguf metadata dir", "path", path, "error", err)
return
}
tmp, err := os.CreateTemp(filepath.Dir(path), ".gguf-metadata-*.tmp")
if err != nil {
slog.Debug("could not create gguf metadata temp file", "path", path, "error", err)
return
}
defer os.Remove(tmp.Name())
if _, err = tmp.Write(data); err == nil {
err = tmp.Sync()
}
if cerr := tmp.Close(); err == nil {
err = cerr
}
if err == nil {
// Can fail transiently on Windows while another process holds the path.
err = os.Rename(tmp.Name(), path)
}
if err == nil {
// Blob removal precedes metadata removal. Rechecking after publication
// ensures either this writer or a concurrent remover cleans up the file.
if _, statErr := os.Stat(blob); errors.Is(statErr, os.ErrNotExist) {
if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
slog.Debug("could not remove gguf metadata file", "path", path, "error", removeErr)
}
return
}
}
if err != nil {
slog.Debug("could not write gguf metadata file", "path", path, "error", err)
}
}
// removeGGUFMetadata drops metadata for deleted blobs. Pass only digests whose
// last reference is gone.
func removeGGUFMetadata(digests ...string) {
for _, digest := range digests {
path, err := ggufMetadataPath(digest)
if err != nil {
continue
}
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
slog.Debug("could not remove gguf metadata file", "path", path, "error", err)
}
}
}
func pruneGGUFMetadata() {
dir := ggufMetadataDir()
entries, err := os.ReadDir(dir)
if errors.Is(err, os.ErrNotExist) {
return
}
if err != nil {
slog.Debug("could not read gguf metadata dir", "path", dir, "error", err)
return
}
for _, entry := range entries {
name := entry.Name()
if entry.IsDir() || !strings.HasSuffix(name, ".json") {
continue
}
digest := strings.TrimSuffix(name, ".json")
blob, err := manifest.BlobsPath(digest)
if err != nil {
continue
}
if _, err := os.Stat(blob); err == nil || !errors.Is(err, os.ErrNotExist) {
continue
}
path := filepath.Join(dir, name)
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
slog.Debug("could not remove gguf metadata file", "path", path, "error", err)
}
}
}
// extractGGUFMetadata reads the whole block; proving a key absent means reaching
// the end. Normalized through JSON so extracted and loaded metadata hold the
// same value types.
func extractGGUFMetadata(path string) (ggufMetadata, error) {
md, err := scanGGUFMetadata(path)
if err != nil {
return ggufMetadata{}, err
}
md.OllamaVersion = version.Version
data, err := json.Marshal(md)
if err != nil {
return ggufMetadata{}, err
}
return decodeGGUFMetadata(data)
}
func decodeGGUFMetadata(data []byte) (ggufMetadata, error) {
// UseNumber keeps integers exact; the default would widen them to float64.
dec := json.NewDecoder(bytes.NewReader(data))
dec.UseNumber()
var md ggufMetadata
if err := dec.Decode(&md); err != nil {
return ggufMetadata{}, err
}
if md.KV == nil {
return ggufMetadata{}, fmt.Errorf("no metadata")
}
return md, nil
}
func scanGGUFMetadata(path string) (ggufMetadata, error) {
f, err := fsgguf.Open(path)
if err != nil {
return ggufMetadata{}, err
}
defer f.Close()
md := ggufMetadata{KV: make(map[string]any)}
for _, kv := range f.KeyValues() {
if omitValue(kv.Any()) {
md.Omitted = append(md.Omitted, kv.Key)
continue
}
md.KV[kv.Key] = kv.Any()
}
if err := f.Err(); err != nil {
return ggufMetadata{}, err
}
return md, nil
}
// omitValue reports values left out of the metadata file: long arrays, which are the
// tokenizer, and non-finite floats, which JSON cannot represent.
func omitValue(v any) bool {
switch v := v.(type) {
case float32:
return !finite(float64(v))
case float64:
return !finite(v)
case []float32:
for _, f := range v {
if !finite(float64(f)) {
return true
}
}
case []float64:
for _, f := range v {
if !finite(f) {
return true
}
}
}
return oversized(v)
}
func finite(f float64) bool {
return !math.IsInf(f, 0) && !math.IsNaN(f)
}
func oversized(v any) bool {
switch v := v.(type) {
case []string:
return len(v) > ggufMetadataMaxArray
case []int8:
return len(v) > ggufMetadataMaxArray
case []int16:
return len(v) > ggufMetadataMaxArray
case []int32:
return len(v) > ggufMetadataMaxArray
case []int64:
return len(v) > ggufMetadataMaxArray
case []uint8:
return len(v) > ggufMetadataMaxArray
case []uint16:
return len(v) > ggufMetadataMaxArray
case []uint32:
return len(v) > ggufMetadataMaxArray
case []uint64:
return len(v) > ggufMetadataMaxArray
case []float32:
return len(v) > ggufMetadataMaxArray
case []float64:
return len(v) > ggufMetadataMaxArray
case []bool:
return len(v) > ggufMetadataMaxArray
}
return false
}

736
server/gguf_metadata_test.go Normal file
View file

@ -0,0 +1,736 @@
package server
import (
"encoding/binary"
"encoding/json"
"errors"
"math"
"net/http"
"os"
"path/filepath"
"reflect"
"runtime"
"slices"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/envconfig"
"github.com/ollama/ollama/fs/ggml"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/types/model"
)
func TestGGUFMetadataExtraction(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
big := make([]string, 5000)
for i := range big {
big[i] = "tok"
}
perLayer := make([]int32, 64)
for i := range perLayer {
perLayer[i] = int32(i)
}
path, _ := createBinFile(t, ggml.KV{
"general.architecture": "bert",
"bert.attention.softcap": float32(math.Inf(-1)),
"general.file_type": uint32(2),
"bert.pooling_type": uint32(1),
"bert.context_length": uint32(512),
"bert.attention.head_count_kv": perLayer,
"tokenizer.chat_template": "{{ messages }}",
"tokenizer.ggml.tokens": big,
"tokenizer.ggml.merges": big,
}, nil)
md, err := extractGGUFMetadata(path)
if err != nil {
t.Fatal(err)
}
if got := md.String("general.architecture"); got != "bert" {
t.Errorf("architecture = %q", got)
}
// Architecture-prefixed lookup, and the key order in the file must not matter.
if !md.Valid("pooling_type") {
t.Error("pooling_type not found")
}
if got := md.Int("context_length"); got != 512 {
t.Errorf("context_length = %d, want 512", got)
}
if got := md.String("tokenizer.chat_template"); got != "{{ messages }}" {
t.Errorf("chat_template = %q", got)
}
if md.Valid("vision.block_count") {
t.Error("absent key reported present")
}
// A 64-element per-layer array is real metadata and must survive.
if v, ok := md.KV["bert.attention.head_count_kv"].([]any); !ok || len(v) != 64 {
t.Errorf("per-layer array = %#v, want 64 elements", md.KV["bert.attention.head_count_kv"])
}
// The tokenizer is recorded as omitted, not silently missing.
want := map[string]bool{"tokenizer.ggml.tokens": true, "tokenizer.ggml.merges": true}
for _, key := range md.Omitted {
delete(want, key)
}
if len(want) != 0 {
t.Errorf("not reported omitted: %v (omitted=%v)", want, md.Omitted)
}
if _, ok := md.KV["tokenizer.ggml.tokens"]; ok {
t.Error("oversized array was copied into the metadata file")
}
// JSON cannot represent an infinity, and gemma3n ships one. Dropping that
// key must not cost the rest of the file.
if !slices.Contains(md.Omitted, "bert.attention.softcap") {
t.Errorf("non-finite float not omitted; omitted=%v", md.Omitted)
}
}
func TestGGUFMetadataFileRoundTrip(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
_, digest := createBinFile(t, ggml.KV{
"general.architecture": "llama",
"llama.block_count": uint32(32),
}, nil)
first, err := readGGUFMetadata(digest)
if err != nil {
t.Fatal(err)
}
if got := first.Int("block_count"); got != 32 {
t.Fatalf("block_count = %d, want 32", got)
}
// With the blob gone, only the metadata file can answer.
if err := removeBlob(t, digest); err != nil {
t.Fatal(err)
}
second, err := readGGUFMetadata(digest)
if err != nil {
t.Fatalf("metadata file read failed: %v", err)
}
if got := second.Int("block_count"); got != 32 {
t.Fatalf("block_count from metadata file = %d, want 32", got)
}
// And it goes away with the blob.
removeGGUFMetadata(digest)
path, err := ggufMetadataPath(digest)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("metadata file still present after removal: %v", err)
}
}
// The metadata file is only ever an optimization: anything unusable must be re-extracted.
func TestGGUFMetadataUnusableFile(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
_, digest := createBinFile(t, ggml.KV{"general.architecture": "llama"}, nil)
if _, err := readGGUFMetadata(digest); err != nil {
t.Fatal(err)
}
path, err := ggufMetadataPath(digest)
if err != nil {
t.Fatal(err)
}
for _, tt := range []struct {
name string
content []byte
}{
{"truncated", []byte(`{"kv":{"general.arch`)},
{"not json", []byte("\x00\x01\x02")},
{"empty", []byte{}},
{"wrong shape", []byte(`[1,2,3]`)},
{"no kv", []byte(`{"ollama_version":"1.2.3"}`)},
{"kv not an object", []byte(`{"kv":[]}`)},
} {
t.Run(tt.name, func(t *testing.T) {
if err := os.WriteFile(path, tt.content, 0o644); err != nil {
t.Fatal(err)
}
if _, ok := loadGGUFMetadata(path); ok {
t.Fatal("unusable metadata file reported usable")
}
md, err := readGGUFMetadata(digest)
if err != nil {
t.Fatal(err)
}
if got := md.String("general.architecture"); got != "llama" {
t.Fatalf("architecture = %q, want llama", got)
}
})
}
}
func TestGGUFMetadataRejectsMalformed(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
dir := t.TempDir()
for _, tt := range []struct {
name string
data []byte
}{
{"empty file", nil},
{"bad magic", []byte("NOPE")},
{"truncated header", []byte("GGUF\x03\x00\x00\x00")},
} {
t.Run(tt.name, func(t *testing.T) {
path := filepath.Join(dir, tt.name)
if err := os.WriteFile(path, tt.data, 0o600); err != nil {
t.Fatal(err)
}
if _, err := extractGGUFMetadata(path); err == nil {
t.Fatal("expected an error")
}
})
}
}
// The omission rule decides what never reaches the metadata file, so every array type
// has to agree on it. A miss here silently drops real metadata.
func TestGGUFMetadataOmitRule(t *testing.T) {
short, long := 64, ggufMetadataMaxArray+1
for _, tt := range []struct {
name string
keep any
omit any
}{
{"string", make([]string, short), make([]string, long)},
{"int8", make([]int8, short), make([]int8, long)},
{"int16", make([]int16, short), make([]int16, long)},
{"int32", make([]int32, short), make([]int32, long)},
{"int64", make([]int64, short), make([]int64, long)},
{"uint8", make([]uint8, short), make([]uint8, long)},
{"uint16", make([]uint16, short), make([]uint16, long)},
{"uint32", make([]uint32, short), make([]uint32, long)},
{"uint64", make([]uint64, short), make([]uint64, long)},
{"float32", make([]float32, short), make([]float32, long)},
{"float64", make([]float64, short), make([]float64, long)},
{"bool", make([]bool, short), make([]bool, long)},
} {
t.Run(tt.name, func(t *testing.T) {
if omitValue(tt.keep) {
t.Errorf("%d-element %s array omitted; real per-layer metadata would be lost", short, tt.name)
}
if !omitValue(tt.omit) {
t.Errorf("%d-element %s array kept; the tokenizer would be copied", long, tt.name)
}
})
}
// Scalars are always kept, except the floats JSON cannot encode.
for _, keep := range []any{"s", int32(0), uint32(0), float32(1.5), float64(1.5), true} {
if omitValue(keep) {
t.Errorf("scalar %#v omitted", keep)
}
}
for _, omit := range []any{
float32(math.Inf(1)), float32(math.Inf(-1)), float64(math.NaN()),
[]float32{1, float32(math.Inf(1))},
[]float64{math.NaN()},
} {
if !omitValue(omit) {
t.Errorf("non-finite %#v kept; json.Marshal would fail the whole file", omit)
}
}
}
func TestGGUFMetadataKeysIncludesOmitted(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
big := make([]string, ggufMetadataMaxArray+1)
path, _ := createBinFile(t, ggml.KV{
"general.architecture": "llama",
"llama.block_count": uint32(4),
"tokenizer.ggml.tokens": big,
}, nil)
md, err := extractGGUFMetadata(path)
if err != nil {
t.Fatal(err)
}
keys := md.Keys()
for _, want := range []string{"general.architecture", "llama.block_count", "tokenizer.ggml.tokens"} {
if !slices.Contains(keys, want) {
t.Errorf("Keys() = %v, missing %s", keys, want)
}
}
}
// Projector audio detection reads through Keys(), including prefixed forms.
func TestProjectorAudioFromMetadata(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
for _, tt := range []struct {
name string
kv ggml.KV
audio bool
}{
{"bare key", ggml.KV{"general.architecture": "clip", "has_audio_encoder": true}, true},
{"prefixed key", ggml.KV{"general.architecture": "clip", "clip.has_audio_encoder": true}, true},
{"present but false", ggml.KV{"general.architecture": "clip", "has_audio_encoder": false}, false},
{"absent", ggml.KV{"general.architecture": "clip"}, false},
} {
t.Run(tt.name, func(t *testing.T) {
path, _ := createBinFile(t, tt.kv, nil)
md, err := extractGGUFMetadata(path)
if err != nil {
t.Fatal(err)
}
if got := projectorHasAudio(md); got != tt.audio {
t.Errorf("projectorHasAudio = %v, want %v", got, tt.audio)
}
})
}
path, _ := createBinFile(t, ggml.KV{
"general.architecture": "clip",
"has_audio_encoder": true,
"clip.vision.projector_type": "gemma3nv",
}, nil)
md, err := extractGGUFMetadata(path)
if err != nil {
t.Fatal(err)
}
if !projectorSuppressesAudioCapability(md) {
t.Error("gemma3nv projector should suppress audio")
}
}
func TestGGUFMetadataPathRejectsBadDigest(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
hex := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
for _, digest := range []string{"", "sha256-short", "md5:" + hex, "../escape", "sha256:" + hex + "x"} {
if _, err := ggufMetadataPath(digest); err == nil {
t.Errorf("ggufMetadataPath(%q) accepted", digest)
}
}
for _, digest := range []string{"sha256:" + hex, "sha256-" + hex} {
if _, err := ggufMetadataPath(digest); err != nil {
t.Errorf("ggufMetadataPath(%q) rejected: %v", digest, err)
}
}
}
// A file can carry no values at all, and every value it does carry can be one
// the omission rule drops. Either way what is left still says which keys the
// file had, so it has to be storable: treating it as corrupt would rescan the
// blob on every request forever.
func TestGGUFMetadataWithNothingToCopy(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
path := filepath.Join(t.TempDir(), "empty.gguf")
if err := os.WriteFile(path, ggufBytes(0, nil), 0o600); err != nil {
t.Fatal(err)
}
md, err := extractGGUFMetadata(path)
if err != nil {
t.Fatalf("a file with no metadata is still a file: %v", err)
}
if len(md.KV) != 0 {
t.Errorf("KV = %v, want empty", md.KV)
}
encoded, err := json.Marshal(md)
if err != nil {
t.Fatal(err)
}
if _, err := decodeGGUFMetadata(encoded); err != nil {
t.Errorf("written metadata cannot be read back, so it would be rescanned forever: %v", err)
}
}
const allocBudget = 1 << 20
func ggufBytes(kvCount uint64, extra []byte) []byte {
b := make([]byte, 0, 24+len(extra))
b = binary.LittleEndian.AppendUint32(b, 0x46554747) // GGUF
b = binary.LittleEndian.AppendUint32(b, 3) // version
b = binary.LittleEndian.AppendUint64(b, 0) // tensor count
b = binary.LittleEndian.AppendUint64(b, kvCount) // kv count
return append(b, extra...)
}
func declaredGGUF(t *testing.T, kvCount uint64, extra []byte) string {
t.Helper()
b := ggufBytes(kvCount, extra)
path := filepath.Join(t.TempDir(), "declared.gguf")
if err := os.WriteFile(path, b, 0o600); err != nil {
t.Fatal(err)
}
t.Logf("file is %d bytes, declares %d KV entries", len(b), kvCount)
return path
}
func allocDelta(t *testing.T, fn func()) uint64 {
t.Helper()
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
fn()
runtime.ReadMemStats(&after)
return after.TotalAlloc - before.TotalAlloc
}
func TestGGUFMetadataBoundsDeclaredKVCount(t *testing.T) {
path := declaredGGUF(t, 1<<20, nil)
got := allocDelta(t, func() {
if _, err := extractGGUFMetadata(path); err == nil {
t.Error("expected an error from a 24-byte file")
}
})
t.Logf("allocated %d bytes before failing", got)
if got > allocBudget {
t.Errorf("allocated %d bytes from a 24-byte file, want at most %d", got, allocBudget)
}
}
// Cover every array element type because strings and fixed-width values use
// separate readers.
func TestGGUFMetadataBoundsDeclaredArray(t *testing.T) {
for _, tt := range []struct {
name string
elemType uint32
}{
{"uint8", 0},
{"int8", 1},
{"uint16", 2},
{"int16", 3},
{"uint32", 4},
{"int32", 5},
{"float32", 6},
{"bool", 7},
{"string", 8},
{"uint64", 10},
{"int64", 11},
{"float64", 12},
} {
t.Run(tt.name, func(t *testing.T) {
var extra []byte
extra = binary.LittleEndian.AppendUint64(extra, 3) // key length
extra = append(extra, "big"...)
extra = binary.LittleEndian.AppendUint32(extra, 9) // type: array
extra = binary.LittleEndian.AppendUint32(extra, tt.elemType)
extra = binary.LittleEndian.AppendUint64(extra, 8_000_000) // element count
path := declaredGGUF(t, 1, extra)
got := allocDelta(t, func() {
if _, err := extractGGUFMetadata(path); err == nil {
t.Error("expected an error")
}
})
if got > allocBudget {
t.Errorf("allocated %d bytes for a declared array of 8000000, want at most %d", got, allocBudget)
}
})
}
}
func removeBlob(t *testing.T, digest string) error {
t.Helper()
path, err := manifest.BlobsPath(digest)
if err != nil {
t.Fatal(err)
}
return os.Remove(path)
}
func metadataCount(t *testing.T) int {
t.Helper()
entries, err := os.ReadDir(filepath.Join(envconfig.Models(), "metadata"))
if err != nil {
return 0
}
return len(entries)
}
// createModelFromBlob and deleteModelNamed drive the handlers rather than
// writing manifests, so the metadata hooks under test are the ones the server
// actually runs.
func createModelFromBlob(t *testing.T, name, digest, tmpl string) {
t.Helper()
var s Server
w := createRequest(t, s.CreateHandler, api.CreateRequest{
Model: name, Files: map[string]string{"model.gguf": digest},
Template: tmpl, Stream: &stream,
})
if w.Code != http.StatusOK {
t.Fatalf("create %s: %d %s", name, w.Code, w.Body.String())
}
}
func deleteModelNamed(t *testing.T, name string) {
t.Helper()
var s Server
w := createRequest(t, s.DeleteHandler, api.DeleteRequest{Model: name})
if w.Code != http.StatusOK {
t.Fatalf("delete %s: %d %s", name, w.Code, w.Body.String())
}
}
// Metadata is derived from a blob, so it lives exactly as long as the blob: it
// survives while any manifest still references it and goes with the last one.
func TestGGUFMetadataRemovedWithLastReference(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("OLLAMA_MODELS", t.TempDir())
_, digest := createBinFile(t, ggml.KV{
"general.architecture": "llama",
"llama.block_count": uint32(1),
}, nil)
createModelFromBlob(t, "alias-one", digest, "")
createModelFromBlob(t, "alias-two", digest, "")
if _, err := readGGUFMetadata(digest); err != nil {
t.Fatal(err)
}
if got := metadataCount(t); got != 1 {
t.Fatalf("metadata files = %d, want 1 (both aliases share one blob)", got)
}
deleteModelNamed(t, "alias-one")
if got := metadataCount(t); got != 1 {
t.Fatalf("metadata files after removing one alias = %d, want 1", got)
}
deleteModelNamed(t, "alias-two")
if got := metadataCount(t); got != 0 {
t.Fatalf("metadata files after removing the last reference = %d, want 0", got)
}
}
// Recreating a model over the same name replaces its blob; the old blob's
// metadata must go with it.
func TestGGUFMetadataRemovedOnReplacement(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("OLLAMA_MODELS", t.TempDir())
_, first := createBinFile(t, ggml.KV{
"general.architecture": "llama",
"llama.block_count": uint32(1),
}, nil)
createModelFromBlob(t, "replace-me", first, "")
if _, err := readGGUFMetadata(first); err != nil {
t.Fatal(err)
}
_, second := createBinFile(t, ggml.KV{
"general.architecture": "llama",
"llama.block_count": uint32(2),
}, nil)
createModelFromBlob(t, "replace-me", second, "")
if _, err := readGGUFMetadata(second); err != nil {
t.Fatal(err)
}
if got := metadataCount(t); got != 1 {
t.Fatalf("metadata files after replacement = %d, want 1 (the old blob's should be gone)", got)
}
old, err := ggufMetadataPath(first)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(old); !os.IsNotExist(err) {
t.Errorf("metadata for the replaced blob still present: %v", err)
}
}
// A blob missing from disk is still a blob whose metadata has to go. It no
// longer appears in the blob scan, so pruning has to find its metadata directly.
func TestGGUFMetadataRemovedForMissingBlob(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
_, orphan := createBinFile(t, ggml.KV{"general.architecture": "llama"}, nil)
if _, err := readGGUFMetadata(orphan); err != nil {
t.Fatal(err)
}
if err := removeBlob(t, orphan); err != nil {
t.Fatal(err)
}
_, live := createBinFile(t, ggml.KV{"general.architecture": "bert"}, nil)
if _, err := readGGUFMetadata(live); err != nil {
t.Fatal(err)
}
if got := metadataCount(t); got != 2 {
t.Fatalf("metadata files before the sweep = %d, want 2", got)
}
if err := PruneLayers(); err != nil {
t.Fatal(err)
}
if got := metadataCount(t); got != 1 {
t.Errorf("metadata files = %d, want 1", got)
}
path, err := ggufMetadataPath(live)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(path); err != nil {
t.Errorf("metadata for existing blob was removed: %v", err)
}
}
func TestGGUFMetadataNotPublishedAfterDelete(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("OLLAMA_MODELS", t.TempDir())
_, digest := createBinFile(t, ggml.KV{
"general.architecture": "llama",
"general.description": strings.Repeat("x", 16<<20),
}, nil)
createModelFromBlob(t, "delete-during-load", digest, "")
loaded := make(chan error, 1)
go func() {
_, err := GetModel("delete-during-load")
loaded <- err
}()
deadline := time.Now().Add(5 * time.Second)
for {
entries, err := os.ReadDir(ggufMetadataDir())
if err != nil && !errors.Is(err, os.ErrNotExist) {
t.Fatal(err)
}
if slices.ContainsFunc(entries, func(entry os.DirEntry) bool {
return strings.HasPrefix(entry.Name(), ".gguf-metadata-")
}) {
break
}
select {
case err := <-loaded:
t.Fatalf("GetModel completed before its metadata write could overlap deletion: %v", err)
default:
}
if time.Now().After(deadline) {
t.Fatal("timed out waiting for metadata write")
}
runtime.Gosched()
}
deleteModelNamed(t, "delete-during-load")
select {
case err := <-loaded:
if err != nil {
t.Fatalf("GetModel: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for GetModel")
}
path, err := ggufMetadataPath(digest)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("metadata published after model deletion: %v", err)
}
}
// The point of the metadata file is that loading a model no longer reads its
// blob. With the blob deleted, GetModel must still describe the model exactly
// as before - which is also the guard against anything reintroducing a
// per-request read of the model file.
func TestGetModelReadsNoBlob(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("OLLAMA_MODELS", t.TempDir())
_, digest := createBinFile(t, ggml.KV{
"general.architecture": "bert",
"bert.pooling_type": uint32(1),
"bert.context_length": uint32(512),
"bert.embedding_length": uint32(384),
}, nil)
createModelFromBlob(t, "embedder", digest, "")
want, err := GetModel("embedder")
if err != nil {
t.Fatal(err)
}
if !slices.Contains(want.Capabilities(), model.CapabilityEmbedding) {
t.Fatalf("capabilities = %v, want embedding", want.Capabilities())
}
if err := removeBlob(t, digest); err != nil {
t.Fatal(err)
}
got, err := GetModel("embedder")
if err != nil {
t.Fatalf("GetModel needed the blob: %v", err)
}
if !slices.Equal(got.Capabilities(), want.Capabilities()) {
t.Errorf("capabilities without the blob = %v, want %v", got.Capabilities(), want.Capabilities())
}
if got.metadata.Int("context_length") != 512 {
t.Errorf("context_length = %d, want 512", got.metadata.Int("context_length"))
}
}
// Extraction either fails or produces metadata that survives a metadata-file
// round trip unchanged.
func FuzzGGUFMetadata(f *testing.F) {
f.Add(ggufBytes(0, nil))
f.Add(ggufBytes(1<<20, nil))
var str []byte
str = binary.LittleEndian.AppendUint64(str, 20)
str = append(str, "general.architecture"...)
str = binary.LittleEndian.AppendUint32(str, 8) // type: string
str = binary.LittleEndian.AppendUint64(str, 5)
str = append(str, "llama"...)
f.Add(ggufBytes(1, str))
var arr []byte
arr = binary.LittleEndian.AppendUint64(arr, 3)
arr = append(arr, "big"...)
arr = binary.LittleEndian.AppendUint32(arr, 9) // type: array
arr = binary.LittleEndian.AppendUint32(arr, 8) // element type: string
arr = binary.LittleEndian.AppendUint64(arr, 8_000_000)
f.Add(ggufBytes(1, arr))
path := filepath.Join(f.TempDir(), "fuzz.gguf")
f.Fuzz(func(t *testing.T, data []byte) {
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
md, err := extractGGUFMetadata(path)
if err != nil {
return
}
if md.KV == nil {
t.Fatal("extraction reported success with no metadata")
}
encoded, err := json.Marshal(md)
if err != nil {
t.Fatalf("extracted metadata cannot be written: %v", err)
}
loaded, err := decodeGGUFMetadata(encoded)
if err != nil {
t.Fatalf("extracted metadata cannot be read back: %v", err)
}
if !reflect.DeepEqual(md, loaded) {
t.Errorf("metadata changed on the trip through a file:\n%#v\n%#v", md, loaded)
}
})
}

172
server/images.go
View file

@ -10,7 +10,6 @@ import (
"io"
"log"
"log/slog"
"math"
"net"
"net/http"
"net/url"
@ -24,7 +23,6 @@ import (
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/envconfig"
"github.com/ollama/ollama/fs/gguf"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/model/parsers"
"github.com/ollama/ollama/parser"
@ -83,8 +81,10 @@ type Model struct {
Template *template.Template
capabilities []model.Capability
capabilitiesCached bool
// Metadata of the model blob and of each projector, read from their
// metadata files when the model is loaded.
metadata ggufMetadata
projectorMetadata []ggufMetadata
}
func (m *Model) IsMLX() bool {
@ -95,47 +95,33 @@ func (m *Model) isGGUF() bool {
return m.Config.ModelFormat == "" || m.Config.ModelFormat == "gguf"
}
func generationDefaultsFromGGUF(f *gguf.File) model.GenerationDefaults {
func generationDefaultsFromMetadata(md ggufMetadata) model.GenerationDefaults {
return model.ParseGGUFGenerationDefaults(
func(key string) (int64, bool) {
return ggufIntGenerationDefault(f.KeyValue(key))
n, ok := md.number(key)
if !ok {
return 0, false
}
if value, err := n.Int64(); err == nil {
return value, true
}
value, err := n.Float64()
if err != nil {
return 0, false
}
return int64(value), true
},
func(key string) (float64, bool) {
return ggufFloatGenerationDefault(f.KeyValue(key))
n, ok := md.number(key)
if !ok {
return 0, false
}
value, err := n.Float64()
return value, err == nil
},
)
}
func ggufIntGenerationDefault(kv gguf.KeyValue) (int64, bool) {
if value, ok := kv.IntOK(); ok {
return value, true
}
if value, ok := kv.UintOK(); ok {
if value > math.MaxInt64 {
return 0, false
}
return int64(value), true
}
if value, ok := kv.FloatOK(); ok {
// Match api.Options.FromMap; rounding may be better for near-integers.
return int64(value), true
}
return 0, false
}
func ggufFloatGenerationDefault(kv gguf.KeyValue) (float64, bool) {
if value, ok := kv.FloatOK(); ok {
return value, true
}
if value, ok := kv.IntOK(); ok {
return float64(value), true
}
if value, ok := kv.UintOK(); ok {
return float64(value), true
}
return 0, false
}
func appendCapability(capabilities []model.Capability, capability model.Capability) []model.Capability {
if slices.Contains(capabilities, capability) {
return capabilities
@ -153,11 +139,7 @@ const (
// Capabilities returns the capabilities that the model supports
func (m *Model) Capabilities() []model.Capability {
if m.capabilitiesCached {
return slices.Clone(m.capabilities)
}
capabilities := m.capabilitiesForTemplate(templateCapabilitySelected, nil)
capabilities := m.capabilitiesForTemplate(templateCapabilitySelected)
if len(capabilities) == 0 {
slog.Warn("unknown capabilities for model", "model", m.Name)
}
@ -165,12 +147,12 @@ func (m *Model) Capabilities() []model.Capability {
return capabilities
}
func (m *Model) capabilitiesForTemplate(source templateCapabilitySource, f *gguf.File) []model.Capability {
func (m *Model) capabilitiesForTemplate(source templateCapabilitySource) []model.Capability {
capabilities := []model.Capability{}
var modelArch string
capabilities = m.configCapabilities(capabilities)
capabilities, modelArch = m.ggufCapabilities(capabilities, source, f)
capabilities, modelArch = m.ggufCapabilities(capabilities, source)
capabilities = m.projectorCapabilities(capabilities)
capabilities = m.templateCapabilities(capabilities, source)
capabilities = m.parserCapabilities(capabilities)
@ -187,44 +169,33 @@ func (m *Model) configCapabilities(capabilities []model.Capability) []model.Capa
return capabilities
}
func (m *Model) ggufCapabilities(capabilities []model.Capability, source templateCapabilitySource, f *gguf.File) ([]model.Capability, string) {
func (m *Model) ggufCapabilities(capabilities []model.Capability, source templateCapabilitySource) ([]model.Capability, string) {
if m.ModelPath == "" || !m.isGGUF() {
return capabilities, ""
}
if f == nil {
var err error
f, err = gguf.Open(m.ModelPath)
if err != nil {
slog.Error("couldn't open model file", "error", err)
return capabilities, ""
}
defer f.Close()
}
modelArch := f.KeyValue("general.architecture").String()
switch source {
case templateCapabilitySelected:
if !usesOllamaRenderedChat(m) {
capabilities = chatTemplateCapabilities(capabilities, f.KeyValue("tokenizer.chat_template").String())
capabilities = chatTemplateCapabilities(capabilities, m.metadata.String("tokenizer.chat_template"))
}
case templateCapabilityChat:
capabilities = chatTemplateCapabilities(capabilities, f.KeyValue("tokenizer.chat_template").String())
capabilities = chatTemplateCapabilities(capabilities, m.metadata.String("tokenizer.chat_template"))
}
if f.KeyValue("pooling_type").Valid() {
if m.metadata.Valid("pooling_type") {
capabilities = appendCapability(capabilities, model.CapabilityEmbedding)
} else {
// If no embedding is specified, we assume the model supports completion.
capabilities = appendCapability(capabilities, model.CapabilityCompletion)
}
if f.KeyValue("vision.block_count").Valid() {
if m.metadata.Valid("vision.block_count") {
capabilities = appendCapability(capabilities, model.CapabilityVision)
}
if f.KeyValue("audio.block_count").Valid() {
if m.metadata.Valid("audio.block_count") {
capabilities = appendCapability(capabilities, model.CapabilityAudio)
}
return capabilities, modelArch
return capabilities, m.metadata.String("general.architecture")
}
func chatTemplateCapabilities(capabilities []model.Capability, chatTemplate string) []model.Capability {
@ -390,28 +361,17 @@ func capabilityLogValue(present bool, capabilities []model.Capability) any {
}
func (m *Model) templateSelectionCapabilities(usesHarmony bool) (goTemplate, chatTemplate, harmony, rendererParser []model.Capability) {
var f *gguf.File
if m.ModelPath != "" && m.isGGUF() {
var err error
f, err = gguf.Open(m.ModelPath)
if err != nil {
slog.Error("couldn't open model file", "error", err)
} else {
defer f.Close()
}
}
if m.HasGoTemplate {
goTemplate = m.capabilitiesForTemplate(templateCapabilityGo, f)
goTemplate = m.capabilitiesForTemplate(templateCapabilityGo)
}
if m.HasChatTemplate {
chatTemplate = m.capabilitiesForTemplate(templateCapabilityChat, f)
chatTemplate = m.capabilitiesForTemplate(templateCapabilityChat)
}
if usesHarmony {
harmony = m.capabilitiesForTemplate(templateCapabilitySelected, f)
harmony = m.capabilitiesForTemplate(templateCapabilitySelected)
}
if m.Config.Renderer != "" || m.Config.Parser != "" {
rendererParser = m.capabilitiesForTemplate(templateCapabilitySelected, f)
rendererParser = m.capabilitiesForTemplate(templateCapabilitySelected)
}
return goTemplate, chatTemplate, harmony, rendererParser
@ -439,16 +399,10 @@ func (m *Model) projectorCapabilities(capabilities []model.Capability) []model.C
}
capabilities = appendCapability(capabilities, model.CapabilityVision)
for _, projectorPath := range m.ProjectorPaths {
f, err := gguf.Open(projectorPath)
if err != nil {
slog.Error("couldn't open projector file", "error", err)
continue
}
if projectorHasAudio(f) && !projectorSuppressesAudioCapability(f) {
for _, md := range m.projectorMetadata {
if projectorHasAudio(md) && !projectorSuppressesAudioCapability(md) {
capabilities = appendCapability(capabilities, model.CapabilityAudio)
}
f.Close()
}
return capabilities
@ -551,22 +505,21 @@ func isNemotron3NanoSafetensorsConfig(cfg model.ConfigV2) bool {
slices.Contains(cfg.ModelFamilies, "nemotron_h_omni"))
}
func projectorHasAudio(f *gguf.File) bool {
if f.KeyValue("has_audio_encoder").Bool() {
return true
}
for _, kv := range f.KeyValues() {
if strings.HasSuffix(kv.Key, ".has_audio_encoder") && kv.Bool() {
return true
func projectorHasAudio(md ggufMetadata) bool {
// read directly: Keys reports qualified keys, the accessors qualify theirs
for _, key := range md.Keys() {
if key == "has_audio_encoder" || strings.HasSuffix(key, ".has_audio_encoder") {
if b, ok := md.KV[key].(bool); ok && b {
return true
}
}
}
return false
}
func projectorSuppressesAudioCapability(f *gguf.File) bool {
switch f.KeyValue("vision.projector_type").String() {
func projectorSuppressesAudioCapability(md ggufMetadata) bool {
switch md.String("vision.projector_type") {
case "gemma3nv":
return true
}
@ -745,6 +698,13 @@ func GetModel(name string) (*Model, error) {
modelHasPooling := false
ggufChatTemplate := ""
for _, layer := range mf.Layers {
// Nothing below reads a tensor layer, and resolving a path costs a
// syscall each. Named rather than allowlisting the types below, so a new
// layer type is slower here instead of silently unread.
if layer.MediaType == manifest.MediaTypeImageTensor {
continue
}
filename, err := manifest.BlobsPath(layer.Digest)
if err != nil {
return nil, err
@ -755,16 +715,16 @@ func GetModel(name string) (*Model, error) {
m.ModelPath = filename
m.ParentModel = layer.From
if m.isGGUF() {
f, err := gguf.Open(filename)
md, err := readGGUFMetadata(layer.Digest)
if err != nil {
slog.Error("couldn't open model file", "error", err)
slog.Error("couldn't read model metadata", "error", err)
break
}
ggufChatTemplate = f.KeyValue("tokenizer.chat_template").String()
m.metadata = md
ggufChatTemplate = md.String("tokenizer.chat_template")
m.HasChatTemplate = ggufChatTemplate != ""
modelHasPooling = f.KeyValue("pooling_type").Valid()
m.GenerationDefaults = generationDefaultsFromGGUF(f)
f.Close()
modelHasPooling = md.Valid("pooling_type")
m.GenerationDefaults = generationDefaultsFromMetadata(md)
}
case manifest.MediaTypeImageDraft:
m.DraftPath = filename
@ -776,6 +736,11 @@ func GetModel(name string) (*Model, error) {
m.AdapterPaths = append(m.AdapterPaths, filename)
case "application/vnd.ollama.image.projector":
m.ProjectorPaths = append(m.ProjectorPaths, filename)
if md, err := readGGUFMetadata(layer.Digest); err != nil {
slog.Error("couldn't read projector metadata", "error", err)
} else {
m.projectorMetadata = append(m.projectorMetadata, md)
}
case "application/vnd.ollama.image.prompt",
"application/vnd.ollama.image.template":
m.HasGoTemplate = true
@ -900,10 +865,11 @@ func deleteUnusedLayers(deleteMap map[string]struct{}) error {
slog.Info(fmt.Sprintf("couldn't get file path for '%s': %v", k, err))
continue
}
if err := os.Remove(fp); err != nil {
if err := os.Remove(fp); err != nil && !errors.Is(err, os.ErrNotExist) {
slog.Info(fmt.Sprintf("couldn't remove file '%s': %v", fp, err))
continue
}
removeGGUFMetadata(k)
}
return nil
@ -960,6 +926,7 @@ func PruneLayers() error {
slog.Error(fmt.Sprintf("couldn't remove unused layers: %v", err))
return nil
}
pruneGGUFMetadata()
slog.Info(fmt.Sprintf("total unused blobs removed: %d", len(deleteMap)))
@ -1122,6 +1089,7 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
if err := os.Remove(fp); err != nil {
slog.Info(fmt.Sprintf("couldn't remove file with digest mismatch '%s': %v", fp, err))
}
removeGGUFMetadata(layer.Digest)
}
return err
}

38
server/images_test.go
View file

@ -14,7 +14,6 @@ import (
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/fs/ggml"
fsgguf "github.com/ollama/ollama/fs/gguf"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/template"
"github.com/ollama/ollama/types/model"
@ -61,7 +60,7 @@ func TestPruneLayersSkipsRecentOrphans(t *testing.T) {
}
}
func TestGenerationDefaultsFromGGUF(t *testing.T) {
func TestGenerationDefaultsFromMetadata(t *testing.T) {
file, err := os.CreateTemp(t.TempDir(), "model-*.gguf")
if err != nil {
t.Fatal(err)
@ -87,13 +86,12 @@ func TestGenerationDefaultsFromGGUF(t *testing.T) {
t.Fatal(err)
}
f, err := fsgguf.Open(file.Name())
md, err := extractGGUFMetadata(file.Name())
if err != nil {
t.Fatal(err)
}
defer f.Close()
defaults := generationDefaultsFromGGUF(f)
defaults := generationDefaultsFromMetadata(md)
check := func(key string, want any) {
t.Helper()
if got := defaults[key]; got != want {
@ -104,10 +102,10 @@ func TestGenerationDefaultsFromGGUF(t *testing.T) {
check("top_k", int64(40))
check("top_p", float64(1))
check("min_p", float64(0))
check("typical_p", float64(float32(0.95)))
check("typical_p", float64(0.95))
check("temperature", float64(1))
check("repeat_last_n", int64(64))
check("repeat_penalty", float64(float32(1.05)))
check("repeat_penalty", float64(1.05))
check("frequency_penalty", float64(0))
check("presence_penalty", float64(0))
if _, ok := defaults["mirostat_tau"]; ok {
@ -354,6 +352,28 @@ func writeTestModelManifest(t *testing.T, name, digest, tmpl string) {
}
}
// loadTestMetadata fills in what GetModel would have read from the metadata
// files, so
// hand-built models resolve capabilities the same way loaded ones do.
func loadTestMetadata(t *testing.T, m *Model) {
t.Helper()
if m.ModelPath != "" {
md, err := extractGGUFMetadata(m.ModelPath)
if err != nil {
t.Fatalf("metadata for %s: %v", m.ModelPath, err)
}
m.metadata = md
}
m.projectorMetadata = nil
for _, path := range m.ProjectorPaths {
md, err := extractGGUFMetadata(path)
if err != nil {
t.Fatalf("projector metadata for %s: %v", path, err)
}
m.projectorMetadata = append(m.projectorMetadata, md)
}
}
func TestModelCapabilities(t *testing.T) {
// Create completion model (llama architecture without vision)
completionModelPath, _ := createBinFile(t, ggml.KV{
@ -618,6 +638,8 @@ func TestModelCapabilities(t *testing.T) {
for _, tt := range testModels {
t.Run(tt.name, func(t *testing.T) {
loadTestMetadata(t, &tt.model)
// Test Capabilities method
caps := tt.model.Capabilities()
if !compareCapabilities(caps, tt.expectedCaps) {
@ -748,6 +770,8 @@ func TestModelCheckCapabilities(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
loadTestMetadata(t, &tt.model)
// Test CheckCapabilities method
err := tt.model.CheckCapabilities(tt.checkCaps...)
if tt.expectedErrMsg == "" {

7
server/model_caches.go
View file

@ -5,16 +5,12 @@ import "context"
type modelCaches struct {
recommendations *modelRecommendationsCache
show *modelShowCache
modelList *modelListCache
inference *inferenceModelCache
}
func newModelCaches() *modelCaches {
return &modelCaches{
recommendations: newModelRecommendationsCache(),
show: newModelShowCache(),
modelList: newModelListCache(),
inference: newInferenceModelCache(),
}
}
@ -28,7 +24,4 @@ func (c *modelCaches) Start(ctx context.Context) {
if c.show != nil {
c.show.Start(ctx)
}
if c.modelList != nil {
c.modelList.Start(ctx)
}
}

121
server/model_inference_cache.go
View file

@ -1,121 +0,0 @@
package server
import (
"maps"
"slices"
"strconv"
"sync"
"github.com/ollama/ollama/envconfig"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/types/model"
"golang.org/x/sync/singleflight"
)
// inferenceModelCache stores fully resolved model metadata and capabilities.
// Model blobs are content-addressed, and the manifest digest is the freshness
// boundary, so cached entries remain valid until the model is recreated or
// pulled with different content.
type inferenceModelCache struct {
mu sync.RWMutex
entries map[inferenceModelCacheKey]inferenceModelCacheEntry
loads singleflight.Group
loadModel func(string) (*Model, error)
}
type inferenceModelCacheKey struct {
name string
goTemplate bool
goTemplateSet bool
}
type inferenceModelCacheEntry struct {
digest string
model *Model
}
func newInferenceModelCache() *inferenceModelCache {
return &inferenceModelCache{
entries: make(map[inferenceModelCacheKey]inferenceModelCacheEntry),
loadModel: GetModel,
}
}
func (c *inferenceModelCache) Get(name string) (*Model, error) {
n := model.ParseName(name)
mf, err := manifest.ParseNamedManifest(n)
if err != nil {
return nil, err
}
key := inferenceModelCacheKey{
name: n.String(),
goTemplate: envconfig.GoTemplate(true),
goTemplateSet: goTemplateEnvSet(),
}
digest := mf.Digest()
c.mu.RLock()
entry, ok := c.entries[key]
c.mu.RUnlock()
if ok && entry.digest == digest {
return cloneInferenceModel(entry.model), nil
}
loadKey := key.name + "\x00" + digest + "\x00" + strconv.FormatBool(key.goTemplate) + "\x00" + strconv.FormatBool(key.goTemplateSet)
v, err, _ := c.loads.Do(loadKey, func() (any, error) {
c.mu.RLock()
entry, ok := c.entries[key]
c.mu.RUnlock()
if ok && entry.digest == digest {
return entry.model, nil
}
m, err := c.loadModel(name)
if err != nil {
return nil, err
}
m.capabilities = m.Capabilities()
m.capabilitiesCached = true
c.mu.Lock()
c.entries[key] = inferenceModelCacheEntry{digest: m.Digest, model: m}
c.mu.Unlock()
return m, nil
})
if err != nil {
return nil, err
}
return cloneInferenceModel(v.(*Model)), nil
}
func cloneInferenceModel(src *Model) *Model {
if src == nil {
return nil
}
dst := *src
dst.Config.ModelFamilies = slices.Clone(src.Config.ModelFamilies)
dst.Config.Capabilities = slices.Clone(src.Config.Capabilities)
if src.Config.Draft != nil {
draft := *src.Config.Draft
dst.Config.Draft = &draft
}
dst.AdapterPaths = slices.Clone(src.AdapterPaths)
dst.ProjectorPaths = slices.Clone(src.ProjectorPaths)
dst.License = slices.Clone(src.License)
dst.Options = maps.Clone(src.Options)
dst.Messages = slices.Clone(src.Messages)
dst.capabilities = slices.Clone(src.capabilities)
return &dst
}
func (s *Server) getModel(name string) (*Model, error) {
if s != nil && s.modelCaches != nil && s.modelCaches.inference != nil {
return s.modelCaches.inference.Get(name)
}
return GetModel(name)
}

121
server/model_inference_cache_test.go
View file

@ -1,121 +0,0 @@
package server
import (
"slices"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/ollama/ollama/fs/ggml"
"github.com/ollama/ollama/types/model"
)
func TestInferenceModelCache(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
t.Setenv("OLLAMA_GO_TEMPLATE", "")
_, completionDigest := createBinFile(t, ggml.KV{
"general.architecture": "llama",
}, nil)
writeTestModelManifest(t, "inference-cache", completionDigest, "{{ .Prompt }}")
cache := newInferenceModelCache()
loadCount := 0
cache.loadModel = func(name string) (*Model, error) {
loadCount++
return GetModel(name)
}
first, err := cache.Get("inference-cache")
if err != nil {
t.Fatal(err)
}
if loadCount != 1 {
t.Fatalf("load count = %d, want 1", loadCount)
}
if !first.capabilitiesCached {
t.Fatal("capabilities were not cached")
}
if got := first.Capabilities(); !slices.Contains(got, model.CapabilityCompletion) {
t.Fatalf("capabilities = %v, want completion", got)
}
// Returned models are request-local clones. Mutating one must not alter the
// cached model or a later request.
first.Config.Parser = "mutated"
first.Config.ModelFamilies[0] = "mutated"
first.capabilities[0] = model.CapabilityImage
second, err := cache.Get("inference-cache")
if err != nil {
t.Fatal(err)
}
if loadCount != 1 {
t.Fatalf("cache hit load count = %d, want 1", loadCount)
}
if second.Config.Parser == "mutated" || slices.Contains(second.Config.ModelFamilies, "mutated") {
t.Fatalf("cached model was mutated: %#v", second.Config)
}
if got := second.Capabilities(); !slices.Contains(got, model.CapabilityCompletion) || slices.Contains(got, model.CapabilityImage) {
t.Fatalf("cached capabilities = %v, want completion only", got)
}
// Recreating the manifest changes its digest and invalidates the entry.
_, embeddingDigest := createBinFile(t, ggml.KV{
"general.architecture": "bert",
"bert.pooling_type": uint32(1),
}, nil)
writeTestModelManifest(t, "inference-cache", embeddingDigest, "{{ .Prompt }}")
third, err := cache.Get("inference-cache")
if err != nil {
t.Fatal(err)
}
if loadCount != 2 {
t.Fatalf("invalidated load count = %d, want 2", loadCount)
}
if got := third.Capabilities(); !slices.Contains(got, model.CapabilityEmbedding) || slices.Contains(got, model.CapabilityCompletion) {
t.Fatalf("refreshed capabilities = %v, want embedding only", got)
}
}
func TestInferenceModelCacheConcurrentMiss(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
t.Setenv("OLLAMA_GO_TEMPLATE", "")
_, digest := createBinFile(t, ggml.KV{
"general.architecture": "llama",
}, nil)
writeTestModelManifest(t, "inference-cache-concurrent", digest, "{{ .Prompt }}")
cache := newInferenceModelCache()
var loadCount atomic.Int32
cache.loadModel = func(name string) (*Model, error) {
loadCount.Add(1)
time.Sleep(10 * time.Millisecond)
return GetModel(name)
}
var wg sync.WaitGroup
errs := make(chan error, 8)
for range 8 {
wg.Add(1)
go func() {
defer wg.Done()
_, err := cache.Get("inference-cache-concurrent")
errs <- err
}()
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
if got := loadCount.Load(); got != 1 {
t.Fatalf("load count = %d, want 1", got)
}
}

133
server/model_list.go Normal file
View file

@ -0,0 +1,133 @@
package server
import (
"cmp"
"context"
"encoding/json"
"log/slog"
"slices"
"time"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/fs/ggml"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/types/model"
)
// listModels builds /api/tags from the manifests and the per-blob metadata
// files, extracting for any blob that has none yet.
func listModels(ctx context.Context) ([]api.ListModelResponse, error) {
manifests, err := manifest.Manifests(true)
if err != nil {
return nil, err
}
models := make([]api.ListModelResponse, 0, len(manifests))
for name, mf := range manifests {
if ctx != nil {
if err := ctx.Err(); err != nil {
return nil, err
}
}
summary, err := describeModel(name, mf)
if err != nil {
slog.Warn("failed to describe model", "model", name.String(), "error", err)
continue
}
models = append(models, summary)
}
sortListModelResponses(models)
return models, nil
}
// describeModel describes one model for /api/tags. Capabilities come from the
// same Model.Capabilities() the inference path uses, so the two cannot drift.
func describeModel(name model.Name, mf *manifest.Manifest) (api.ListModelResponse, error) {
cfg, err := readModelListConfig(mf)
if err != nil {
return api.ListModelResponse{}, err
}
var modified time.Time
if fi := mf.FileInfo(); fi != nil {
modified = fi.ModTime()
}
summary := api.ListModelResponse{
Model: name.DisplayShortest(),
Name: name.DisplayShortest(),
RemoteModel: cfg.RemoteModel,
RemoteHost: cfg.RemoteHost,
Size: mf.Size(),
Digest: mf.Digest(),
ModifiedAt: modified,
Details: api.ModelDetails{
Format: cfg.ModelFormat,
Family: cfg.ModelFamily,
Families: append([]string(nil), cfg.ModelFamilies...),
ParameterSize: cfg.ModelType,
QuantizationLevel: cfg.FileType,
ContextLength: cfg.ContextLen,
EmbeddingLength: cfg.EmbedLen,
},
}
m, err := GetModel(name.String())
if err != nil {
// A model that will not load is the one a user most needs to see, in
// order to remove it. Report what the manifest says.
slog.Warn("could not load model to describe it", "model", name.String(), "error", err)
return summary, nil
}
summary.Details.ParentModel = m.ParentModel
summary.Capabilities = m.Capabilities()
if m.ModelPath != "" && m.isGGUF() {
if summary.Details.ContextLength == 0 {
summary.Details.ContextLength = int(m.metadata.Int("context_length"))
}
if summary.Details.EmbeddingLength == 0 {
summary.Details.EmbeddingLength = int(m.metadata.Int("embedding_length"))
}
if m.metadata.Valid("general.file_type") {
fileType := ggml.FileType(m.metadata.Int("general.file_type")).String()
if isUnknownQuantization(summary.Details.QuantizationLevel) && !isUnknownQuantization(fileType) {
summary.Details.QuantizationLevel = fileType
}
}
}
return summary, nil
}
func readModelListConfig(mf *manifest.Manifest) (model.ConfigV2, error) {
var cfg model.ConfigV2
if mf == nil || mf.Config.Digest == "" {
return cfg, nil
}
f, err := mf.Config.Open()
if err != nil {
return cfg, err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&cfg); err != nil {
return cfg, err
}
return cfg, nil
}
func isUnknownQuantization(quantization string) bool {
return quantization == "" || quantization == "unknown"
}
func sortListModelResponses(models []api.ListModelResponse) {
slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
// Preserve the existing /api/tags order: most recently modified first.
return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
})
}

888
server/model_list_cache.go
View file

@ -1,888 +0,0 @@
package server
import (
"bufio"
"cmp"
"context"
"encoding/binary"
"encoding/json"
"fmt"
"io"
"log/slog"
"os"
"slices"
"strings"
"sync"
"time"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/fs/ggml"
fsgguf "github.com/ollama/ollama/fs/gguf"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/model/parsers"
ollamatemplate "github.com/ollama/ollama/template"
"github.com/ollama/ollama/thinking"
"github.com/ollama/ollama/types/model"
)
type modelListSummary struct {
Model string
Name string
RemoteModel string
RemoteHost string
Size int64
Digest string
ModifiedAt time.Time
Details api.ModelDetails
Capabilities []model.Capability
}
type modelListCacheEntry struct {
Digest string
Summary modelListSummary
}
type modelListCache struct {
mu sync.RWMutex
entries map[string]modelListCacheEntry
once sync.Once
readyOnce sync.Once
ready chan struct{}
hydrateErr error
build func(model.Name, *manifest.Manifest) (modelListSummary, error)
}
func newModelListCache() *modelListCache {
return &modelListCache{
entries: make(map[string]modelListCacheEntry),
ready: make(chan struct{}),
build: buildModelListSummary,
}
}
func (c *modelListCache) Start(ctx context.Context) {
if c == nil {
return
}
c.once.Do(func() {
slog.Debug("starting model list cache")
go func() {
err := c.hydrate(ctx)
c.markReady(err)
if err != nil {
if ctx != nil && ctx.Err() != nil {
return
}
slog.Warn("model list cache hydration failed", "error", err)
}
}()
})
}
func (c *modelListCache) hydrate(ctx context.Context) error {
start := time.Now()
manifests, err := manifest.Manifests(true)
if err != nil {
return err
}
var hydrated, failed int
for name, mf := range manifests {
if ctx != nil {
if err := ctx.Err(); err != nil {
return err
}
}
summary, err := c.build(name, mf)
if err != nil {
failed++
slog.Warn("failed to hydrate model list cache", "model", name.String(), "error", err)
continue
}
c.set(name, mf.Digest(), summary)
hydrated++
}
slog.Info("model list cache hydration complete", "models", hydrated, "failures", failed, "elapsed", time.Since(start))
return nil
}
func (c *modelListCache) markReady(err error) {
c.mu.Lock()
c.hydrateErr = err
c.mu.Unlock()
c.readyOnce.Do(func() {
close(c.ready)
})
}
func (c *modelListCache) Wait(ctx context.Context) error {
if c == nil {
return nil
}
if ctx == nil {
ctx = context.Background()
}
select {
case <-c.ready:
c.mu.RLock()
err := c.hydrateErr
c.mu.RUnlock()
return err
case <-ctx.Done():
return ctx.Err()
}
}
func (c *modelListCache) List(ctx context.Context) ([]api.ListModelResponse, error) {
if err := c.Wait(ctx); err != nil {
return nil, err
}
if err := c.syncManifests(ctx); err != nil {
return nil, err
}
c.mu.RLock()
models := make([]api.ListModelResponse, 0, len(c.entries))
for _, entry := range c.entries {
models = append(models, entry.Summary.ListModelResponse())
}
c.mu.RUnlock()
sortListModelResponses(models)
return models, nil
}
func (c *modelListCache) syncManifests(ctx context.Context) error {
manifests, err := manifest.Manifests(true)
if err != nil {
return err
}
c.mu.RLock()
current := make(map[string]string, len(c.entries))
for name, entry := range c.entries {
current[name] = entry.Digest
}
c.mu.RUnlock()
type update struct {
name model.Name
digest string
summary modelListSummary
}
seen := make(map[string]struct{}, len(manifests))
stale := make(map[string]struct{})
var updates []update
for name, mf := range manifests {
if ctx != nil {
if err := ctx.Err(); err != nil {
return err
}
}
key := name.String()
digest := mf.Digest()
seen[key] = struct{}{}
if current[key] == digest {
continue
}
summary, err := c.build(name, mf)
if err != nil {
slog.Warn("failed to refresh model list cache", "model", key, "error", err)
if _, ok := current[key]; ok {
stale[key] = struct{}{}
}
continue
}
updates = append(updates, update{name: name, digest: digest, summary: summary})
}
c.mu.Lock()
for name := range c.entries {
if _, ok := seen[name]; !ok {
delete(c.entries, name)
continue
}
if _, ok := stale[name]; ok {
delete(c.entries, name)
}
}
for _, update := range updates {
c.entries[update.name.String()] = modelListCacheEntry{
Digest: update.digest,
Summary: cloneModelListSummary(update.summary),
}
}
c.mu.Unlock()
return nil
}
func (c *modelListCache) RefreshModel(name model.Name) error {
if c == nil {
return nil
}
if !name.IsFullyQualified() {
var err error
name, err = getExistingName(name)
if err != nil {
return err
}
}
mf, err := manifest.ParseNamedManifest(name)
if err != nil {
c.DeleteModel(name)
return err
}
summary, err := c.build(name, mf)
if err != nil {
c.DeleteModel(name)
return err
}
c.set(name, mf.Digest(), summary)
return nil
}
func (c *modelListCache) DeleteModel(name model.Name) {
if c == nil {
return
}
c.mu.Lock()
delete(c.entries, name.String())
c.mu.Unlock()
}
func (c *modelListCache) Get(name model.Name) (modelListSummary, bool) {
if c == nil {
return modelListSummary{}, false
}
if !name.IsFullyQualified() {
if existing, err := getExistingName(name); err == nil {
name = existing
}
}
c.mu.RLock()
entry, ok := c.entries[name.String()]
c.mu.RUnlock()
if !ok {
return modelListSummary{}, false
}
return cloneModelListSummary(entry.Summary), true
}
func (c *modelListCache) Len() int {
if c == nil {
return 0
}
c.mu.RLock()
defer c.mu.RUnlock()
return len(c.entries)
}
func (c *modelListCache) set(name model.Name, digest string, summary modelListSummary) {
c.mu.Lock()
c.entries[name.String()] = modelListCacheEntry{
Digest: digest,
Summary: cloneModelListSummary(summary),
}
c.mu.Unlock()
}
func buildModelListSummary(name model.Name, mf *manifest.Manifest) (modelListSummary, error) {
cfg, err := readModelListConfig(mf)
if err != nil {
return modelListSummary{}, err
}
var modified time.Time
if fi := mf.FileInfo(); fi != nil {
modified = fi.ModTime()
}
summary := modelListSummary{
Model: name.DisplayShortest(),
Name: name.DisplayShortest(),
RemoteModel: cfg.RemoteModel,
RemoteHost: cfg.RemoteHost,
Size: mf.Size(),
Digest: mf.Digest(),
ModifiedAt: modified,
Details: api.ModelDetails{
Format: cfg.ModelFormat,
Family: cfg.ModelFamily,
Families: append([]string(nil), cfg.ModelFamilies...),
ParameterSize: cfg.ModelType,
QuantizationLevel: cfg.FileType,
ContextLength: cfg.ContextLen,
EmbeddingLength: cfg.EmbedLen,
},
}
modelPath, projectorCount, tmpl, err := readModelListLayers(mf, &summary)
if err != nil {
return modelListSummary{}, err
}
if cfg.RemoteHost == "" && cfg.RemoteModel == "" && modelPath != "" {
info, err := readModelListGGUF(modelPath)
if err != nil {
slog.Debug("failed to read gguf model metadata", "model", name.String(), "error", err)
} else {
summary.Capabilities = appendModelListCapabilities(summary.Capabilities, info.Capabilities...)
if summary.Details.ContextLength == 0 {
summary.Details.ContextLength = info.ContextLength
}
if summary.Details.EmbeddingLength == 0 {
summary.Details.EmbeddingLength = info.EmbeddingLength
}
if isUnknownQuantization(summary.Details.QuantizationLevel) && !isUnknownQuantization(info.FileType) {
summary.Details.QuantizationLevel = info.FileType
}
}
}
for _, c := range cfg.Capabilities {
summary.Capabilities = appendModelListCapability(summary.Capabilities, model.Capability(c))
}
builtinParser := parsers.ParserForName(cfg.Parser)
if tmpl != nil {
vars, err := tmpl.Vars()
if err != nil {
slog.Warn("model template contains errors", "model", name.String(), "error", err)
}
if slices.Contains(vars, "tools") || (builtinParser != nil && builtinParser.HasToolSupport()) {
summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityTools)
}
if slices.Contains(vars, "suffix") {
summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityInsert)
}
openingTag, closingTag := thinking.InferTags(tmpl.Template)
hasTags := openingTag != "" && closingTag != ""
isGptoss := slices.Contains([]string{"gptoss", "gpt-oss"}, cfg.ModelFamily)
if !slices.Contains(summary.Capabilities, model.CapabilityThinking) &&
(hasTags || isGptoss || (builtinParser != nil && builtinParser.HasThinkingSupport())) {
summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityThinking)
}
}
if projectorCount > 0 {
summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityVision)
}
summary.Capabilities = filterUnsupportedModelListCapabilities(summary.Capabilities, cfg)
return summary, nil
}
func filterUnsupportedModelListCapabilities(capabilities []model.Capability, cfg model.ConfigV2) []model.Capability {
if cfg.ModelFormat == "safetensors" && isNemotron3NanoSafetensorsConfig(cfg) {
capabilities = slices.DeleteFunc(capabilities, func(c model.Capability) bool {
return c == model.CapabilityVision || c == model.CapabilityAudio
})
}
// Mirrors suppressAudioCapability in images.go so /api/tags and /api/show
// agree for safetensors models whose MLX runner serves vision but not audio.
if cfg.ModelFormat == "safetensors" && cfg.Renderer == "glimmer" {
capabilities = slices.DeleteFunc(capabilities, func(c model.Capability) bool {
return c == model.CapabilityAudio
})
}
return capabilities
}
func readModelListConfig(mf *manifest.Manifest) (model.ConfigV2, error) {
var cfg model.ConfigV2
if mf == nil || mf.Config.Digest == "" {
return cfg, nil
}
f, err := mf.Config.Open()
if err != nil {
return cfg, err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&cfg); err != nil {
return cfg, err
}
return cfg, nil
}
func readModelListLayers(mf *manifest.Manifest, summary *modelListSummary) (string, int, *ollamatemplate.Template, error) {
var modelPath string
var projectorCount int
tmpl := ollamatemplate.DefaultTemplate
for _, layer := range mf.Layers {
switch layer.MediaType {
case "application/vnd.ollama.image.model":
filename, err := manifest.BlobsPath(layer.Digest)
if err != nil {
return "", 0, nil, err
}
modelPath = filename
summary.Details.ParentModel = layer.From
case "application/vnd.ollama.image.projector":
projectorCount++
case "application/vnd.ollama.image.prompt",
"application/vnd.ollama.image.template":
filename, err := manifest.BlobsPath(layer.Digest)
if err != nil {
return "", 0, nil, err
}
bts, err := os.ReadFile(filename)
if err != nil {
return "", 0, nil, err
}
tmpl, err = ollamatemplate.Parse(string(bts))
if err != nil {
return "", 0, nil, err
}
}
}
return modelPath, projectorCount, tmpl, nil
}
type modelListGGUF struct {
Capabilities []model.Capability
ContextLength int
EmbeddingLength int
FileType string
}
const (
modelListGGUFMagicLE = 0x46554747
modelListGGUFMagicBE = 0x47475546
)
const (
modelListGGUFTypeUint8 uint32 = iota
modelListGGUFTypeInt8
modelListGGUFTypeUint16
modelListGGUFTypeInt16
modelListGGUFTypeUint32
modelListGGUFTypeInt32
modelListGGUFTypeFloat32
modelListGGUFTypeBool
modelListGGUFTypeString
modelListGGUFTypeArray
modelListGGUFTypeUint64
modelListGGUFTypeInt64
modelListGGUFTypeFloat64
)
// readModelListGGUF scans only the small GGUF header values launch needs
// and stops before tokenizer arrays. Using gguf.File.KeyValue for missing keys
// can otherwise advance through large arrays just to discover absence.
func readModelListGGUF(path string) (modelListGGUF, error) {
f, err := os.Open(path)
if err != nil {
return modelListGGUF{}, err
}
defer f.Close()
r := bufio.NewReaderSize(f, 32<<10)
var magic uint32
if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
return modelListGGUF{}, err
}
var byteOrder binary.ByteOrder = binary.LittleEndian
switch magic {
case modelListGGUFMagicLE:
case modelListGGUFMagicBE:
byteOrder = binary.BigEndian
default:
return modelListGGUF{}, fmt.Errorf("invalid file magic")
}
var version uint32
if err := binary.Read(r, byteOrder, &version); err != nil {
return modelListGGUF{}, err
}
var numKV uint64
switch version {
case 1:
var header struct {
NumTensor uint32
NumKV uint32
}
if err := binary.Read(r, byteOrder, &header); err != nil {
return modelListGGUF{}, err
}
numKV = uint64(header.NumKV)
default:
var header struct {
NumTensor uint64
NumKV uint64
}
if err := binary.Read(r, byteOrder, &header); err != nil {
return modelListGGUF{}, err
}
numKV = header.NumKV
}
info := modelListGGUF{}
var architecture string
var hasPoolingType bool
for range numKV {
key, err := readModelListGGUFString(r, byteOrder, version)
if err != nil {
return modelListGGUF{}, err
}
var valueType uint32
if err := binary.Read(r, byteOrder, &valueType); err != nil {
return modelListGGUF{}, err
}
if key == "general.architecture" {
value, err := readModelListGGUFStringValue(r, byteOrder, version, valueType)
if err != nil {
return modelListGGUF{}, err
}
architecture = value
continue
}
if key == "general.file_type" {
value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)
if err != nil {
return modelListGGUF{}, err
}
info.FileType = ggml.FileType(value).String()
continue
}
if architecture != "" && strings.HasPrefix(key, "tokenizer.") {
break
}
if architecture != "" && strings.HasPrefix(key, architecture+".") {
switch strings.TrimPrefix(key, architecture+".") {
case "pooling_type":
hasPoolingType = true
case "vision.block_count":
info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityVision)
case "audio.block_count":
info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityAudio)
case "context_length":
value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)
if err != nil {
return modelListGGUF{}, err
}
info.ContextLength = value
continue
case "embedding_length":
value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)
if err != nil {
return modelListGGUF{}, err
}
info.EmbeddingLength = value
continue
}
}
if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
return modelListGGUF{}, err
}
}
if hasPoolingType {
info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityEmbedding)
} else {
info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityCompletion)
}
return info, nil
}
func readModelListGGUFStringValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (string, error) {
if valueType != modelListGGUFTypeString {
if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
return "", err
}
return "", fmt.Errorf("unexpected gguf string type %d", valueType)
}
return readModelListGGUFString(r, byteOrder, version)
}
func readModelListGGUFIntValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (int, error) {
switch valueType {
case modelListGGUFTypeUint8:
var value uint8
if err := binary.Read(r, byteOrder, &value); err != nil {
return 0, err
}
return int(value), nil
case modelListGGUFTypeInt8:
var value int8
if err := binary.Read(r, byteOrder, &value); err != nil {
return 0, err
}
return int(value), nil
case modelListGGUFTypeUint16:
var value uint16
if err := binary.Read(r, byteOrder, &value); err != nil {
return 0, err
}
return int(value), nil
case modelListGGUFTypeInt16:
var value int16
if err := binary.Read(r, byteOrder, &value); err != nil {
return 0, err
}
return int(value), nil
case modelListGGUFTypeUint32:
var value uint32
if err := binary.Read(r, byteOrder, &value); err != nil {
return 0, err
}
return int(value), nil
case modelListGGUFTypeInt32:
var value int32
if err := binary.Read(r, byteOrder, &value); err != nil {
return 0, err
}
return int(value), nil
case modelListGGUFTypeUint64:
var value uint64
if err := binary.Read(r, byteOrder, &value); err != nil {
return 0, err
}
return int(value), nil
case modelListGGUFTypeInt64:
var value int64
if err := binary.Read(r, byteOrder, &value); err != nil {
return 0, err
}
return int(value), nil
default:
if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
return 0, err
}
return 0, fmt.Errorf("unexpected gguf integer type %d", valueType)
}
}
func skipModelListGGUFValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) error {
switch valueType {
case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool:
return discardModelListGGUFBytes(r, 1)
case modelListGGUFTypeUint16, modelListGGUFTypeInt16:
return discardModelListGGUFBytes(r, 2)
case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32:
return discardModelListGGUFBytes(r, 4)
case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64:
return discardModelListGGUFBytes(r, 8)
case modelListGGUFTypeString:
return skipModelListGGUFString(r, byteOrder, version)
case modelListGGUFTypeArray:
var arrayType uint32
if err := binary.Read(r, byteOrder, &arrayType); err != nil {
return err
}
var count uint64
if err := binary.Read(r, byteOrder, &count); err != nil {
return err
}
return skipModelListGGUFArray(r, byteOrder, version, arrayType, count)
default:
return fmt.Errorf("unsupported gguf value type %d", valueType)
}
}
func skipModelListGGUFArray(r io.Reader, byteOrder binary.ByteOrder, version uint32, arrayType uint32, count uint64) error {
if _, err := checkedModelListGGUFLength(count, "array size", fsgguf.MaxArraySize); err != nil {
return err
}
var size uint64
switch arrayType {
case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool:
size = 1
case modelListGGUFTypeUint16, modelListGGUFTypeInt16:
size = 2
case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32:
size = 4
case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64:
size = 8
case modelListGGUFTypeString:
for range count {
if err := skipModelListGGUFString(r, byteOrder, version); err != nil {
return err
}
}
return nil
default:
return fmt.Errorf("unsupported gguf array type %d", arrayType)
}
if count > uint64(maxModelListGGUFInt64())/size {
return fmt.Errorf("gguf array byte length %d*%d exceeds maximum %d", count, size, maxModelListGGUFInt64())
}
return discardModelListGGUFBytes(r, int64(count*size))
}
func readModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) (string, error) {
var length uint64
if err := binary.Read(r, byteOrder, &length); err != nil {
return "", err
}
n, err := checkedModelListGGUFLength(length, "string", fsgguf.MaxStringLength)
if err != nil {
return "", err
}
bts := make([]byte, n)
if _, err := io.ReadFull(r, bts); err != nil {
return "", err
}
if version == 1 && len(bts) > 0 && bts[len(bts)-1] == 0 {
bts = bts[:len(bts)-1]
}
return string(bts), nil
}
func skipModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) error {
var length uint64
if err := binary.Read(r, byteOrder, &length); err != nil {
return err
}
n, err := checkedModelListGGUFLength(length, "string", fsgguf.MaxStringLength)
if err != nil {
return err
}
return discardModelListGGUFBytes(r, int64(n))
}
func discardModelListGGUFBytes(r io.Reader, n int64) error {
if n <= 0 {
return nil
}
_, err := io.CopyN(io.Discard, r, n)
return err
}
func checkedModelListGGUFLength(n uint64, kind string, max uint64) (int, error) {
if n > uint64(maxModelListGGUFInt()) {
return 0, fmt.Errorf("gguf %s %d exceeds maximum %d", kind, n, maxModelListGGUFInt())
}
if n > max {
return 0, fmt.Errorf("gguf %s %d exceeds maximum %d", kind, n, max)
}
return int(n), nil
}
func maxModelListGGUFInt() int {
return int(^uint(0) >> 1)
}
func maxModelListGGUFInt64() int64 {
return 1<<63 - 1
}
func appendModelListCapabilities(capabilities []model.Capability, values ...model.Capability) []model.Capability {
for _, capability := range values {
capabilities = appendModelListCapability(capabilities, capability)
}
return capabilities
}
func appendModelListCapability(capabilities []model.Capability, capability model.Capability) []model.Capability {
if capability == "" || slices.Contains(capabilities, capability) {
return capabilities
}
return append(capabilities, capability)
}
func isUnknownQuantization(quantization string) bool {
return quantization == "" || quantization == "unknown"
}
func cloneModelListSummary(summary modelListSummary) modelListSummary {
summary.Details.Families = append([]string(nil), summary.Details.Families...)
summary.Capabilities = append([]model.Capability(nil), summary.Capabilities...)
return summary
}
func (s modelListSummary) ListModelResponse() api.ListModelResponse {
resp := api.ListModelResponse{
Model: s.Model,
Name: s.Name,
RemoteModel: s.RemoteModel,
RemoteHost: s.RemoteHost,
Size: s.Size,
Digest: s.Digest,
ModifiedAt: s.ModifiedAt,
Details: api.ModelDetails{
ParentModel: s.Details.ParentModel,
Format: s.Details.Format,
Family: s.Details.Family,
Families: append([]string(nil), s.Details.Families...),
ParameterSize: s.Details.ParameterSize,
QuantizationLevel: s.Details.QuantizationLevel,
ContextLength: s.Details.ContextLength,
EmbeddingLength: s.Details.EmbeddingLength,
},
}
resp.Capabilities = append([]model.Capability(nil), s.Capabilities...)
return resp
}
func sortListModelResponses(models []api.ListModelResponse) {
slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
// Preserve the existing /api/tags order: most recently modified first.
return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
})
}
func (s *Server) refreshModelListCache(name model.Name) {
if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil {
return
}
if err := s.modelCaches.modelList.RefreshModel(name); err != nil {
slog.Warn("failed to refresh model list cache", "model", name.String(), "error", err)
}
}
func (s *Server) deleteModelListCache(name model.Name) {
if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil {
return
}
s.modelCaches.modelList.DeleteModel(name)
}

371
server/model_list_cache_test.go
View file

@ -1,371 +0,0 @@
package server
import (
"bytes"
"context"
"encoding/binary"
"errors"
"net/http"
"os"
"slices"
"strings"
"testing"
"github.com/gin-gonic/gin"
"github.com/ollama/ollama/api"
fsgguf "github.com/ollama/ollama/fs/gguf"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/types/model"
)
func TestModelListCacheHydratesSummary(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
createListCacheModel(t, "list-cache", map[string]any{
"test.context_length": uint32(4096),
"test.embedding_length": uint32(384),
}, "{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")
cache := newModelListCache()
if err := cache.hydrate(context.Background()); err != nil {
t.Fatalf("hydrate failed: %v", err)
}
summary, ok := cache.Get(model.ParseName("list-cache"))
if !ok {
t.Fatal("list summary missing")
}
if summary.Model != "list-cache:latest" || summary.Name != "list-cache:latest" {
t.Fatalf("summary model/name = %q/%q, want list-cache:latest", summary.Model, summary.Name)
}
if summary.Digest == "" {
t.Fatal("summary digest is empty")
}
if summary.Size == 0 {
t.Fatal("summary size is zero")
}
if summary.Details.Family != "test" || summary.Details.Format != "gguf" {
t.Fatalf("summary details = %+v, want gguf/test", summary.Details)
}
if summary.Details.ContextLength != 4096 {
t.Fatalf("context length = %d, want 4096", summary.Details.ContextLength)
}
if summary.Details.EmbeddingLength != 384 {
t.Fatalf("embedding length = %d, want 384", summary.Details.EmbeddingLength)
}
for _, capability := range []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert} {
if !slices.Contains(summary.Capabilities, capability) {
t.Fatalf("capabilities = %v, want %s", summary.Capabilities, capability)
}
}
listModel := summary.ListModelResponse()
if !slices.Contains(listModel.Capabilities, model.CapabilityTools) ||
listModel.Details.ContextLength != 4096 ||
listModel.Details.EmbeddingLength != 384 {
t.Fatalf("list response = %+v, want capabilities/context/embedding", listModel)
}
}
func TestModelListCacheSuppressesNemotronSafetensorsMedia(t *testing.T) {
caps := []model.Capability{
model.CapabilityCompletion,
model.CapabilityTools,
model.CapabilityThinking,
model.CapabilityVision,
model.CapabilityAudio,
}
got := filterUnsupportedModelListCapabilities(caps, model.ConfigV2{
ModelFormat: "safetensors",
Renderer: "nemotron-3-nano",
Parser: "nemotron-3-nano",
})
for _, capability := range []model.Capability{
model.CapabilityCompletion,
model.CapabilityTools,
model.CapabilityThinking,
} {
if !slices.Contains(got, capability) {
t.Fatalf("capabilities = %v, want %s", got, capability)
}
}
for _, capability := range []model.Capability{model.CapabilityVision, model.CapabilityAudio} {
if slices.Contains(got, capability) {
t.Fatalf("capabilities = %v, did not expect %s", got, capability)
}
}
}
func TestModelListCacheRefreshUpdatesEntry(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
createListCacheModel(t, "list-refresh", map[string]any{"test.context_length": uint32(1024)}, "")
cache := newModelListCache()
if err := cache.hydrate(context.Background()); err != nil {
t.Fatalf("hydrate failed: %v", err)
}
name := model.ParseName("list-refresh")
first, ok := cache.Get(name)
if !ok {
t.Fatal("list summary missing")
}
changeShowCacheManifest(t, "list-refresh")
if err := cache.RefreshModel(name); err != nil {
t.Fatalf("refresh failed: %v", err)
}
refreshed, ok := cache.Get(name)
if !ok {
t.Fatal("refreshed list summary missing")
}
if refreshed.Digest == first.Digest {
t.Fatalf("digest did not change after refresh: %s", refreshed.Digest)
}
if cache.Len() != 1 {
t.Fatalf("cache entries = %d, want 1", cache.Len())
}
}
func TestModelListCacheMutationHooks(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
cache := newModelListCache()
s := Server{modelCaches: &modelCaches{modelList: cache}}
_, digest := createBinFile(t, map[string]any{"test.context_length": uint32(2048)}, nil)
w := createRequest(t, s.CreateHandler, api.CreateRequest{
Model: "list-hooks",
Files: map[string]string{"model.gguf": digest},
Stream: &stream,
})
if w.Code != http.StatusOK {
t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())
}
if _, ok := cache.Get(model.ParseName("list-hooks")); !ok {
t.Fatal("create did not refresh model list cache")
}
w = createRequest(t, s.CopyHandler, api.CopyRequest{
Source: "list-hooks",
Destination: "list-hooks-copy",
})
if w.Code != http.StatusOK {
t.Fatalf("copy model status = %d, want 200: %s", w.Code, w.Body.String())
}
if _, ok := cache.Get(model.ParseName("list-hooks-copy")); !ok {
t.Fatal("copy did not refresh model list cache")
}
w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Model: "list-hooks-copy"})
if w.Code != http.StatusOK {
t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String())
}
if _, ok := cache.Get(model.ParseName("list-hooks-copy")); ok {
t.Fatal("delete did not remove model list cache entry")
}
}
func TestModelListCacheSyncsManifestChanges(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
createListCacheModel(t, "list-sync-a", map[string]any{"test.context_length": uint32(1024)}, "")
cache := newModelListCache()
cache.Start(context.Background())
if err := cache.Wait(context.Background()); err != nil {
t.Fatalf("wait failed: %v", err)
}
createListCacheModel(t, "list-sync-b", map[string]any{"test.context_length": uint32(2048)}, "")
models, err := cache.List(context.Background())
if err != nil {
t.Fatalf("list failed: %v", err)
}
names := make([]string, 0, len(models))
for _, m := range models {
names = append(names, m.Name)
}
for _, want := range []string{"list-sync-a:latest", "list-sync-b:latest"} {
if !slices.Contains(names, want) {
t.Fatalf("names = %v, want %s", names, want)
}
}
var other Server
w := createRequest(t, other.DeleteHandler, api.DeleteRequest{Model: "list-sync-a"})
if w.Code != http.StatusOK {
t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String())
}
models, err = cache.List(context.Background())
if err != nil {
t.Fatalf("list after delete failed: %v", err)
}
names = names[:0]
for _, m := range models {
names = append(names, m.Name)
}
if slices.Contains(names, "list-sync-a:latest") || !slices.Contains(names, "list-sync-b:latest") {
t.Fatalf("names after delete = %v, want only list-sync-b", names)
}
}
func TestModelListCacheSyncDropsStaleEntryOnRefreshFailure(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
createListCacheModel(t, "list-stale", map[string]any{"test.context_length": uint32(1024)}, "")
cache := newModelListCache()
cache.Start(context.Background())
if err := cache.Wait(context.Background()); err != nil {
t.Fatalf("wait failed: %v", err)
}
name := model.ParseName("list-stale")
if _, ok := cache.Get(name); !ok {
t.Fatal("list summary missing")
}
changeShowCacheManifest(t, "list-stale")
cache.build = func(model.Name, *manifest.Manifest) (modelListSummary, error) {
return modelListSummary{}, errors.New("refresh failed")
}
models, err := cache.List(context.Background())
if err != nil {
t.Fatalf("list failed: %v", err)
}
if len(models) != 0 {
t.Fatalf("models = %+v, want stale entry removed", models)
}
if _, ok := cache.Get(name); ok {
t.Fatal("stale entry remained in cache after refresh failure")
}
}
func TestReadModelListGGUFRejectsMalformedMetadata(t *testing.T) {
cases := []struct {
name string
data []byte
want string
}{
{
name: "oversized key string",
data: modelListGGUFTestFile(func(b *bytes.Buffer) {
writeModelListGGUFHeader(t, b, 1)
writeModelListGGUFUint64(t, b, fsgguf.MaxStringLength+1)
}),
want: "string",
},
{
name: "oversized skipped string",
data: modelListGGUFTestFile(func(b *bytes.Buffer) {
writeModelListGGUFHeader(t, b, 1)
writeModelListGGUFString(t, b, "unused")
writeModelListGGUFUint32(t, b, modelListGGUFTypeString)
writeModelListGGUFUint64(t, b, fsgguf.MaxStringLength+1)
}),
want: "string",
},
{
name: "oversized skipped array",
data: modelListGGUFTestFile(func(b *bytes.Buffer) {
writeModelListGGUFHeader(t, b, 1)
writeModelListGGUFString(t, b, "unused")
writeModelListGGUFUint32(t, b, modelListGGUFTypeArray)
writeModelListGGUFUint32(t, b, modelListGGUFTypeUint8)
writeModelListGGUFUint64(t, b, fsgguf.MaxArraySize+1)
}),
want: "array size",
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
defer func() {
if r := recover(); r != nil {
t.Fatalf("readModelListGGUF panicked: %v", r)
}
}()
path := t.TempDir() + "/model.gguf"
if err := os.WriteFile(path, tt.data, 0o600); err != nil {
t.Fatal(err)
}
_, err := readModelListGGUF(path)
if err == nil {
t.Fatal("expected error")
}
if !strings.Contains(err.Error(), tt.want) {
t.Fatalf("error = %v, want substring %q", err, tt.want)
}
})
}
}
func createListCacheModel(t *testing.T, name string, kv map[string]any, tmpl string) {
t.Helper()
_, digest := createBinFile(t, kv, nil)
req := api.CreateRequest{
Model: name,
Files: map[string]string{"model.gguf": digest},
Stream: &stream,
}
if tmpl != "" {
req.Template = tmpl
}
var s Server
w := createRequest(t, s.CreateHandler, req)
if w.Code != http.StatusOK {
t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())
}
}
func modelListGGUFTestFile(fn func(*bytes.Buffer)) []byte {
var b bytes.Buffer
fn(&b)
return b.Bytes()
}
func writeModelListGGUFHeader(t *testing.T, b *bytes.Buffer, numKV uint64) {
t.Helper()
writeModelListGGUFUint32(t, b, modelListGGUFMagicLE)
writeModelListGGUFUint32(t, b, 3)
writeModelListGGUFUint64(t, b, 0)
writeModelListGGUFUint64(t, b, numKV)
}
func writeModelListGGUFString(t *testing.T, b *bytes.Buffer, s string) {
t.Helper()
writeModelListGGUFUint64(t, b, uint64(len(s)))
if _, err := b.WriteString(s); err != nil {
t.Fatal(err)
}
}
func writeModelListGGUFUint32(t *testing.T, b *bytes.Buffer, v uint32) {
t.Helper()
if err := binary.Write(b, binary.LittleEndian, v); err != nil {
t.Fatal(err)
}
}
func writeModelListGGUFUint64(t *testing.T, b *bytes.Buffer, v uint64) {
t.Helper()
if err := binary.Write(b, binary.LittleEndian, v); err != nil {
t.Fatal(err)
}
}

168
server/model_list_test.go Normal file
View file

@ -0,0 +1,168 @@
package server
import (
"context"
"os"
"slices"
"testing"
"github.com/gin-gonic/gin"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/types/model"
)
func listedModel(t *testing.T, name string) api.ListModelResponse {
t.Helper()
models, err := listModels(context.Background())
if err != nil {
t.Fatalf("listModels failed: %v", err)
}
for _, m := range models {
if m.Name == name {
return m
}
}
t.Fatalf("%s not listed; got %v", name, models)
return api.ListModelResponse{}
}
func TestListModelsDescribesModel(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
createListedModelFromKV(t, "list-describe", map[string]any{
"test.context_length": uint32(4096),
"test.embedding_length": uint32(384),
}, "{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")
got := listedModel(t, "list-describe:latest")
if got.Model != "list-describe:latest" {
t.Errorf("model = %q", got.Model)
}
if got.Digest == "" || got.Size == 0 {
t.Errorf("digest = %q size = %d, want both set", got.Digest, got.Size)
}
if got.Details.Family != "test" || got.Details.Format != "gguf" {
t.Errorf("details = %+v, want gguf/test", got.Details)
}
if got.Details.ContextLength != 4096 {
t.Errorf("context length = %d, want 4096", got.Details.ContextLength)
}
if got.Details.EmbeddingLength != 384 {
t.Errorf("embedding length = %d, want 384", got.Details.EmbeddingLength)
}
// The test GGUF has no general.file_type; the list must not guess F32
// (FileType 0) from a missing key.
if !isUnknownQuantization(got.Details.QuantizationLevel) {
t.Errorf("quantization = %q, want unknown (no general.file_type in GGUF)", got.Details.QuantizationLevel)
}
for _, capability := range []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert} {
if !slices.Contains(got.Capabilities, capability) {
t.Errorf("capabilities = %v, want %s", got.Capabilities, capability)
}
}
}
// Listing reads the manifests every time, so a model created or deleted after
// the last call shows up without anything needing to be told about it.
func TestListModelsFollowsManifestChanges(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
createListedModelFromKV(t, "list-follow-a", map[string]any{"test.context_length": uint32(1024)}, "")
listedModel(t, "list-follow-a:latest")
createListedModelFromKV(t, "list-follow-b", map[string]any{"test.context_length": uint32(2048)}, "")
listedModel(t, "list-follow-a:latest")
listedModel(t, "list-follow-b:latest")
deleteModelNamed(t, "list-follow-a")
models, err := listModels(context.Background())
if err != nil {
t.Fatal(err)
}
names := make([]string, 0, len(models))
for _, m := range models {
names = append(names, m.Name)
}
if slices.Contains(names, "list-follow-a:latest") || !slices.Contains(names, "list-follow-b:latest") {
t.Fatalf("names after delete = %v, want only list-follow-b", names)
}
}
func TestCapabilitiesSuppressNemotronSafetensorsMedia(t *testing.T) {
caps := []model.Capability{
model.CapabilityCompletion,
model.CapabilityTools,
model.CapabilityThinking,
model.CapabilityVision,
model.CapabilityAudio,
}
got := (&Model{Config: model.ConfigV2{
ModelFormat: "safetensors",
Renderer: "nemotron-3-nano",
Parser: "nemotron-3-nano",
}}).filterUnsupportedCapabilities(caps, "")
for _, capability := range []model.Capability{
model.CapabilityCompletion,
model.CapabilityTools,
model.CapabilityThinking,
} {
if !slices.Contains(got, capability) {
t.Errorf("capabilities = %v, want %s", got, capability)
}
}
for _, capability := range []model.Capability{model.CapabilityVision, model.CapabilityAudio} {
if slices.Contains(got, capability) {
t.Errorf("capabilities = %v, did not expect %s", got, capability)
}
}
}
// A model whose layers no longer parse must still be listed: it is the one the
// user most needs to see, in order to remove it.
func TestListModelsKeepsUnloadableModel(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
createListedModelFromKV(t, "broken", map[string]any{"test.context_length": uint32(1024)}, "{{ .Prompt }}")
mf, err := manifest.ParseNamedManifest(model.ParseName("broken"))
if err != nil {
t.Fatal(err)
}
var corrupted bool
for _, layer := range mf.Layers {
if layer.MediaType != "application/vnd.ollama.image.template" {
continue
}
path, err := manifest.BlobsPath(layer.Digest)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte("{{ if }"), 0o644); err != nil {
t.Fatal(err)
}
corrupted = true
}
if !corrupted {
t.Fatal("no template layer to corrupt")
}
if _, err := GetModel("broken"); err == nil {
t.Fatal("model still loads, so the listing is not being asked the question")
}
got := listedModel(t, "broken:latest")
if got.Details.Family != "test" {
t.Errorf("details = %+v, want the manifest config's family", got.Details)
}
}
func createListedModelFromKV(t *testing.T, name string, kv map[string]any, tmpl string) {
t.Helper()
_, digest := createBinFile(t, kv, nil)
createModelFromBlob(t, name, digest, tmpl)
}

25
server/routes.go
View file

@ -286,7 +286,7 @@ func (s *Server) GenerateHandler(c *gin.Context) {
return
}
m, err := s.getModel(name.String())
m, err := GetModel(name.String())
if err != nil {
switch {
case errors.Is(err, fs.ErrNotExist):
@ -850,7 +850,7 @@ func (s *Server) EmbedHandler(c *gin.Context) {
return
}
m, err := s.getModel(name.String())
m, err := GetModel(name.String())
if err != nil {
handleScheduleError(c, req.Model, err)
return
@ -1068,7 +1068,7 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) {
name := modelRef.Name
m, err := s.getModel(name.String())
m, err := GetModel(name.String())
if err != nil {
handleScheduleError(c, req.Model, err)
return
@ -1153,8 +1153,6 @@ func (s *Server) PullHandler(c *gin.Context) {
ch <- gin.H{"error": err.Error()}
return
}
s.refreshModelListCache(name)
}()
if req.Stream != nil && !*req.Stream {
@ -1294,9 +1292,9 @@ func (s *Server) DeleteHandler(c *gin.Context) {
return
}
s.deleteModelListCache(n)
if err := m.RemoveLayers(); err != nil {
removed, err := m.RemoveLayers()
removeGGUFMetadata(removed...)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
@ -1635,12 +1633,7 @@ func selectedModelTemplate(m *Model, kv ggml.KV) string {
}
func (s *Server) ListHandler(c *gin.Context) {
if s.modelCaches == nil || s.modelCaches.modelList == nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "model list cache unavailable"})
return
}
models, err := s.modelCaches.modelList.List(c.Request.Context())
models, err := listModels(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
@ -1685,8 +1678,6 @@ func (s *Server) CopyHandler(c *gin.Context) {
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
} else if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
} else {
s.refreshModelListCache(dst)
}
}
@ -2492,7 +2483,7 @@ func (s *Server) ChatHandler(c *gin.Context) {
return
}
m, err := s.getModel(name.String())
m, err := GetModel(name.String())
if err != nil {
switch {
case os.IsNotExist(err):

17
server/routes_list_test.go
View file

@ -1,7 +1,6 @@
package server
import (
"context"
"encoding/json"
"io"
"net/http"
@ -36,11 +35,7 @@ func TestList(t *testing.T) {
"myhost/mynamespace/lips:code",
}
s := Server{modelCaches: &modelCaches{modelList: newModelListCache()}}
s.modelCaches.modelList.Start(context.Background())
if err := s.modelCaches.modelList.Wait(context.Background()); err != nil {
t.Fatal(err)
}
var s Server
for _, n := range expectNames {
_, digest := createBinFile(t, nil, nil)
@ -88,12 +83,7 @@ func TestOpenAIListMatchesTagsModels(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
cache := newModelListCache()
s := Server{modelCaches: &modelCaches{modelList: cache}}
cache.Start(context.Background())
if err := cache.Wait(context.Background()); err != nil {
t.Fatal(err)
}
var s Server
createModel := func(name string) {
t.Helper()
@ -123,9 +113,6 @@ func TestOpenAIListMatchesTagsModels(t *testing.T) {
if err := os.Chtimes(path, modified, modified); err != nil {
t.Fatalf("set manifest time for %s: %v", name, err)
}
if err := cache.RefreshModel(parsed); err != nil {
t.Fatalf("refresh %s: %v", name, err)
}
}
older := time.Unix(1000, 0).UTC()

7
server/routes_test.go
View file

@ -95,11 +95,7 @@ func TestRoutes(t *testing.T) {
Expected func(t *testing.T, resp *http.Response)
}
s := &Server{modelCaches: &modelCaches{modelList: newModelListCache()}}
s.modelCaches.modelList.Start(context.Background())
if err := s.modelCaches.modelList.Wait(context.Background()); err != nil {
t.Fatal(err)
}
s := &Server{}
createTestModel := func(t *testing.T, name string) {
t.Helper()
@ -135,7 +131,6 @@ func TestRoutes(t *testing.T) {
if err := createModel(r, modelName, baseLayers, config, fn); err != nil {
t.Fatal(err)
}
s.refreshModelListCache(modelName)
}
testCases := []testCase{