CERBERUS/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2026-08-04 14:56:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

llama-server: allow GPU offload for projectors (#16473)

Browse source 
Special case Metal iGPUs to enable GPU offload.
This commit is contained in:
Daniel Hiltgen 2026-06-03 13:58:40 -07:00 • committed by GitHub
parent e5a38739b4
commit 01557ff313
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 10 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llm/llama_server.go
View file

@ -631,7 +631,7 @@ func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLay
}
for _, gpu := range gpus {
if gpu.Integrated {
if gpu.Integrated && gpu.Library != "Metal" {
return true, "shared-memory-gpu"
}
memory := gpu.FreeMemory

10
llm/llama_server_test.go
View file

@ -1839,13 +1839,21 @@ func TestAppendMMProjArgs(t *testing.T) {
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
name: "integrated gpu disables projector offload",
name: "integrated rocm gpu disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "ROCm"}, Integrated: true, FreeMemory: 32 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
name: "integrated metal gpu keeps projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true, FreeMemory: 32 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
},
{
name: "cpu only request disables projector offload",
projectors: []string{"model.gguf"},