From 01557ff313adef5ebfbb0fefb6b3125d22d99723 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Wed, 3 Jun 2026 13:58:40 -0700 Subject: [PATCH] llama-server: allow GPU offload for projectors (#16473) Special case Metal iGPUs to enable GPU offload. --- llm/llama_server.go | 2 +- llm/llama_server_test.go | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/llm/llama_server.go b/llm/llama_server.go index b38d887cb..85701351f 100644 --- a/llm/llama_server.go +++ b/llm/llama_server.go @@ -631,7 +631,7 @@ func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLay } for _, gpu := range gpus { - if gpu.Integrated { + if gpu.Integrated && gpu.Library != "Metal" { return true, "shared-memory-gpu" } memory := gpu.FreeMemory diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go index d5915c4df..b870bea28 100644 --- a/llm/llama_server_test.go +++ b/llm/llama_server_test.go @@ -1839,13 +1839,21 @@ func TestAppendMMProjArgs(t *testing.T) { want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"}, }, { - name: "integrated gpu disables projector offload", + name: "integrated rocm gpu disables projector offload", projectors: []string{"model.gguf"}, opts: defaultOpts, gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "ROCm"}, Integrated: true, FreeMemory: 32 << 30}}, modelLayers: 81, want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"}, }, + { + name: "integrated metal gpu keeps projector offload", + projectors: []string{"model.gguf"}, + opts: defaultOpts, + gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true, FreeMemory: 32 << 30}}, + modelLayers: 81, + want: []string{"base", "--mmproj", "model.gguf"}, + }, { name: "cpu only request disables projector offload", projectors: []string{"model.gguf"},