mirror of
https://github.com/ollama/ollama.git
synced 2026-07-10 17:54:12 +00:00
discover: fix inverted iGPU/dGPU Vulkan classification on Windows hybrid graphics (#16669)
On Windows hybrid-graphics systems (Intel iGPU + NVIDIA dGPU), discovery could classify the integrated GPU as discrete and the discrete GPU as integrated, dropping the dGPU's Vulkan device and scheduling models onto the iGPU's shared system RAM (#16667). Two index-keyed correlations between independently-ordered device enumerations caused this: 1. The native probe's stderr was concatenated into the output passed to parseVulkanUMA. The probe enumerates Vulkan devices in its own order, so its ggml_vulkan uma lines overwrote llama-server's index-keyed UMA map with inverted values. Parse UMA metadata only from llama-server's own output. 2. applyWindowsVulkanRefinement required the raw vkEnumeratePhysicalDevices count to equal llama-server's Vulkan device count. The raw enumeration is a superset on real systems (D3D12 mapping-layer devices, Microsoft Basic Render Driver), so the refinement that reads the authoritative VkPhysicalDeviceType was always skipped. Match devices by name against the probed superset instead, bailing only when a device has no match or matches conflicting device types. Verified on the hardware from #16667 (Intel RaptorLake-S + RTX 4080 Laptop): the raw probe returns 5 devices vs llama-server's 2; with this change the iGPU is dropped as integrated, the dGPU's Vulkan device dedupes against CUDA0, and the model loads on the dGPU with no environment overrides. Fixes #16667
This commit is contained in:
Sahil Kadadekar
GitHub
parent
e434a93884
commit
fc58544422
4 changed files with 100 additions and 18 deletions
|
|
@ -160,8 +160,8 @@ func llamaServerDiscoverDevices(ctx context.Context, libDirs []string, extraEnvs
|
|||
logNativeProbeFailure(nativeErr, nativeStderr, libDirs)
|
||||
}
|
||||
|
||||
combined := string(listOutput) + "\n" + strings.Join(stderrLines, "\n") + "\n" + nativeStderr
|
||||
return parseLlamaServerDevicesWithNative(combined, libDirs, nativeDevices), status, nil
|
||||
llamaOutput := string(listOutput) + "\n" + strings.Join(stderrLines, "\n")
|
||||
return parseLlamaServerDevicesWithNative(llamaOutput, nativeStderr, libDirs, nativeDevices), status, nil
|
||||
}
|
||||
|
||||
func llamaServerDiscoveryOutput(ctx context.Context) io.Writer {
|
||||
|
|
@ -203,13 +203,20 @@ var (
|
|||
// It extracts device info, ROCm gfx targets, CUDA compute capabilities, and
|
||||
// CUDA compiled architecture lists.
|
||||
func parseLlamaServerDevices(output string, libDirs []string) []ml.DeviceInfo {
|
||||
return parseLlamaServerDevicesWithNative(output, libDirs, nil)
|
||||
return parseLlamaServerDevicesWithNative(output, "", libDirs, nil)
|
||||
}
|
||||
|
||||
func parseLlamaServerDevicesWithNative(output string, libDirs []string, nativeDevices []nativeProbeDevice) []ml.DeviceInfo {
|
||||
func parseLlamaServerDevicesWithNative(output, nativeOutput string, libDirs []string, nativeDevices []nativeProbeDevice) []ml.DeviceInfo {
|
||||
combined := output
|
||||
if nativeOutput != "" {
|
||||
combined += "\n" + nativeOutput
|
||||
}
|
||||
// Extract per-device metadata from stderr
|
||||
gfxByIndex := parseROCmGFXTargets(output)
|
||||
gfxByIndex := parseROCmGFXTargets(combined)
|
||||
rocmGFXOverride := hsaOverrideGFXTarget()
|
||||
// The native probe enumerates Vulkan devices in its own order, which can
|
||||
// differ from llama-server's, so its ggml_vulkan uma lines must not key
|
||||
// into llama-server's device indexes.
|
||||
integratedByIndex := parseVulkanUMA(output)
|
||||
ccByIndex := make(map[int]cudaComputeCapability)
|
||||
var cudaArchs []string // compiled architectures for this variant
|
||||
|
|
@ -222,7 +229,7 @@ func parseLlamaServerDevicesWithNative(output string, libDirs []string, nativeDe
|
|||
}
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(output))
|
||||
scanner := bufio.NewScanner(strings.NewReader(combined))
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if matches := cudaCCRegex.FindStringSubmatch(line); matches != nil {
|
||||
|
|
|
|||
|
|
@ -399,7 +399,7 @@ Available devices:
|
|||
TotalMemory: 16107 * 1024 * 1024,
|
||||
}}
|
||||
|
||||
devices := parseLlamaServerDevicesWithNative(output, []string{"/lib/ollama", "/lib/ollama/vulkan"}, nativeDevices)
|
||||
devices := parseLlamaServerDevicesWithNative(output, "", []string{"/lib/ollama", "/lib/ollama/vulkan"}, nativeDevices)
|
||||
if len(devices) != 1 {
|
||||
t.Fatalf("got %d devices, want 1", len(devices))
|
||||
}
|
||||
|
|
@ -411,6 +411,33 @@ Available devices:
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("native probe uma lines do not key into llama-server device order", func(t *testing.T) {
|
||||
// llama-server enumerates [Intel iGPU, NVIDIA dGPU]; the native probe
|
||||
// enumerates the same devices in the opposite order. Its ggml_vulkan
|
||||
// uma lines must not overwrite llama-server's index-keyed UMA map,
|
||||
// otherwise the classification inverts (#16667).
|
||||
output := `ggml_vulkan: 0 = Intel(R) RaptorLake-S Mobile Graphics Controller (Intel Corporation) | uma: 1 | fp16: 1 | bf16: 0 | warp size: 32 | shared memory: 65536 | int dot: 1 | matrix cores: none
|
||||
ggml_vulkan: 1 = NVIDIA GeForce RTX 4080 Laptop GPU (NVIDIA) | uma: 0 | fp16: 1 | bf16: 0 | warp size: 32 | shared memory: 49152 | int dot: 1 | matrix cores: KHR_coopmat
|
||||
Available devices:
|
||||
Vulkan0: Intel(R) RaptorLake-S Mobile Graphics Controller (32550 MiB, 31800 MiB free)
|
||||
Vulkan1: NVIDIA GeForce RTX 4080 Laptop GPU (12282 MiB, 11000 MiB free)
|
||||
`
|
||||
nativeOutput := `ggml_vulkan: 0 = NVIDIA GeForce RTX 4080 Laptop GPU (NVIDIA) | uma: 0 | fp16: 1 | bf16: 0 | warp size: 32 | shared memory: 49152 | int dot: 1 | matrix cores: KHR_coopmat
|
||||
ggml_vulkan: 1 = Intel(R) RaptorLake-S Mobile Graphics Controller (Intel Corporation) | uma: 1 | fp16: 1 | bf16: 0 | warp size: 32 | shared memory: 65536 | int dot: 1 | matrix cores: none
|
||||
`
|
||||
|
||||
devices := parseLlamaServerDevicesWithNative(output, nativeOutput, []string{"/lib/ollama", "/lib/ollama/vulkan"}, nil)
|
||||
if len(devices) != 2 {
|
||||
t.Fatalf("got %d devices, want 2", len(devices))
|
||||
}
|
||||
if !devices[0].Integrated {
|
||||
t.Fatal("device 0 (Intel iGPU) Integrated = false, want true")
|
||||
}
|
||||
if devices[1].Integrated {
|
||||
t.Fatal("device 1 (NVIDIA dGPU) Integrated = true, want false")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cuda runtime version", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "libcudart.so.12.8.90"), nil, 0o644); err != nil {
|
||||
|
|
@ -471,10 +498,50 @@ Available devices:
|
|||
want: []bool{false, false, false},
|
||||
},
|
||||
{
|
||||
name: "skips when counts do not line up",
|
||||
name: "skips when probe finds fewer devices",
|
||||
probed: []vulkanPhysicalDevice{{Name: "AMD Radeon(TM) Graphics", Integrated: true}},
|
||||
want: []bool{false, false, false},
|
||||
},
|
||||
{
|
||||
name: "matches subset when probe enumerates extra devices",
|
||||
devices: []ml.DeviceInfo{
|
||||
{DeviceID: ml.DeviceID{ID: "0", Library: "Vulkan"}, Description: "Intel(R) RaptorLake-S Mobile Graphics Controller"},
|
||||
{DeviceID: ml.DeviceID{ID: "1", Library: "Vulkan"}, Description: "NVIDIA GeForce RTX 4080 Laptop GPU"},
|
||||
},
|
||||
probed: []vulkanPhysicalDevice{
|
||||
{Name: "NVIDIA GeForce RTX 4080 Laptop GPU", Integrated: false},
|
||||
{Name: "Intel(R) RaptorLake-S Mobile Graphics Controller", Integrated: true},
|
||||
{Name: "Microsoft Direct3D12 (NVIDIA GeForce RTX 4080 Laptop GPU)", Integrated: false},
|
||||
{Name: "Microsoft Direct3D12 (Intel(R) RaptorLake-S Mobile Graphics Controller)", Integrated: true},
|
||||
{Name: "llvmpipe (LLVM 17.0.6, 256 bits)", Integrated: false},
|
||||
},
|
||||
want: []bool{true, false},
|
||||
applied: true,
|
||||
},
|
||||
{
|
||||
name: "skips ambiguous duplicate names with conflicting types",
|
||||
devices: []ml.DeviceInfo{
|
||||
{DeviceID: ml.DeviceID{ID: "0", Library: "Vulkan"}, Description: "AMD Radeon Graphics"},
|
||||
},
|
||||
probed: []vulkanPhysicalDevice{
|
||||
{Name: "AMD Radeon Graphics", Integrated: true},
|
||||
{Name: "AMD Radeon Graphics", Integrated: false},
|
||||
},
|
||||
want: []bool{false},
|
||||
},
|
||||
{
|
||||
name: "matches duplicate names with agreeing types",
|
||||
devices: []ml.DeviceInfo{
|
||||
{DeviceID: ml.DeviceID{ID: "0", Library: "Vulkan"}, Description: "AMD Radeon RX 7900 XTX"},
|
||||
{DeviceID: ml.DeviceID{ID: "1", Library: "Vulkan"}, Description: "AMD Radeon RX 7900 XTX"},
|
||||
},
|
||||
probed: []vulkanPhysicalDevice{
|
||||
{Name: "AMD Radeon RX 7900 XTX", Integrated: false},
|
||||
{Name: "AMD Radeon RX 7900 XTX", Integrated: false},
|
||||
},
|
||||
want: []bool{false, false},
|
||||
applied: true,
|
||||
},
|
||||
{
|
||||
name: "overwrites stale classification",
|
||||
devices: []ml.DeviceInfo{
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func TestParseLlamaServerDevicesUsesNativeCUDAComputeCapability(t *testing.T) {
|
|||
Available devices:
|
||||
CUDA0: NVIDIA GeForce GTX 1060 6GB (6063 MiB, 5900 MiB free)
|
||||
`
|
||||
devices := parseLlamaServerDevicesWithNative(output, []string{"/lib/ollama", "/lib/ollama/cuda_v13"}, []nativeProbeDevice{{
|
||||
devices := parseLlamaServerDevicesWithNative(output, "", []string{"/lib/ollama", "/lib/ollama/cuda_v13"}, []nativeProbeDevice{{
|
||||
Library: "CUDA",
|
||||
Index: 0,
|
||||
IndexMatchesBackend: true,
|
||||
|
|
@ -64,7 +64,7 @@ Available devices:
|
|||
Available devices:
|
||||
CUDA0: NVIDIA GeForce GTX 1060 6GB (6063 MiB, 5900 MiB free)
|
||||
`
|
||||
devices = parseLlamaServerDevicesWithNative(output, []string{"/lib/ollama", "/lib/ollama/cuda_v12"}, []nativeProbeDevice{{
|
||||
devices = parseLlamaServerDevicesWithNative(output, "", []string{"/lib/ollama", "/lib/ollama/cuda_v12"}, []nativeProbeDevice{{
|
||||
Library: "CUDA",
|
||||
Index: 0,
|
||||
IndexMatchesBackend: true,
|
||||
|
|
@ -97,7 +97,7 @@ func TestParseLlamaServerDevicesUsesNativeROCmMetadata(t *testing.T) {
|
|||
Available devices:
|
||||
ROCm0: AMD Radeon RX 7600 (8176 MiB, 7900 MiB free)
|
||||
`
|
||||
devices := parseLlamaServerDevicesWithNative(output, []string{"/lib/ollama", "/lib/ollama/rocm_v7_2"}, []nativeProbeDevice{{
|
||||
devices := parseLlamaServerDevicesWithNative(output, "", []string{"/lib/ollama", "/lib/ollama/rocm_v7_2"}, []nativeProbeDevice{{
|
||||
Library: "ROCm",
|
||||
Index: 0,
|
||||
IndexMatchesBackend: true,
|
||||
|
|
|
|||
|
|
@ -95,32 +95,40 @@ func applyWindowsVulkanRefinement(devices []ml.DeviceInfo, probed []vulkanPhysic
|
|||
}
|
||||
}
|
||||
|
||||
if len(probed) != len(vulkanIndexes) {
|
||||
slog.Debug("windows vulkan device refinement skipped: device count mismatch",
|
||||
if len(probed) < len(vulkanIndexes) {
|
||||
slog.Debug("windows vulkan device refinement skipped: fewer probed devices than llama-server devices",
|
||||
"llama_server_count", len(vulkanIndexes), "vulkan_count", len(probed))
|
||||
return false
|
||||
}
|
||||
|
||||
// Raw Vulkan enumeration can be a superset of llama-server's device list
|
||||
// (extra ICDs, D3D12 mapping-layer devices) and the two orders can differ,
|
||||
// so match by name rather than requiring equal counts or matching indexes.
|
||||
matches := make([]int, len(vulkanIndexes))
|
||||
for i := range matches {
|
||||
matches[i] = -1
|
||||
}
|
||||
used := make([]bool, len(probed))
|
||||
for i, deviceIndex := range vulkanIndexes {
|
||||
matches[i] = -1
|
||||
description := devices[deviceIndex].Description
|
||||
for j, probedDevice := range probed {
|
||||
if used[j] || !sameVulkanDeviceName(description, probedDevice.Name) {
|
||||
continue
|
||||
}
|
||||
if matches[i] >= 0 {
|
||||
if probed[matches[i]].Integrated != probedDevice.Integrated {
|
||||
slog.Debug("windows vulkan device refinement skipped: ambiguous device name match",
|
||||
"index", i, "llama_server_name", description)
|
||||
return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
matches[i] = j
|
||||
used[j] = true
|
||||
break
|
||||
}
|
||||
if matches[i] < 0 {
|
||||
slog.Debug("windows vulkan device refinement skipped: device name mismatch",
|
||||
"index", i, "llama_server_name", description)
|
||||
return false
|
||||
}
|
||||
used[matches[i]] = true
|
||||
}
|
||||
|
||||
for i, probedIndex := range matches {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue