From fc58544422d1a80102c8d36cb68762b482f4f4e5 Mon Sep 17 00:00:00 2001 From: Sahil Kadadekar <147995121+Sahil170595@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:52:03 -0400 Subject: [PATCH] discover: fix inverted iGPU/dGPU Vulkan classification on Windows hybrid graphics (#16669) On Windows hybrid-graphics systems (Intel iGPU + NVIDIA dGPU), discovery could classify the integrated GPU as discrete and the discrete GPU as integrated, dropping the dGPU's Vulkan device and scheduling models onto the iGPU's shared system RAM (#16667). Two index-keyed correlations between independently-ordered device enumerations caused this: 1. The native probe's stderr was concatenated into the output passed to parseVulkanUMA. The probe enumerates Vulkan devices in its own order, so its ggml_vulkan uma lines overwrote llama-server's index-keyed UMA map with inverted values. Parse UMA metadata only from llama-server's own output. 2. applyWindowsVulkanRefinement required the raw vkEnumeratePhysicalDevices count to equal llama-server's Vulkan device count. The raw enumeration is a superset on real systems (D3D12 mapping-layer devices, Microsoft Basic Render Driver), so the refinement that reads the authoritative VkPhysicalDeviceType was always skipped. Match devices by name against the probed superset instead, bailing only when a device has no match or matches conflicting device types. Verified on the hardware from #16667 (Intel RaptorLake-S + RTX 4080 Laptop): the raw probe returns 5 devices vs llama-server's 2; with this change the iGPU is dropped as integrated, the dGPU's Vulkan device dedupes against CUDA0, and the model loads on the dGPU with no environment overrides. Fixes #16667 --- discover/llama_server.go | 19 +++++++--- discover/llama_server_test.go | 71 ++++++++++++++++++++++++++++++++++- discover/native_probe_test.go | 6 +-- discover/vulkan.go | 22 +++++++---- 4 files changed, 100 insertions(+), 18 deletions(-) diff --git a/discover/llama_server.go b/discover/llama_server.go index 7a79318ee..a66c34e87 100644 --- a/discover/llama_server.go +++ b/discover/llama_server.go @@ -160,8 +160,8 @@ func llamaServerDiscoverDevices(ctx context.Context, libDirs []string, extraEnvs logNativeProbeFailure(nativeErr, nativeStderr, libDirs) } - combined := string(listOutput) + "\n" + strings.Join(stderrLines, "\n") + "\n" + nativeStderr - return parseLlamaServerDevicesWithNative(combined, libDirs, nativeDevices), status, nil + llamaOutput := string(listOutput) + "\n" + strings.Join(stderrLines, "\n") + return parseLlamaServerDevicesWithNative(llamaOutput, nativeStderr, libDirs, nativeDevices), status, nil } func llamaServerDiscoveryOutput(ctx context.Context) io.Writer { @@ -203,13 +203,20 @@ var ( // It extracts device info, ROCm gfx targets, CUDA compute capabilities, and // CUDA compiled architecture lists. func parseLlamaServerDevices(output string, libDirs []string) []ml.DeviceInfo { - return parseLlamaServerDevicesWithNative(output, libDirs, nil) + return parseLlamaServerDevicesWithNative(output, "", libDirs, nil) } -func parseLlamaServerDevicesWithNative(output string, libDirs []string, nativeDevices []nativeProbeDevice) []ml.DeviceInfo { +func parseLlamaServerDevicesWithNative(output, nativeOutput string, libDirs []string, nativeDevices []nativeProbeDevice) []ml.DeviceInfo { + combined := output + if nativeOutput != "" { + combined += "\n" + nativeOutput + } // Extract per-device metadata from stderr - gfxByIndex := parseROCmGFXTargets(output) + gfxByIndex := parseROCmGFXTargets(combined) rocmGFXOverride := hsaOverrideGFXTarget() + // The native probe enumerates Vulkan devices in its own order, which can + // differ from llama-server's, so its ggml_vulkan uma lines must not key + // into llama-server's device indexes. integratedByIndex := parseVulkanUMA(output) ccByIndex := make(map[int]cudaComputeCapability) var cudaArchs []string // compiled architectures for this variant @@ -222,7 +229,7 @@ func parseLlamaServerDevicesWithNative(output string, libDirs []string, nativeDe } } - scanner := bufio.NewScanner(strings.NewReader(output)) + scanner := bufio.NewScanner(strings.NewReader(combined)) for scanner.Scan() { line := scanner.Text() if matches := cudaCCRegex.FindStringSubmatch(line); matches != nil { diff --git a/discover/llama_server_test.go b/discover/llama_server_test.go index 907fa9361..563da94e1 100644 --- a/discover/llama_server_test.go +++ b/discover/llama_server_test.go @@ -399,7 +399,7 @@ Available devices: TotalMemory: 16107 * 1024 * 1024, }} - devices := parseLlamaServerDevicesWithNative(output, []string{"/lib/ollama", "/lib/ollama/vulkan"}, nativeDevices) + devices := parseLlamaServerDevicesWithNative(output, "", []string{"/lib/ollama", "/lib/ollama/vulkan"}, nativeDevices) if len(devices) != 1 { t.Fatalf("got %d devices, want 1", len(devices)) } @@ -411,6 +411,33 @@ Available devices: } }) + t.Run("native probe uma lines do not key into llama-server device order", func(t *testing.T) { + // llama-server enumerates [Intel iGPU, NVIDIA dGPU]; the native probe + // enumerates the same devices in the opposite order. Its ggml_vulkan + // uma lines must not overwrite llama-server's index-keyed UMA map, + // otherwise the classification inverts (#16667). + output := `ggml_vulkan: 0 = Intel(R) RaptorLake-S Mobile Graphics Controller (Intel Corporation) | uma: 1 | fp16: 1 | bf16: 0 | warp size: 32 | shared memory: 65536 | int dot: 1 | matrix cores: none +ggml_vulkan: 1 = NVIDIA GeForce RTX 4080 Laptop GPU (NVIDIA) | uma: 0 | fp16: 1 | bf16: 0 | warp size: 32 | shared memory: 49152 | int dot: 1 | matrix cores: KHR_coopmat +Available devices: + Vulkan0: Intel(R) RaptorLake-S Mobile Graphics Controller (32550 MiB, 31800 MiB free) + Vulkan1: NVIDIA GeForce RTX 4080 Laptop GPU (12282 MiB, 11000 MiB free) +` + nativeOutput := `ggml_vulkan: 0 = NVIDIA GeForce RTX 4080 Laptop GPU (NVIDIA) | uma: 0 | fp16: 1 | bf16: 0 | warp size: 32 | shared memory: 49152 | int dot: 1 | matrix cores: KHR_coopmat +ggml_vulkan: 1 = Intel(R) RaptorLake-S Mobile Graphics Controller (Intel Corporation) | uma: 1 | fp16: 1 | bf16: 0 | warp size: 32 | shared memory: 65536 | int dot: 1 | matrix cores: none +` + + devices := parseLlamaServerDevicesWithNative(output, nativeOutput, []string{"/lib/ollama", "/lib/ollama/vulkan"}, nil) + if len(devices) != 2 { + t.Fatalf("got %d devices, want 2", len(devices)) + } + if !devices[0].Integrated { + t.Fatal("device 0 (Intel iGPU) Integrated = false, want true") + } + if devices[1].Integrated { + t.Fatal("device 1 (NVIDIA dGPU) Integrated = true, want false") + } + }) + t.Run("cuda runtime version", func(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, "libcudart.so.12.8.90"), nil, 0o644); err != nil { @@ -471,10 +498,50 @@ Available devices: want: []bool{false, false, false}, }, { - name: "skips when counts do not line up", + name: "skips when probe finds fewer devices", probed: []vulkanPhysicalDevice{{Name: "AMD Radeon(TM) Graphics", Integrated: true}}, want: []bool{false, false, false}, }, + { + name: "matches subset when probe enumerates extra devices", + devices: []ml.DeviceInfo{ + {DeviceID: ml.DeviceID{ID: "0", Library: "Vulkan"}, Description: "Intel(R) RaptorLake-S Mobile Graphics Controller"}, + {DeviceID: ml.DeviceID{ID: "1", Library: "Vulkan"}, Description: "NVIDIA GeForce RTX 4080 Laptop GPU"}, + }, + probed: []vulkanPhysicalDevice{ + {Name: "NVIDIA GeForce RTX 4080 Laptop GPU", Integrated: false}, + {Name: "Intel(R) RaptorLake-S Mobile Graphics Controller", Integrated: true}, + {Name: "Microsoft Direct3D12 (NVIDIA GeForce RTX 4080 Laptop GPU)", Integrated: false}, + {Name: "Microsoft Direct3D12 (Intel(R) RaptorLake-S Mobile Graphics Controller)", Integrated: true}, + {Name: "llvmpipe (LLVM 17.0.6, 256 bits)", Integrated: false}, + }, + want: []bool{true, false}, + applied: true, + }, + { + name: "skips ambiguous duplicate names with conflicting types", + devices: []ml.DeviceInfo{ + {DeviceID: ml.DeviceID{ID: "0", Library: "Vulkan"}, Description: "AMD Radeon Graphics"}, + }, + probed: []vulkanPhysicalDevice{ + {Name: "AMD Radeon Graphics", Integrated: true}, + {Name: "AMD Radeon Graphics", Integrated: false}, + }, + want: []bool{false}, + }, + { + name: "matches duplicate names with agreeing types", + devices: []ml.DeviceInfo{ + {DeviceID: ml.DeviceID{ID: "0", Library: "Vulkan"}, Description: "AMD Radeon RX 7900 XTX"}, + {DeviceID: ml.DeviceID{ID: "1", Library: "Vulkan"}, Description: "AMD Radeon RX 7900 XTX"}, + }, + probed: []vulkanPhysicalDevice{ + {Name: "AMD Radeon RX 7900 XTX", Integrated: false}, + {Name: "AMD Radeon RX 7900 XTX", Integrated: false}, + }, + want: []bool{false, false}, + applied: true, + }, { name: "overwrites stale classification", devices: []ml.DeviceInfo{ diff --git a/discover/native_probe_test.go b/discover/native_probe_test.go index 3fcff594b..99c810fe9 100644 --- a/discover/native_probe_test.go +++ b/discover/native_probe_test.go @@ -46,7 +46,7 @@ func TestParseLlamaServerDevicesUsesNativeCUDAComputeCapability(t *testing.T) { Available devices: CUDA0: NVIDIA GeForce GTX 1060 6GB (6063 MiB, 5900 MiB free) ` - devices := parseLlamaServerDevicesWithNative(output, []string{"/lib/ollama", "/lib/ollama/cuda_v13"}, []nativeProbeDevice{{ + devices := parseLlamaServerDevicesWithNative(output, "", []string{"/lib/ollama", "/lib/ollama/cuda_v13"}, []nativeProbeDevice{{ Library: "CUDA", Index: 0, IndexMatchesBackend: true, @@ -64,7 +64,7 @@ Available devices: Available devices: CUDA0: NVIDIA GeForce GTX 1060 6GB (6063 MiB, 5900 MiB free) ` - devices = parseLlamaServerDevicesWithNative(output, []string{"/lib/ollama", "/lib/ollama/cuda_v12"}, []nativeProbeDevice{{ + devices = parseLlamaServerDevicesWithNative(output, "", []string{"/lib/ollama", "/lib/ollama/cuda_v12"}, []nativeProbeDevice{{ Library: "CUDA", Index: 0, IndexMatchesBackend: true, @@ -97,7 +97,7 @@ func TestParseLlamaServerDevicesUsesNativeROCmMetadata(t *testing.T) { Available devices: ROCm0: AMD Radeon RX 7600 (8176 MiB, 7900 MiB free) ` - devices := parseLlamaServerDevicesWithNative(output, []string{"/lib/ollama", "/lib/ollama/rocm_v7_2"}, []nativeProbeDevice{{ + devices := parseLlamaServerDevicesWithNative(output, "", []string{"/lib/ollama", "/lib/ollama/rocm_v7_2"}, []nativeProbeDevice{{ Library: "ROCm", Index: 0, IndexMatchesBackend: true, diff --git a/discover/vulkan.go b/discover/vulkan.go index 1e7d50e53..5d481eddc 100644 --- a/discover/vulkan.go +++ b/discover/vulkan.go @@ -95,32 +95,40 @@ func applyWindowsVulkanRefinement(devices []ml.DeviceInfo, probed []vulkanPhysic } } - if len(probed) != len(vulkanIndexes) { - slog.Debug("windows vulkan device refinement skipped: device count mismatch", + if len(probed) < len(vulkanIndexes) { + slog.Debug("windows vulkan device refinement skipped: fewer probed devices than llama-server devices", "llama_server_count", len(vulkanIndexes), "vulkan_count", len(probed)) return false } + // Raw Vulkan enumeration can be a superset of llama-server's device list + // (extra ICDs, D3D12 mapping-layer devices) and the two orders can differ, + // so match by name rather than requiring equal counts or matching indexes. matches := make([]int, len(vulkanIndexes)) - for i := range matches { - matches[i] = -1 - } used := make([]bool, len(probed)) for i, deviceIndex := range vulkanIndexes { + matches[i] = -1 description := devices[deviceIndex].Description for j, probedDevice := range probed { if used[j] || !sameVulkanDeviceName(description, probedDevice.Name) { continue } + if matches[i] >= 0 { + if probed[matches[i]].Integrated != probedDevice.Integrated { + slog.Debug("windows vulkan device refinement skipped: ambiguous device name match", + "index", i, "llama_server_name", description) + return false + } + continue + } matches[i] = j - used[j] = true - break } if matches[i] < 0 { slog.Debug("windows vulkan device refinement skipped: device name mismatch", "index", i, "llama_server_name", description) return false } + used[matches[i]] = true } for i, probedIndex := range matches {