diff --git a/llm/llama_server.go b/llm/llama_server.go index b58b285e2..0991bf377 100644 --- a/llm/llama_server.go +++ b/llm/llama_server.go @@ -460,6 +460,12 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec. if err = cmd.Start(); err != nil { return nil, 0, err } + + if err = AddRunnerToJob(cmd.Process.Pid); err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return nil, 0, fmt.Errorf("assign llama-server to runner job: %w", err) + } return cmd, port, nil } diff --git a/llm/runner_job_other.go b/llm/runner_job_other.go new file mode 100644 index 000000000..9ebe27ed3 --- /dev/null +++ b/llm/runner_job_other.go @@ -0,0 +1,8 @@ +//go:build !windows + +package llm + +// AddRunnerToJob is a no-op on platforms without Windows Job Objects. +func AddRunnerToJob(int) error { + return nil +} diff --git a/llm/runner_job_windows.go b/llm/runner_job_windows.go new file mode 100644 index 000000000..d09f5922f --- /dev/null +++ b/llm/runner_job_windows.go @@ -0,0 +1,64 @@ +package llm + +import ( + "sync" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + // runnerJobMu guards runnerJob initialization and process assignment. + runnerJobMu sync.Mutex + runnerJob windows.Handle +) + +// AddRunnerToJob assigns a runner process to Ollama's process-lifetime Job Object. +// A failed Job initialization is retried by the next call. +func AddRunnerToJob(pid int) error { + runnerJobMu.Lock() + defer runnerJobMu.Unlock() + + if runnerJob == 0 { + job, err := createRunnerJob() + if err != nil { + return err + } + runnerJob = job + } + + return assignProcessToJob(runnerJob, pid) +} + +func createRunnerJob() (windows.Handle, error) { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return 0, err + } + + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} + info.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject( + job, + windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)), + ); err != nil { + _ = windows.CloseHandle(job) + return 0, err + } + + // Keep the job open for the lifetime of ollama. Windows closes the handle + // and terminates every assigned runner when ollama exits. + return job, nil +} + +func assignProcessToJob(job windows.Handle, pid int) error { + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(pid)) + if err != nil { + return err + } + defer windows.CloseHandle(process) + + return windows.AssignProcessToJobObject(job, process) +} diff --git a/llm/runner_job_windows_test.go b/llm/runner_job_windows_test.go new file mode 100644 index 000000000..04e200893 --- /dev/null +++ b/llm/runner_job_windows_test.go @@ -0,0 +1,78 @@ +package llm + +import ( + "os" + "os/exec" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +const runnerJobHelperEnv = "OLLAMA_TEST_RUNNER_JOB_HELPER" + +func TestRunnerJobKillsOnlyAssignedProcess(t *testing.T) { + owned := startRunnerJobHelper(t) + unrelated := startRunnerJobHelper(t) + + job, err := createRunnerJob() + if err != nil { + t.Fatalf("createRunnerJob(): %v", err) + } + if err := assignProcessToJob(job, owned.Process.Pid); err != nil { + _ = windows.CloseHandle(job) + t.Fatalf("assignProcessToJob(%d): %v", owned.Process.Pid, err) + } + if err := windows.CloseHandle(job); err != nil { + t.Fatalf("close job: %v", err) + } + + wait := make(chan error, 1) + go func() { + wait <- owned.Wait() + }() + select { + case <-wait: + case <-time.After(5 * time.Second): + t.Fatal("assigned process remained alive after its job was closed") + } + + process, err := windows.OpenProcess(windows.SYNCHRONIZE, false, uint32(unrelated.Process.Pid)) + if err != nil { + t.Fatalf("open unrelated process %d: %v", unrelated.Process.Pid, err) + } + defer windows.CloseHandle(process) + + status, err := windows.WaitForSingleObject(process, 0) + if err != nil { + t.Fatalf("query unrelated process %d: %v", unrelated.Process.Pid, err) + } + if status != uint32(windows.WAIT_TIMEOUT) { + t.Fatalf("unrelated process wait status = %d, want WAIT_TIMEOUT", status) + } +} + +func TestRunnerJobHelper(t *testing.T) { + if os.Getenv(runnerJobHelperEnv) == "" { + return + } + + time.Sleep(time.Hour) +} + +func startRunnerJobHelper(t *testing.T) *exec.Cmd { + t.Helper() + + cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestRunnerJobHelper$") + cmd.Env = append(os.Environ(), runnerJobHelperEnv+"=1") + if err := cmd.Start(); err != nil { + t.Fatalf("start helper process: %v", err) + } + t.Cleanup(func() { + if cmd.ProcessState == nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }) + return cmd +} diff --git a/mlxrunner/client.go b/mlxrunner/client.go index 195c4d4fc..6c92727b8 100644 --- a/mlxrunner/client.go +++ b/mlxrunner/client.go @@ -518,6 +518,11 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D if err := cmd.Start(); err != nil { return nil, fmt.Errorf("failed to start mlx runner: %w", err) } + if err := llm.AddRunnerToJob(cmd.Process.Pid); err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return nil, fmt.Errorf("assign mlx runner to runner job: %w", err) + } c.cmd = cmd // Reap subprocess when it exits