ollama/mlx/memory.go
Daniel Hiltgen 73ee71ddcc
mlxrunner: adapt saved history to memory pressure (#19678)
* mlxrunner: adapt saved history to memory pressure

Retain Metal prefix history using observed memory headroom and execution
workspace rather than a fixed 8 GiB allowance. Release unused buffers and
cold history when system compression grows, preserving reusable paths and
active requests.

Cover pressure reclamation, snapshot ownership, memory observation, and
cacheless request isolation.

* address comments

* address comments
2026-11-19 15:41:42 -08:00

147 lines
3.3 KiB
Go

package mlx
// #include "generated.h"
// #include <stdlib.h>
import "C"
import (
"fmt"
"log/slog"
"strconv"
"unsafe"
)
func (b Byte) String() string {
return strconv.FormatInt(int64(b), 10) + " B"
}
func (b KibiByte) String() string {
return strconv.FormatFloat(float64(b)/(1<<10), 'f', 2, 64) + " KiB"
}
func (b MebiByte) String() string {
return strconv.FormatFloat(float64(b)/(1<<(2*10)), 'f', 2, 64) + " MiB"
}
func (b GibiByte) String() string {
return strconv.FormatFloat(float64(b)/(1<<(3*10)), 'f', 2, 64) + " GiB"
}
func (b TebiByte) String() string {
return strconv.FormatFloat(float64(b)/(1<<(4*10)), 'f', 2, 64) + " TiB"
}
func PrettyBytes(n int) fmt.Stringer {
switch {
case n < 1<<10:
return Byte(n)
case n < 1<<(2*10):
return KibiByte(n)
case n < 1<<(3*10):
return MebiByte(n)
case n < 1<<(4*10):
return GibiByte(n)
default:
return TebiByte(n)
}
}
func ActiveMemory() int {
var active C.size_t
mlxCheck(C.mlx_get_active_memory(&active))
return int(active)
}
func CacheMemory() int {
var cache C.size_t
mlxCheck(C.mlx_get_cache_memory(&cache))
return int(cache)
}
// priorPeak preserves the reporting interval while measuring smaller operations.
// Like other MLX state, it is accessed only on the MLX thread.
var priorPeak int
func PeakMemory() int {
return max(priorPeak, nativePeakMemory())
}
func nativePeakMemory() int {
var peak C.size_t
mlxCheck(C.mlx_get_peak_memory(&peak))
return int(peak)
}
func ResetPeakMemory() {
priorPeak = 0
mlxCheck(C.mlx_reset_peak_memory())
}
// MeasurePeakMemory returns the allocation peak during f while preserving the
// peak since ResetPeakMemory for request logs and other cumulative reporting.
func MeasurePeakMemory(f func()) int {
priorPeak = PeakMemory()
mlxCheck(C.mlx_reset_peak_memory())
f()
return nativePeakMemory()
}
// MaxRecommendedWorkingSetSize returns the device's recommended upper bound
// for resident Metal allocations.
func MaxRecommendedWorkingSetSize() (int, error) {
info := mlxCheck(C.mlx_device_info_new())
if err := mlxError(C.mlx_device_info_get(&info, DefaultDevice().ctx)); err != nil {
return 0, err
}
defer freeDeviceInfo(info)
key := C.CString("max_recommended_working_set_size")
defer C.free(unsafe.Pointer(key))
var size C.size_t
rc := C.mlx_device_info_get_size(&size, info, key)
if err := lastError(); err != nil {
return 0, err
}
if rc != 0 {
// mlx-c reports a missing key with a non-zero return and no message.
return 0, fmt.Errorf("mlx: no max_recommended_working_set_size in device info")
}
return int(size), nil
}
// SetWiredLimit sets the maximum amount of Metal memory MLX keeps resident and
// returns the previous limit.
func SetWiredLimit(limit int) (int, error) {
if limit < 0 {
return 0, fmt.Errorf("mlx: wired limit must be non-negative")
}
var previous C.size_t
if err := mlxError(C.mlx_set_wired_limit(&previous, C.size_t(limit))); err != nil {
return 0, err
}
return int(previous), nil
}
type Memory struct{}
func (Memory) LogValue() slog.Value {
return slog.GroupValue(
slog.Any("active", PrettyBytes(ActiveMemory())),
slog.Any("cache", PrettyBytes(CacheMemory())),
slog.Any("peak", PrettyBytes(PeakMemory())),
)
}
type (
Byte int
KibiByte int
MebiByte int
GibiByte int
TebiByte int
)
func ClearCache() {
mlxCheck(C.mlx_clear_cache())
}