ollama/mlxrunner/nn/linear.go
Daniel Hiltgen ab9295d03c
mlx: support concurrent generation (#19197)
* mlx: batch requests through a shared scheduler

Use one request queue and scheduler for generation, decisions, embeddings
and speech, including parallelism one. Share compatible model operations
while keeping request caches, sampling state and response streams independent.

Interleave prompt chunks with decoding and retain speculative decoding when
generation runs alone. Cover isolation, cancellation, slow readers, cache
reuse and mixed requests with unit and integration tests. Remove the stale
llama-server architecture restriction on parallel requests.

* mlx: prevent cache snapshots from retaining batch peers

Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row.

Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers.

Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks.

* address comments
2026-11-10 13:24:12 -08:00

117 lines
3.3 KiB
Go

package nn
import "github.com/ollama/ollama/mlx"
// LinearLayer is an interface for linear layers (both regular and quantized).
type LinearLayer interface {
Forward(x *mlx.Array) *mlx.Array
OutputDim() int32
}
// Linear applies an affine transformation: y = x @ W.T + b
type Linear struct {
Weight *mlx.Array
Bias *mlx.Array
}
func NewLinear(weight *mlx.Array, bias *mlx.Array) *Linear {
if bias != nil && bias.DType() != weight.DType() {
bias = bias.AsType(weight.DType())
}
return &Linear{Weight: weight, Bias: bias}
}
func (l *Linear) Forward(x *mlx.Array) *mlx.Array {
w := l.Weight.Transpose(1, 0)
if l.Bias != nil {
return l.Bias.Addmm(x, w, 1.0, 1.0)
}
return x.Matmul(w)
}
func (l *Linear) OutputDim() int32 {
return int32(l.Weight.Dim(0))
}
// QuantizedLinear applies an affine transformation using quantized weights.
type QuantizedLinear struct {
Weight *mlx.Array // Quantized weight data
Scales *mlx.Array // Scale factors for dequantization
QBiases *mlx.Array // Quantization biases (nil for nvfp4)
Bias *mlx.Array // Layer bias [output_dims] or nil
GlobalScale *mlx.Array // Per-tensor or per-row global scale for double-scale nvfp4 (nil for standard)
GroupSize int
Bits int
Mode string
}
func NewQuantizedLinear(weight *mlx.Array, bias *mlx.Array, groupSize, bits int, mode string) *QuantizedLinear {
qw, scales, qbiases := mlx.Quantize(weight, groupSize, bits, mode)
if qbiases != nil {
mlx.Eval(qw, scales, qbiases)
} else {
mlx.Eval(qw, scales)
}
if bias != nil && bias.DType() != weight.DType() {
bias = bias.AsType(weight.DType())
}
return &QuantizedLinear{
Weight: qw,
Scales: scales,
QBiases: qbiases,
Bias: bias,
GroupSize: groupSize,
Bits: bits,
Mode: mode,
}
}
func (ql *QuantizedLinear) Forward(x *mlx.Array) *mlx.Array {
out := ql.matmul(x, ql.GlobalScale)
if ql.Bias != nil {
bias := ql.Bias
if bias.DType() != out.DType() {
bias = bias.AsType(out.DType())
}
out = out.Add(bias)
}
return out
}
func (ql *QuantizedLinear) matmul(x, globalScale *mlx.Array) *mlx.Array {
var shape []int
if x.NumDims() == 3 && x.Dim(0) > 1 {
// All requests use the same weight matrix. Present their tokens as
// matrix rows so MLX can reuse weights instead of dispatching a
// separate batched matvec for each request.
shape = x.Dims()
x = x.Reshape(-1, shape[2])
}
out := mlx.QuantizedMatmul(x, ql.Weight, ql.Scales, ql.QBiases, true,
ql.GroupSize, ql.Bits, ql.Mode, globalScale)
if shape != nil {
shape[2] = out.Dim(-1)
out = out.Reshape(shape...)
}
return out
}
// SwiGLU applies gate and up projections followed by a SwiGLU activation.
// Quantized projections without bias defer their global scales so the scale
// and activation operations can be fused.
func SwiGLU(gate, up LinearLayer, x *mlx.Array) *mlx.Array {
gateOut, gateScale := forwardDeferScale(gate, x)
upOut, upScale := forwardDeferScale(up, x)
return mlx.SwiGLUScaled(gateOut, gateScale, upOut, upScale)
}
func forwardDeferScale(l LinearLayer, x *mlx.Array) (out, pending *mlx.Array) {
if ql, ok := l.(*QuantizedLinear); ok && ql.GlobalScale != nil && ql.Bias == nil {
return ql.matmul(x, nil), ql.GlobalScale
}
return l.Forward(x), nil
}
func (ql *QuantizedLinear) OutputDim() int32 {
return int32(ql.Weight.Dim(0))
}