mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
* mlx: batch requests through a shared scheduler Use one request queue and scheduler for generation, decisions, embeddings and speech, including parallelism one. Share compatible model operations while keeping request caches, sampling state and response streams independent. Interleave prompt chunks with decoding and retain speculative decoding when generation runs alone. Cover isolation, cancellation, slow readers, cache reuse and mixed requests with unit and integration tests. Remove the stale llama-server architecture restriction on parallel requests. * mlx: prevent cache snapshots from retaining batch peers Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row. Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers. Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks. * address comments
117 lines
3.3 KiB
Go
117 lines
3.3 KiB
Go
package nn
|
|
|
|
import "github.com/ollama/ollama/mlx"
|
|
|
|
// LinearLayer is an interface for linear layers (both regular and quantized).
|
|
type LinearLayer interface {
|
|
Forward(x *mlx.Array) *mlx.Array
|
|
OutputDim() int32
|
|
}
|
|
|
|
// Linear applies an affine transformation: y = x @ W.T + b
|
|
type Linear struct {
|
|
Weight *mlx.Array
|
|
Bias *mlx.Array
|
|
}
|
|
|
|
func NewLinear(weight *mlx.Array, bias *mlx.Array) *Linear {
|
|
if bias != nil && bias.DType() != weight.DType() {
|
|
bias = bias.AsType(weight.DType())
|
|
}
|
|
return &Linear{Weight: weight, Bias: bias}
|
|
}
|
|
|
|
func (l *Linear) Forward(x *mlx.Array) *mlx.Array {
|
|
w := l.Weight.Transpose(1, 0)
|
|
if l.Bias != nil {
|
|
return l.Bias.Addmm(x, w, 1.0, 1.0)
|
|
}
|
|
return x.Matmul(w)
|
|
}
|
|
|
|
func (l *Linear) OutputDim() int32 {
|
|
return int32(l.Weight.Dim(0))
|
|
}
|
|
|
|
// QuantizedLinear applies an affine transformation using quantized weights.
|
|
type QuantizedLinear struct {
|
|
Weight *mlx.Array // Quantized weight data
|
|
Scales *mlx.Array // Scale factors for dequantization
|
|
QBiases *mlx.Array // Quantization biases (nil for nvfp4)
|
|
Bias *mlx.Array // Layer bias [output_dims] or nil
|
|
GlobalScale *mlx.Array // Per-tensor or per-row global scale for double-scale nvfp4 (nil for standard)
|
|
GroupSize int
|
|
Bits int
|
|
Mode string
|
|
}
|
|
|
|
func NewQuantizedLinear(weight *mlx.Array, bias *mlx.Array, groupSize, bits int, mode string) *QuantizedLinear {
|
|
qw, scales, qbiases := mlx.Quantize(weight, groupSize, bits, mode)
|
|
if qbiases != nil {
|
|
mlx.Eval(qw, scales, qbiases)
|
|
} else {
|
|
mlx.Eval(qw, scales)
|
|
}
|
|
if bias != nil && bias.DType() != weight.DType() {
|
|
bias = bias.AsType(weight.DType())
|
|
}
|
|
return &QuantizedLinear{
|
|
Weight: qw,
|
|
Scales: scales,
|
|
QBiases: qbiases,
|
|
Bias: bias,
|
|
GroupSize: groupSize,
|
|
Bits: bits,
|
|
Mode: mode,
|
|
}
|
|
}
|
|
|
|
func (ql *QuantizedLinear) Forward(x *mlx.Array) *mlx.Array {
|
|
out := ql.matmul(x, ql.GlobalScale)
|
|
if ql.Bias != nil {
|
|
bias := ql.Bias
|
|
if bias.DType() != out.DType() {
|
|
bias = bias.AsType(out.DType())
|
|
}
|
|
out = out.Add(bias)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (ql *QuantizedLinear) matmul(x, globalScale *mlx.Array) *mlx.Array {
|
|
var shape []int
|
|
if x.NumDims() == 3 && x.Dim(0) > 1 {
|
|
// All requests use the same weight matrix. Present their tokens as
|
|
// matrix rows so MLX can reuse weights instead of dispatching a
|
|
// separate batched matvec for each request.
|
|
shape = x.Dims()
|
|
x = x.Reshape(-1, shape[2])
|
|
}
|
|
out := mlx.QuantizedMatmul(x, ql.Weight, ql.Scales, ql.QBiases, true,
|
|
ql.GroupSize, ql.Bits, ql.Mode, globalScale)
|
|
if shape != nil {
|
|
shape[2] = out.Dim(-1)
|
|
out = out.Reshape(shape...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// SwiGLU applies gate and up projections followed by a SwiGLU activation.
|
|
// Quantized projections without bias defer their global scales so the scale
|
|
// and activation operations can be fused.
|
|
func SwiGLU(gate, up LinearLayer, x *mlx.Array) *mlx.Array {
|
|
gateOut, gateScale := forwardDeferScale(gate, x)
|
|
upOut, upScale := forwardDeferScale(up, x)
|
|
return mlx.SwiGLUScaled(gateOut, gateScale, upOut, upScale)
|
|
}
|
|
|
|
func forwardDeferScale(l LinearLayer, x *mlx.Array) (out, pending *mlx.Array) {
|
|
if ql, ok := l.(*QuantizedLinear); ok && ql.GlobalScale != nil && ql.Bias == nil {
|
|
return ql.matmul(x, nil), ql.GlobalScale
|
|
}
|
|
return l.Forward(x), nil
|
|
}
|
|
|
|
func (ql *QuantizedLinear) OutputDim() int32 {
|
|
return int32(ql.Weight.Dim(0))
|
|
}
|