ollama/mlxrunner/speech.go
Daniel Hiltgen ab9295d03c
mlx: support concurrent generation (#19197)
* mlx: batch requests through a shared scheduler

Use one request queue and scheduler for generation, decisions, embeddings
and speech, including parallelism one. Share compatible model operations
while keeping request caches, sampling state and response streams independent.

Interleave prompt chunks with decoding and retain speculative decoding when
generation runs alone. Cover isolation, cancellation, slow readers, cache
reuse and mixed requests with unit and integration tests. Remove the stale
llama-server architecture restriction on parallel requests.

* mlx: prevent cache snapshots from retaining batch peers

Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row.

Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers.

Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks.

* address comments
2026-11-10 13:24:12 -08:00

236 lines
5.5 KiB
Go

package mlxrunner
import (
"context"
"encoding/binary"
"encoding/json"
"errors"
"net/http"
"sync"
"time"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/llm"
"github.com/ollama/ollama/mlxrunner/model"
"github.com/ollama/ollama/mlxrunner/model/audio"
)
// speechHandler streams 24 kHz 16-bit PCM as the model generates it.
func (r *Runner) speechHandler(w http.ResponseWriter, req *http.Request) {
_, ok := r.Model.(model.SpeechModel)
if !ok {
http.Error(w, "this model does not support speech", http.StatusNotImplemented)
return
}
var input llm.SpeechRequest
if err := json.NewDecoder(req.Body).Decode(&input); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Generation queues audio for a separate writer instead of waiting on the
// client, so a client reading at playback speed does not hold the MLX worker.
ctx, cancel := context.WithCancel(req.Context())
defer cancel()
queue := &audioQueue{ready: make(chan struct{}, 1)}
var streaming bool
var writeErr error
written := make(chan struct{})
go func() {
defer close(written)
streaming, writeErr = writeAudio(w, queue)
if writeErr != nil {
cancel()
}
}()
// The writer must not outlive the handler, even if generation panics.
defer func() { queue.close(); <-written }()
speech := &speechRequest{ctx: ctx, input: input, queue: queue, done: make(chan error, 1)}
var err error
select {
case <-ctx.Done():
err = ctx.Err()
case r.Requests <- speech:
// The worker must release request state before the handler exits.
err = <-speech.done
}
queue.close()
<-written
if err == nil {
err = writeErr
}
switch {
case err == nil:
case streaming:
// Truncate the stream so the client sees the failure.
panic(http.ErrAbortHandler)
default:
status := http.StatusInternalServerError
var statusErr api.StatusError
if errors.As(err, &statusErr) {
status = statusErr.StatusCode
}
http.Error(w, err.Error(), status)
}
}
// writeAudio streams queued audio until the queue is closed and reports
// whether any audio was written.
func writeAudio(w http.ResponseWriter, queue *audioQueue) (streaming bool, err error) {
controller := http.NewResponseController(w)
defer controller.SetWriteDeadline(time.Time{})
for {
chunks, ok := queue.next()
if !ok {
return streaming, nil
}
for _, pcm := range chunks {
// A stalled reader must not hold the connection forever.
if err := controller.SetWriteDeadline(time.Now().Add(30 * time.Second)); err != nil {
return streaming, err
}
if !streaming {
w.Header().Set("Content-Type", "audio/pcm")
streaming = true
}
if _, err := w.Write(pcm); err != nil {
return streaming, err
}
}
if err := controller.Flush(); err != nil {
return streaming, err
}
}
}
// audioQueue hands audio from generation to the writer without ever blocking
// generation.
type audioQueue struct {
mu sync.Mutex
pending [][]byte
closed bool
ready chan struct{}
}
func (q *audioQueue) push(pcm []byte) {
q.mu.Lock()
q.pending = append(q.pending, pcm)
q.mu.Unlock()
q.wake()
}
func (q *audioQueue) close() {
q.mu.Lock()
q.closed = true
q.mu.Unlock()
q.wake()
}
func (q *audioQueue) wake() {
select {
case q.ready <- struct{}{}:
default:
}
}
// next waits for queued audio. It returns false once the queue is closed and
// empty.
func (q *audioQueue) next() ([][]byte, bool) {
for {
q.mu.Lock()
pending, closed := q.pending, q.closed
q.pending = nil
q.mu.Unlock()
if len(pending) > 0 || closed {
return pending, len(pending) > 0
}
<-q.ready
}
}
func pcm16(samples []float32) []byte {
b := make([]byte, 2*len(samples))
for i, s := range samples {
binary.LittleEndian.PutUint16(b[2*i:], uint16(audio.PCM16(s)))
}
return b
}
func (s *batchScheduler) admitSpeech(request *speechRequest) {
s.keepScratch = false
if err := request.ctx.Err(); err != nil {
request.reject(err)
return
}
state, err := s.r.Model.(model.SpeechModel).NewSpeech(request.ctx, request.input)
if err != nil {
request.done <- err
return
}
request.state = state
for i, slot := range s.slots {
if slot != nil && slot.busy() {
continue
}
if slot == nil {
slot = &batchSlot{id: i}
s.slots[i] = slot
}
slot.speech = request
return
}
panic("scheduler admitted speech without a free slot")
}
func (s *batchScheduler) finishSpeech(slot *batchSlot, err error) {
request := slot.speech
request.state.Close()
slot.speech = nil
request.done <- err
}
// decodeSpeech calls one model batch operation, not one request loop per slot.
func (s *batchScheduler) decodeSpeech() bool {
var slots []*batchSlot
var states []model.SpeechState
progressed := false
for _, slot := range s.slots {
if slot == nil || slot.speech == nil {
continue
}
if err := slot.speech.ctx.Err(); err != nil {
s.finishSpeech(slot, err)
progressed = true
continue
}
slots, states = append(slots, slot), append(states, slot.speech.state)
}
if len(slots) == 0 {
return progressed
}
outputs, err := s.r.Model.(model.SpeechModel).DecodeSpeech(states)
if err != nil {
for _, slot := range slots {
s.finishSpeech(slot, err)
}
return true
}
if len(outputs) != len(slots) {
panic("speech output rows do not match requests")
}
for i, slot := range slots {
out := outputs[i]
if err := slot.speech.ctx.Err(); err != nil {
s.finishSpeech(slot, err)
continue
}
if len(out.Samples) > 0 {
slot.speech.queue.push(pcm16(out.Samples))
}
if out.Done || out.Err != nil {
s.finishSpeech(slot, out.Err)
}
}
return true
}