mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
* mlx: batch requests through a shared scheduler Use one request queue and scheduler for generation, decisions, embeddings and speech, including parallelism one. Share compatible model operations while keeping request caches, sampling state and response streams independent. Interleave prompt chunks with decoding and retain speculative decoding when generation runs alone. Cover isolation, cancellation, slow readers, cache reuse and mixed requests with unit and integration tests. Remove the stale llama-server architecture restriction on parallel requests. * mlx: prevent cache snapshots from retaining batch peers Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row. Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers. Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks. * address comments
103 lines
3 KiB
Go
103 lines
3 KiB
Go
package batch
|
|
|
|
import "github.com/ollama/ollama/mlx"
|
|
|
|
// Batch is the per-forward-pass input handed to a model.
|
|
type Batch struct {
|
|
// InputIDs is the input token IDs for this forward pass, shape (B, L).
|
|
InputIDs *mlx.Array
|
|
|
|
// SeqOffsets gives each row's current position within its sequence —
|
|
// where the chunk in InputIDs starts. Length equals the batch dimension
|
|
// of InputIDs.
|
|
SeqOffsets []int32
|
|
|
|
// SeqQueryLens is each row's real query length in this forward. Values
|
|
// less than L mean the row's tail is padding that must be masked out.
|
|
// Length equals the batch dimension of InputIDs.
|
|
SeqQueryLens []int32
|
|
|
|
// Hidden is the draft-conditioning state for a draft model's forward.
|
|
// It is nil for ordinary forward passes.
|
|
Hidden *mlx.Array
|
|
|
|
// Media lists a row's media items, on prefill forwards and draft
|
|
// forwards that embed prompt tokens; items outside the query range
|
|
// ride featureless. Nil at decode and for text-only requests.
|
|
Media []MediaItem
|
|
|
|
// Layout carries each row's opaque model-owned metadata,
|
|
// identical on every forward of the request; the runner never reads
|
|
// it. Nil entries derive nothing from layout.
|
|
Layout []any
|
|
|
|
// Memo is per-forward memoization used to cache results, such as masks,
|
|
// which are often the same across layers.
|
|
Memo Memo
|
|
}
|
|
|
|
type rowKey int
|
|
|
|
// Row returns a single row with its own per-forward memo. Repeated calls reuse
|
|
// the same view, so cache mask dispatch remains memoized across model layers.
|
|
func (b *Batch) Row(i int) *Batch {
|
|
if v, ok := b.Memo.Get(rowKey(i)); ok {
|
|
return v.(*Batch)
|
|
}
|
|
row := &Batch{
|
|
InputIDs: b.InputIDs.Slice(mlx.Slice(i, i+1), mlx.Slice()),
|
|
SeqOffsets: b.SeqOffsets[i : i+1],
|
|
SeqQueryLens: b.SeqQueryLens[i : i+1],
|
|
}
|
|
if b.Hidden != nil {
|
|
row.Hidden = b.Hidden.Slice(mlx.Slice(i, i+1), mlx.Slice(), mlx.Slice())
|
|
}
|
|
if len(b.Layout) > 0 {
|
|
row.Layout = b.Layout[i : i+1]
|
|
}
|
|
for _, item := range b.Media {
|
|
if item.Seq == i {
|
|
item.Seq = 0
|
|
row.Media = append(row.Media, item)
|
|
}
|
|
}
|
|
b.Memo.Put(rowKey(i), row)
|
|
return row
|
|
}
|
|
|
|
// MediaItem is one media occurrence in a row's sequence. The runner
|
|
// knows only where the expansion was spliced; which positions bear
|
|
// features is the model's, derived from Pos and Opaque.
|
|
type MediaItem struct {
|
|
// Seq is the batch row the item belongs to.
|
|
Seq int
|
|
|
|
// Pos is the absolute sequence position of the expansion's first token.
|
|
Pos int
|
|
|
|
// Features is the item's whole feature-row array, attached only while
|
|
// the item's token range overlaps this forward's query range.
|
|
Features *mlx.Array
|
|
|
|
// Opaque is the item's PreparedMedia.Opaque, round-tripped untouched.
|
|
Opaque any
|
|
}
|
|
|
|
type Memo struct {
|
|
entries map[any]any
|
|
}
|
|
|
|
// Get returns the memoized value for key and true if present, or nil
|
|
// and false otherwise.
|
|
func (m *Memo) Get(key any) (any, bool) {
|
|
v, ok := m.entries[key]
|
|
return v, ok
|
|
}
|
|
|
|
// Put stores value under key, allocating on first use.
|
|
func (m *Memo) Put(key, value any) {
|
|
if m.entries == nil {
|
|
m.entries = map[any]any{}
|
|
}
|
|
m.entries[key] = value
|
|
}
|