mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
* mlxrunner: add Inkling 2 Mini * mlx: fuse Inkling 2 Mini residual mixing on Metal Fuse RMS normalization and query scoring in one kernel. Apply the softmax weights in a second kernel to avoid storing per-query products. * create: expose Inkling 2 Mini context length in model metadata Copy model_max_length to max_position_embeddings during import so ollama show reports the checkpoint's context length. Reject configs that specify conflicting limits. * mlxrunner: reduce Metal dispatches in Inkling 2 Mini decode Reuse the prefill mask shader to compute relative position bias in one dispatch for single-query attention. On M3 Ultra with Metal 3, compute the loop controller's gain and forget projections in one matrix multiply.
348 lines
12 KiB
Go
348 lines
12 KiB
Go
//go:build integration
|
|
|
|
package integration
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ollama/ollama/types/model"
|
|
|
|
"github.com/ollama/ollama/api"
|
|
"github.com/ollama/ollama/envconfig"
|
|
)
|
|
|
|
// Inkling can spend most of a request's budget reasoning. Give its longer
|
|
// requests their own deadline so earlier cases cannot consume it.
|
|
func inklingBatchContext(t *testing.T, ctx context.Context, info *api.ShowResponse, timeout time.Duration) context.Context {
|
|
if info.Renderer != "inkling2" {
|
|
return ctx
|
|
}
|
|
ctx, cancel := context.WithTimeout(t.Context(), timeout)
|
|
t.Cleanup(cancel)
|
|
return ctx
|
|
}
|
|
|
|
// runMLXBatching adds cache reuse and slot lifecycle checks to the release
|
|
// concurrency group. Capacity one exercises the same requests serially.
|
|
func runMLXBatching(t *testing.T, modelName string) {
|
|
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
|
|
defer cancel()
|
|
client, _, cleanup := InitServerConnection(ctx, t)
|
|
defer cleanup()
|
|
pullOrSkip(ctx, t, client, modelName)
|
|
info, err := client.Show(ctx, &api.ShowRequest{Model: modelName})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if info.Details.Format != "safetensors" {
|
|
t.Skip("cache reuse checks require an MLX model")
|
|
}
|
|
request := func(prompt string, n int) api.GenerateRequest {
|
|
return api.GenerateRequest{
|
|
Model: modelName, Runner: "mlx", Prompt: prompt,
|
|
Think: &api.ThinkValue{Value: false}, KeepAlive: &api.Duration{Duration: time.Minute},
|
|
Options: map[string]any{"num_predict": n, "temperature": 0, "num_ctx": 4096},
|
|
}
|
|
}
|
|
preloadConcurrentModel(ctx, t, client, request("", 128))
|
|
cities := []string{"Paris", "Tokyo", "Rome", "Madrid", "London", "Berlin", "Ottawa", "Cairo"}
|
|
countries := []string{"France", "Japan", "Italy", "Spain", "the United Kingdom", "Germany", "Canada", "Egypt"}
|
|
requests := make([]api.GenerateRequest, len(cities))
|
|
for i := range requests {
|
|
requests[i] = request("What is the capital of "+countries[i]+"? Answer with just the city name.", 128)
|
|
}
|
|
t.Run("isolation", func(t *testing.T) {
|
|
ctx := inklingBatchContext(t, ctx, info, 5*time.Minute)
|
|
type result struct {
|
|
text string
|
|
final api.GenerateResponse
|
|
err error
|
|
}
|
|
for range 2 {
|
|
results := make(chan result, len(requests))
|
|
start := make(chan struct{})
|
|
for i, req := range requests {
|
|
go func() {
|
|
<-start
|
|
r := result{}
|
|
r.err = client.Generate(ctx, &req, func(chunk api.GenerateResponse) error {
|
|
r.text += chunk.Response
|
|
if chunk.Done {
|
|
r.final = chunk
|
|
}
|
|
return nil
|
|
})
|
|
if !strings.Contains(r.text, cities[i]) {
|
|
r.err = fmt.Errorf("want %s, got %q (%v)", cities[i], r.text, r.err)
|
|
}
|
|
results <- r
|
|
}()
|
|
}
|
|
close(start)
|
|
for range requests {
|
|
r := <-results
|
|
if r.err != nil || !r.final.Done {
|
|
t.Fatalf("response: %+v, error: %v", r, r.err)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
t.Run("reuse", func(t *testing.T) {
|
|
ctx := inklingBatchContext(t, ctx, info, 5*time.Minute)
|
|
// When every slot is free, a repeat can select its cached prefix. A
|
|
// concurrent repeat may instead land on another available slot.
|
|
for _, req := range requests {
|
|
for round := range 2 {
|
|
var final api.GenerateResponse
|
|
err := client.Generate(ctx, &req, func(chunk api.GenerateResponse) error {
|
|
if chunk.Done {
|
|
final = chunk
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil || !final.Done {
|
|
t.Fatalf("repeat response: %+v, %v", final, err)
|
|
}
|
|
if round == 1 {
|
|
cached := final.PromptEvalCachedCount
|
|
if cached == nil {
|
|
t.Fatal("repeat did not report cached tokens")
|
|
}
|
|
if *cached < final.PromptEvalCount-8 {
|
|
t.Fatalf("repeat reused %d of %d prompt tokens", *cached, final.PromptEvalCount)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
storyBudget := 256
|
|
if info.Renderer == "inkling2" {
|
|
storyBudget = 512
|
|
}
|
|
for _, tc := range []struct {
|
|
name string
|
|
cancel bool
|
|
}{{"cancel", true}, {"finish", false}} {
|
|
t.Run("join_and_leave/"+tc.name, func(t *testing.T) {
|
|
ctx := inklingBatchContext(t, ctx, info, 5*time.Minute)
|
|
longCtx, stop := context.WithCancel(ctx)
|
|
defer stop()
|
|
first := make(chan struct{}, 1)
|
|
finished := make(chan error, 1)
|
|
var final api.GenerateResponse
|
|
long := request("Write a detailed 2000-word story about a lighthouse keeper. Start the story immediately and keep writing.", storyBudget)
|
|
go func() {
|
|
finished <- client.Generate(longCtx, &long, func(chunk api.GenerateResponse) error {
|
|
if chunk.Response != "" || chunk.Thinking != "" {
|
|
select {
|
|
case first <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
if chunk.Done {
|
|
final = chunk
|
|
}
|
|
return nil
|
|
})
|
|
}()
|
|
select {
|
|
case <-first:
|
|
case err := <-finished:
|
|
t.Fatalf("long request ended before output: %v", err)
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
// Capacity one queues the peer; cancellation still frees the slot.
|
|
if tc.cancel && envconfig.NumParallel() <= 1 {
|
|
stop()
|
|
}
|
|
short := requests[1]
|
|
var text string
|
|
err := client.Generate(ctx, &short, func(chunk api.GenerateResponse) error { text += chunk.Response; return nil })
|
|
if err != nil || !strings.Contains(text, "Tokyo") {
|
|
t.Fatalf("joining request: %q, %v", text, err)
|
|
}
|
|
if envconfig.NumParallel() > 1 {
|
|
select {
|
|
case err := <-finished:
|
|
t.Fatalf("long request finished before short joining request: %v", err)
|
|
default:
|
|
}
|
|
}
|
|
if tc.cancel {
|
|
stop()
|
|
}
|
|
err = <-finished
|
|
if tc.cancel {
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("cancel: %v", err)
|
|
}
|
|
} else if err != nil || !final.Done || final.EvalCount != storyBudget {
|
|
t.Fatalf("surviving request: %+v, %v", final, err)
|
|
}
|
|
text = ""
|
|
err = client.Generate(ctx, &requests[2], func(chunk api.GenerateResponse) error { text += chunk.Response; return nil })
|
|
if err != nil || !strings.Contains(text, "Rome") {
|
|
t.Fatalf("slot reuse: %q, %v", text, err)
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("independent_grammar", func(t *testing.T) {
|
|
ctx := inklingBatchContext(t, ctx, info, 5*time.Minute)
|
|
errs := make(chan error, 2)
|
|
for _, city := range cities[:2] {
|
|
go func() {
|
|
req := request("Return the city in JSON.", 32)
|
|
req.Format = []byte(fmt.Sprintf(`{"type":"object","properties":{"city":{"type":"string","enum":[%q]}},"required":["city"],"additionalProperties":false}`, city))
|
|
var text string
|
|
err := client.Generate(ctx, &req, func(chunk api.GenerateResponse) error { text += chunk.Response; return nil })
|
|
if err == nil && !strings.Contains(text, city) {
|
|
err = fmt.Errorf("grammar want %s, got %q", city, text)
|
|
}
|
|
errs <- err
|
|
}()
|
|
}
|
|
for range 2 {
|
|
if err := <-errs; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestMLXBatchingVision mixes text and two different images at the same prompt
|
|
// position. Repeats exercise both cached image identity and per-request layout.
|
|
func TestMLXBatchingVision(t *testing.T) {
|
|
if testModel == "" {
|
|
t.Skip("set OLLAMA_TEST_MODEL to a published MLX vision model")
|
|
}
|
|
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
|
|
defer cancel()
|
|
client, _, cleanup := InitServerConnection(ctx, t)
|
|
defer cleanup()
|
|
info, err := client.Show(ctx, &api.ShowRequest{Model: testModel})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Contains(info.Capabilities, model.CapabilityVision) {
|
|
t.Skip("model has no vision capability")
|
|
}
|
|
answerBudget := 128
|
|
if info.Renderer == "inkling2" {
|
|
answerBudget = 1024
|
|
}
|
|
road, desk, _ := decodeTestImages(t)
|
|
cases := []struct {
|
|
image api.ImageData
|
|
prompt, want string
|
|
}{
|
|
{road, "Is this scene crossing a road or working at a desk? Begin with CROSSING or WORKING, then describe the image in detail.", "crossing"},
|
|
{desk, "Is this scene crossing a road or working at a desk? Begin with CROSSING or WORKING, then describe the image in detail.", "working"},
|
|
{nil, "What is the capital of Japan? Begin with the city name and then describe the city's history in detail.", "tokyo"},
|
|
}
|
|
for round := range 2 {
|
|
roundCtx := inklingBatchContext(t, ctx, info, 8*time.Minute)
|
|
start := make(chan struct{})
|
|
results := make(chan error, len(cases))
|
|
for _, tc := range cases {
|
|
go func() {
|
|
<-start
|
|
req := api.GenerateRequest{Model: testModel, Runner: "mlx", Prompt: tc.prompt, Think: &api.ThinkValue{Value: false}, KeepAlive: &api.Duration{Duration: time.Minute}, Options: map[string]any{"num_predict": answerBudget, "num_ctx": 4096, "temperature": 0}}
|
|
if tc.image != nil {
|
|
req.Images = []api.ImageData{tc.image}
|
|
}
|
|
var text string
|
|
var final api.GenerateResponse
|
|
err := client.Generate(roundCtx, &req, func(chunk api.GenerateResponse) error {
|
|
text += chunk.Response
|
|
if chunk.Done {
|
|
final = chunk
|
|
}
|
|
return nil
|
|
})
|
|
if err == nil && (!final.Done || final.EvalCount == 0 || !strings.Contains(strings.ToLower(text), tc.want)) {
|
|
err = fmt.Errorf("want %q in image/text response, got %q (done=%v tokens=%d)", tc.want, text, final.Done, final.EvalCount)
|
|
}
|
|
results <- err
|
|
}()
|
|
}
|
|
close(start)
|
|
for range cases {
|
|
if err := <-results; err != nil {
|
|
t.Errorf("round %d: %v", round, err)
|
|
}
|
|
}
|
|
if t.Failed() {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestMLXBatchingLongPrompts crosses the prefill chunk and sparse-attention
|
|
// boundaries with unequal histories. Longer replies keep the short-prompt row
|
|
// generating while its peer processes later prompt chunks.
|
|
func TestMLXBatchingLongPrompts(t *testing.T) {
|
|
if testModel == "" {
|
|
t.Skip("set OLLAMA_TEST_MODEL to a published MLX model")
|
|
}
|
|
ctx, cancel := context.WithTimeout(t.Context(), 8*time.Minute)
|
|
defer cancel()
|
|
client, _, cleanup := InitServerConnection(ctx, t)
|
|
defer cleanup()
|
|
info, err := client.Show(ctx, &api.ShowRequest{Model: testModel})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx = inklingBatchContext(t, ctx, info, 15*time.Minute)
|
|
start := make(chan struct{})
|
|
cities := []string{"Paris", "Tokyo", "Rome", "Madrid", "London", "Berlin", "Ottawa", "Cairo"}
|
|
countries := []string{"France", "Japan", "Italy", "Spain", "the United Kingdom", "Germany", "Canada", "Egypt"}
|
|
historyBudget := 512
|
|
if info.Renderer == "inkling2" {
|
|
historyBudget = 3072
|
|
}
|
|
rows := max(2, min(len(cities), int(envconfig.NumParallel())))
|
|
t.Logf("keeping %d requests active across unequal prompt lengths", rows)
|
|
results := make(chan error, rows)
|
|
for i, city := range cities[:rows] {
|
|
go func() {
|
|
repetitions := 20
|
|
if i%2 == 1 {
|
|
repetitions = 180
|
|
}
|
|
prompt := strings.Repeat("The island has an old lighthouse, a harbor, and a village beside the sea. ", repetitions)
|
|
req := api.GenerateRequest{Model: testModel, Runner: "mlx", Prompt: prompt + "\nIgnore the preceding background. What is the capital of " + countries[i] + "? Begin with the city name, then write a detailed 1000-word history of the city.", Think: &api.ThinkValue{Value: false}, KeepAlive: &api.Duration{Duration: time.Minute}, Options: map[string]any{"num_predict": historyBudget, "num_ctx": 8192, "temperature": 0}}
|
|
<-start
|
|
var text, thinking string
|
|
var final api.GenerateResponse
|
|
err := client.Generate(ctx, &req, func(chunk api.GenerateResponse) error {
|
|
text += chunk.Response
|
|
thinking += chunk.Thinking
|
|
if chunk.Done {
|
|
final = chunk
|
|
}
|
|
return nil
|
|
})
|
|
if err == nil && (!final.Done || !strings.Contains(text, city)) {
|
|
err = fmt.Errorf("row %d want %s, got %q (done=%v tokens=%d reason=%s thinking=%q)", i, city, text, final.Done, final.EvalCount, final.DoneReason, thinking)
|
|
}
|
|
if err == nil && i%2 == 1 && final.PromptEvalCount <= 2048 {
|
|
err = fmt.Errorf("long prompt only has %d tokens; did not cross chunk/sparse boundary", final.PromptEvalCount)
|
|
}
|
|
results <- err
|
|
}()
|
|
}
|
|
close(start)
|
|
for range rows {
|
|
if err := <-results; err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
}
|