diff --git a/packages/api/src/agents/activityLabels/__tests__/host.spec.ts b/packages/api/src/agents/activityLabels/__tests__/host.spec.ts index 50a3ff6196..ea49989e09 100644 --- a/packages/api/src/agents/activityLabels/__tests__/host.spec.ts +++ b/packages/api/src/agents/activityLabels/__tests__/host.spec.ts @@ -159,6 +159,31 @@ describe('resolveActivityLabelModel model precedence', () => { expect.objectContaining({ model_parameters: { model: 'label-model' } }), ); }); + + /** The label often runs a cheaper model than the primary generation, so + * primary-only options (thinking, output caps) must be stripped exactly + * like the title path — while proxy headers survive. */ + it('strips primary-generation options but keeps the Anthropic header carrier', async () => { + mockGetOptions.mockResolvedValueOnce({ + llmConfig: { + model: 'resolved', + thinking: { type: 'enabled', budget_tokens: 4096 }, + maxOutputTokens: 8192, + streaming: true, + modelKwargs: { max_output_tokens: 8192, service_tier: 'flex' }, + clientOptions: { defaultHeaders: { 'x-proxy-key': 'abc' } }, + }, + } as never); + const resolved = await resolve({ activityLabel: true, activityModel: 'label-model' }); + const clientOptions = resolved.clientOptions as Record; + expect(clientOptions.thinking).toBeUndefined(); + expect(clientOptions.maxOutputTokens).toBeUndefined(); + expect(clientOptions.streaming).toBeUndefined(); + expect(clientOptions.modelKwargs).toEqual({ service_tier: 'flex' }); + expect(clientOptions.clientOptions).toEqual({ + defaultHeaders: { 'x-proxy-key': 'abc' }, + }); + }); }); describe('mapCollectedMetadataToUsage cache tokens', () => { diff --git a/packages/api/src/agents/activityLabels/__tests__/runtime.spec.ts b/packages/api/src/agents/activityLabels/__tests__/runtime.spec.ts index 59ecdae66d..e74e956501 100644 --- a/packages/api/src/agents/activityLabels/__tests__/runtime.spec.ts +++ b/packages/api/src/agents/activityLabels/__tests__/runtime.spec.ts @@ -175,6 +175,38 @@ describe('buildPrompt', () => { expect(tight).toContain('…'); }); + /** Per-entry truncation alone leaves the batch dimension unbounded: a + * parallel batch of hundreds of calls must not build a prompt past the + * fast model's window. */ + it('bounds the total entries section for giant parallel batches', () => { + const entries = Array.from({ length: 200 }, (_, i) => ({ + toolName: `tool_${i}`, + toolInput: { i }, + toolUseId: `t${i}`, + status: 'success' as const, + toolOutput: 'y'.repeat(500), + })); + const prompt = buildPrompt(entries, 600); + expect(prompt.length).toBeLessThan(15_000); + expect(prompt).toMatch(/\(\+\d+ more tool calls not shown\)/); + /** Prefix semantics: the first entry is always present in full. */ + expect(prompt).toContain('- tool_0('); + expect(prompt).not.toContain('- tool_199('); + }); + + it('shows every entry when the batch fits the budget', () => { + const entries = Array.from({ length: 3 }, (_, i) => ({ + toolName: `tool_${i}`, + toolInput: { i }, + toolUseId: `t${i}`, + status: 'success' as const, + toolOutput: 'ok', + })); + const prompt = buildPrompt(entries, 600); + expect(prompt).toContain('- tool_2('); + expect(prompt).not.toContain('more tool calls not shown'); + }); + it('serializes small structured values exactly like JSON.stringify', () => { const toolInput = { q: 'docs', filters: { lang: 'en', page: 2 }, ids: [1, 2] }; const prompt = buildPrompt( diff --git a/packages/api/src/agents/activityLabels/host.ts b/packages/api/src/agents/activityLabels/host.ts index 846f874d1d..fdb30c8227 100644 --- a/packages/api/src/agents/activityLabels/host.ts +++ b/packages/api/src/agents/activityLabels/host.ts @@ -8,6 +8,7 @@ import type { EndpointDbMethods, OpenAIConfiguration, ServerRequest } from '~/ty import type { ActivityLabelLLM } from './runtime'; import { getProviderConfig } from '~/endpoints/config/providers'; import { resolveConfigHeaders } from '~/utils/headers'; +import { omitTitleOptions } from '~/agents/client'; import { createSafeUser } from '~/utils/env'; /** Cache-token details in the LangChain-standard normalized shape. */ @@ -289,7 +290,36 @@ export async function resolveActivityLabelModel({ ) { provider = Providers.AZURE; } - const clientOptions = { ...(llmConfig ?? {}) } as MaybeAzureConfig; + /** Sanitized copy, exactly like the title path: the label often runs a + * DIFFERENT (cheaper) model than the primary generation, so + * primary-generation options must not ride along. `omitTitleOptions` + * drops thinking/streaming/output-cap keys that can make the label + * request fail outright on the substitute model — or spend extended + * thinking on a 4–9 word header and blow the settlement window. The + * `modelKwargs` output caps go for the same reason (copied, not mutated: + * `llmConfig` is shared with the memoized provider resolution). */ + const rawOptions = { ...(llmConfig ?? {}) } as MaybeAzureConfig & { + modelKwargs?: Record; + clientOptions?: { defaultHeaders?: unknown }; + }; + if (rawOptions.modelKwargs != null) { + const modelKwargs = { ...rawOptions.modelKwargs }; + delete modelKwargs.max_completion_tokens; + delete modelKwargs.max_output_tokens; + rawOptions.modelKwargs = modelKwargs; + } + /** The filter drops the Anthropic `clientOptions` carrier (thinking, + * streaming), which would also drop its `defaultHeaders` — restore the + * SAME object reference so gateway/proxy metadata still reaches label + * requests and `resolveConfigHeaders` mutates the object the client is + * actually built from. */ + const anthropicCarrier = rawOptions.clientOptions; + const clientOptions = Object.fromEntries( + Object.entries(rawOptions).filter(([key]) => !omitTitleOptions.has(key)), + ) as MaybeAzureConfig & { clientOptions?: { defaultHeaders?: unknown } }; + if (anthropicCarrier?.defaultHeaders != null && clientOptions.clientOptions == null) { + clientOptions.clientOptions = anthropicCarrier; + } if (options.configOptions) { clientOptions.configuration = options.configOptions; } diff --git a/packages/api/src/agents/activityLabels/runtime.ts b/packages/api/src/agents/activityLabels/runtime.ts index 53a90d1073..f623f75c3d 100644 --- a/packages/api/src/agents/activityLabels/runtime.ts +++ b/packages/api/src/agents/activityLabels/runtime.ts @@ -174,6 +174,13 @@ const DEFAULT_CHAR_LIMIT = 600; * reach a distinguishing path or query past the first 200 characters. */ const INTENT_CHAR_LIMIT = 200; const SUMMARY_TIMEOUT_MS = 12_000; +/** Total budget for the entries section. Per-entry truncation alone leaves + * the batch dimension unbounded — a parallel batch of hundreds of calls + * would build a prompt past the fast model's window and bill input tokens + * far beyond what a one-line header justifies. Scales with the configured + * per-entry limit so a raised `activityCharLimit` still fits several + * entries; entries past the budget are skipped WITHOUT serializing them. */ +const ENTRIES_CHAR_BUDGET = 8_000; /** Hard bound on the PERSISTED label. The instruction asks for 4–9 words, but * a model that ignores it — or is steered by injection through untrusted * tool output — could otherwise turn one header into thousands of tokens @@ -348,14 +355,32 @@ export function buildPrompt( .join('\n'), ); } - const lines = entries.map((entry) => { + const budget = Math.max(ENTRIES_CHAR_BUDGET, charLimit * 4); + const lines: string[] = []; + let used = 0; + let omitted = 0; + for (const entry of entries) { + /** Prefix cut, never a mid-list sample: once the budget is spent the + * remaining entries are skipped unserialized (a giant batch must not + * even pay the stringify cost for lines that will be dropped). The + * first entry always fits, and the line that crosses the budget is + * kept, so at least one complete call is always shown. */ + if (lines.length > 0 && used >= budget) { + omitted += 1; + continue; + } const input = truncate(stringifyUnknown(entry.toolInput, charLimit), charLimit); const outcome = entry.status === 'error' ? `ERROR: ${truncate(entry.error ?? 'unknown error', charLimit)}` : truncate(stringifyUnknown(entry.toolOutput, charLimit), charLimit); - return `- ${entry.toolName}(${input}) → ${outcome}`; - }); + const line = `- ${entry.toolName}(${input}) → ${outcome}`; + lines.push(line); + used += line.length; + } + if (omitted > 0) { + lines.push(`- (+${omitted} more tool calls not shown)`); + } /** Flagged as reference material: without this the model tends to read the * list as the thing to summarize and hands back a transcription of it. */ sections.push(`What it called, and what came back (do not restate these):\n${lines.join('\n')}`);