diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index 2240718ef1..5a97325f32 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -218,6 +218,9 @@ const baseEnv = { ASSISTANTS_API_KEY: 'e2e-mock-assistants-key', ASSISTANTS_BASE_URL: `http://127.0.0.1:${ASSISTANTS_PORT}/v1`, ASSISTANTS_MODELS: 'gpt-4o-mini', + /** Enables the native Google provider for agents. The run hook replaces the model, so the + * key never reaches Google; it only lets attachments take the Google encoding path. */ + GOOGLE_KEY: 'e2e-mock-google-key', ...(process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN ? { E2E_CODE_BRIDGE_ADMIN_TOKEN: process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN } : {}), diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index 73402f90a3..ad3fa68d68 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -34,6 +34,7 @@ const INVOKE_SKILL_MARKER = 'E2E_INVOKE_SKILL:'; const ASSERT_PROVIDER_FILE_MARKER = 'E2E_ASSERT_PROVIDER_FILE:'; const ASSERT_AGENT_CONTEXT_MARKER = 'E2E_ASSERT_AGENT_CONTEXT:'; const REPORT_LLM_CONFIG_MARKER = 'E2E_REPORT_LLM_CONFIG'; +const REPORT_CONTEXT_TOKENS_MARKER = 'E2E_REPORT_CONTEXT_TOKENS'; const ASSERT_HISTORY_MARKER = 'E2E_ASSERT_HISTORY:'; const ASSERT_PROJECT_CONTEXT_MARKER = 'E2E_ASSERT_PROJECT_CONTEXT:'; const ASSERT_QUOTE_MARKER = 'E2E_ASSERT_QUOTE:'; @@ -534,6 +535,25 @@ function llmConfigReportResponses(graph) { return { responses: [`E2E llm config: ${JSON.stringify(report)}`] }; } +/** + * Reports the precomputed per-message token counts the primary agent prunes against, read + * before the fake model runs: the latest message's count and the total for the history. + */ +function contextTokensReportResponses(graph) { + const agent = graph?.agentContexts?.values().next().value; + const tokenMap = agent?.baseIndexTokenCountMap ?? agent?.indexTokenCountMap ?? {}; + const indices = Object.keys(tokenMap) + .map(Number) + .filter((index) => Number.isInteger(index)); + const latestIndex = indices.length > 0 ? Math.max(...indices) : null; + const report = { + provider: agent?.provider ?? null, + latest: latestIndex == null ? null : (tokenMap[latestIndex] ?? null), + total: indices.reduce((sum, index) => sum + (tokenMap[index] ?? 0), 0), + }; + return { responses: [`E2E context tokens: ${JSON.stringify(report)}`] }; +} + function collectSystemPromptText(messages) { return (messages ?? []) .filter((message) => messageType(message) === 'system') @@ -3787,6 +3807,10 @@ function resolveResponses({ graph, messages, text, toolNames }) { return llmConfigReportResponses(graph); } + if (text.includes(REPORT_CONTEXT_TOKENS_MARKER)) { + return contextTokensReportResponses(graph); + } + if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) { return { responses: [MOCK_REPLY], diff --git a/e2e/specs/mock/scenarios/google-media-tokens.spec.ts b/e2e/specs/mock/scenarios/google-media-tokens.spec.ts new file mode 100644 index 0000000000..00c7144278 --- /dev/null +++ b/e2e/specs/mock/scenarios/google-media-tokens.spec.ts @@ -0,0 +1,120 @@ +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import type { AgentDetail } from '../agents.helpers'; +import type { UploadedFile } from '../helpers'; +import { cleanupAgent, openAgentBuilder, uniqueAgentName } from '../agents.helpers'; +import { + NEW_CHAT_PATH, + getAccessToken, + messagesView, + requestJson, + sendMessageAndWaitForCompletion, + uniqueName, + uploadViaUnifiedButton, +} from '../helpers'; + +const REPORT_MARKER = 'E2E_REPORT_CONTEXT_TOKENS'; +/** About 300k base64 characters once encoded, which the counter estimates as four pages. */ +const PDF_BODY_BYTES = 225_000; +/** One estimated PDF page; a text-only probe message counts a few dozen tokens at most. */ +const ONE_PAGE_TOKENS = 1500; + +type ContextTokensReport = { provider: string | null; latest: number | null; total: number }; + +function syntheticPdf(): string { + return `%PDF-1.4\n%${'A'.repeat(PDF_BODY_BYTES)}\n%%EOF\n`; +} + +async function createAndSelectGeminiAgent(page: Page): Promise { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + const token = await getAccessToken(page); + const agentName = uniqueAgentName('E2E Gemini Media Agent'); + const agent = await requestJson(page, { + path: '/api/agents', + token, + method: 'POST', + body: { + name: agentName, + description: 'Counts native Google attachments toward the context budget.', + instructions: 'Read the attachment.', + provider: 'google', + model: 'gemini-2.5-flash', + model_parameters: { resendFiles: true }, + }, + }); + + const form = await openAgentBuilder(page); + await form.getByRole('combobox', { name: 'Agent', exact: true }).click(); + await page.getByRole('option', { name: agentName }).click(); + await expect(form.getByLabel('Agent name')).toHaveValue(agentName); + await form.getByRole('button', { name: 'Select Agent' }).click(); + return agent.id; +} + +async function reportContextTokens(page: Page): Promise { + const before = await messagesView(page) + .getByText(/E2E context tokens: /) + .count(); + await sendMessageAndWaitForCompletion(page, `${REPORT_MARKER} ${Date.now()}`); + const reply = messagesView(page).getByText(/E2E context tokens: /); + await expect(reply).toHaveCount(before + 1, { timeout: 15000 }); + const text = (await reply.last().textContent()) ?? ''; + return JSON.parse(text.slice(text.indexOf('{'))) as ContextTokensReport; +} + +/** Set up through the agent builder, which the mobile layout keeps in its drawer. */ +test.use({ viewport: { width: 1280, height: 860 }, hasTouch: false, isMobile: false }); + +test.describe('Google native media in the context budget', () => { + let agentId: string | undefined; + + test.afterEach(async ({ page }) => { + await cleanupAgent(page, agentId); + agentId = undefined; + }); + + test('a PDF sent to a Gemini agent counts toward the context budget @scenario:gemini-pdf-counts-toward-context-budget', async ({ + page, + }) => { + test.setTimeout(120000); + agentId = await createAndSelectGeminiAgent(page); + + const fileName = `${uniqueName('report')}.pdf`; + const upload = await uploadViaUnifiedButton(page, { + name: fileName, + mimeType: 'application/pdf', + content: syntheticPdf(), + }); + expect(upload.ok()).toBeTruthy(); + expect(((await upload.json()) as UploadedFile).llmDeliveryPath).toBe('provider'); + await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 }); + + const report = await reportContextTokens(page); + expect(report.provider).toBe('google'); + expect(report.latest).toBeGreaterThan(ONE_PAGE_TOKENS); + }); + + test('a PDF resent from an earlier Gemini turn still counts on the next turn @scenario:gemini-resent-pdf-counts-on-later-turns', async ({ + page, + }) => { + test.setTimeout(120000); + agentId = await createAndSelectGeminiAgent(page); + + const fileName = `${uniqueName('history')}.pdf`; + const upload = await uploadViaUnifiedButton(page, { + name: fileName, + mimeType: 'application/pdf', + content: syntheticPdf(), + }); + expect(upload.ok()).toBeTruthy(); + await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 }); + + const first = await reportContextTokens(page); + expect(first.latest).toBeGreaterThan(ONE_PAGE_TOKENS); + + const second = await reportContextTokens(page); + expect(second.provider).toBe('google'); + expect(second.latest).toBeLessThan(ONE_PAGE_TOKENS); + expect(second.total).toBeGreaterThan(ONE_PAGE_TOKENS); + }); +}); diff --git a/packages/api/src/agents/client.spec.ts b/packages/api/src/agents/client.spec.ts index 9ffb3a7d9e..3acacd9054 100644 --- a/packages/api/src/agents/client.spec.ts +++ b/packages/api/src/agents/client.spec.ts @@ -11,6 +11,7 @@ import { collectToolCallIds, countRetainedToolTokens, createCachedTokenCounter, + countFormattedMessageTokens, payloadParser, prependQuotes, prependFileContext, @@ -19,6 +20,76 @@ import { import { ATTACHMENT_ONLY_TEXT } from '~/files/context'; import Tokenizer from '~/utils/tokenizer'; +describe('countFormattedMessageTokens media blocks', () => { + const text = { type: 'text', text: 'Read the attachment.' }; + const added = (block: Record, encoding: EncodingName = 'o200k_base') => + countFormattedMessageTokens({ role: 'user', content: [text, block] }, encoding) - + countFormattedMessageTokens({ role: 'user', content: [text] }, encoding); + + it('counts an inline Google PDF by its estimated page count', () => { + const small = added({ type: 'media', mimeType: 'application/pdf', data: 'A'.repeat(75_000) }); + const large = added({ + type: 'media', + mimeType: 'application/pdf', + data: 'A'.repeat(10_000_000), + }); + expect(small).toBeGreaterThan(0); + expect(large).toBeGreaterThan(190_000); + expect( + added({ type: 'media', mimeType: 'application/pdf', data: 'A'.repeat(75_000) }, 'claude'), + ).toBeGreaterThan(small); + }); + + it('counts PDF, audio and video references that carry no inline data', () => { + for (const mimeType of ['application/pdf', 'audio/mpeg', 'video/mp4']) { + expect(added({ type: 'media', mimeType, fileUri: 'gs://bucket/file' })).toBeGreaterThan(0); + } + }); + + it('counts inline audio and video by size', () => { + const audio = added({ type: 'media', mimeType: 'audio/wav', data: 'A'.repeat(1_000_000) }); + const video = added({ type: 'media', mimeType: 'video/mp4', data: 'A'.repeat(10_000_000) }); + expect(audio).toBeGreaterThan(added({ type: 'media', mimeType: 'audio/wav', data: 'AAAA' })); + expect(video).toBeGreaterThan(added({ type: 'media', mimeType: 'video/mp4', data: 'AAAA' })); + }); + + it('counts inline images and unknown media types', () => { + expect(added({ type: 'media', mimeType: 'image/png', data: 'AAAA' })).toBeGreaterThan(0); + expect(added({ type: 'media', mimeType: 'text/csv', data: 'AAAA' })).toBeGreaterThan(0); + }); + + it('rejects an oversized Google PDF before invoking the model', async () => { + const content = [ + text, + { type: 'media', mimeType: 'application/pdf', data: 'A'.repeat(10_000_000) }, + ]; + const graph = new StandardGraph({ + runId: 'oversized-native-pdf', + agents: [ + { + agentId: 'primary', + provider: Providers.GOOGLE, + maxContextTokens: 190_000, + summarizationEnabled: false, + }, + ], + tokenCounter: await createCachedTokenCounter('o200k_base'), + indexTokenCountMap: { + 0: countFormattedMessageTokens({ role: 'user', content }, 'o200k_base'), + }, + }); + graph.overrideTestModel(['should not be called']); + await expect( + graph + .createAgentNode('primary') + .invoke( + { messages: [new HumanMessage({ content })] }, + { configurable: { thread_id: graph.runId }, recursionLimit: 12 }, + ), + ).rejects.toThrow('empty_messages'); + }); +}); + describe('createCachedTokenCounter', () => { const encodings: EncodingName[] = ['o200k_base', 'claude']; diff --git a/packages/api/src/agents/client.ts b/packages/api/src/agents/client.ts index e83836bd2d..1146fb7b80 100644 --- a/packages/api/src/agents/client.ts +++ b/packages/api/src/agents/client.ts @@ -10,6 +10,7 @@ import { getTokenCountForMessage, estimateOpenAIImageTokens, estimateAnthropicImageTokens, + estimateTimedMediaBlockTokens, markTokenCounterCacheCompatible, } from '@librechat/agents'; import type { MessageContentComplex, TokenCounter } from '@librechat/agents'; @@ -80,6 +81,7 @@ type ContentBlock = { source?: { type?: string; data?: string; media_type?: string; content?: unknown[] }; source_type?: string; mime_type?: string; + mimeType?: string; data?: string; text?: string; tool_call?: { id?: string; name?: string; args?: string; output?: string }; @@ -300,6 +302,45 @@ function estimateDocumentBlockTokens( return URL_DOCUMENT_FALLBACK_TOKENS; } +/** + * Google/Vertex native media blocks (`{ type: 'media', mimeType, data | fileUri }`), as built by + * the file encoders and YouTube links. Mirrors the agents SDK's media accounting so these blocks + * are not skipped by the precomputed context budget. + */ +function estimateMediaBlockTokens( + block: ContentBlock, + isClaude: boolean, + countTokens: (text: string) => number, +): number { + const mime = (block.mimeType ?? block.mime_type ?? '').split(';')[0].trim().toLowerCase(); + const data = typeof block.data === 'string' && block.data.length > 0 ? block.data : undefined; + + if (mime.startsWith('audio/') || mime.startsWith('video/')) { + const timed = estimateTimedMediaBlockTokens({ ...block, mimeType: mime }); + return Math.ceil(timed * IMAGE_TOKEN_SAFETY_MARGIN); + } + if (data == null) { + return mime.startsWith('image/') ? 1024 : URL_DOCUMENT_FALLBACK_TOKENS; + } + if (mime.startsWith('image/')) { + return estimateImageDataTokens(data, isClaude); + } + if (mime === 'application/pdf') { + const comma = data.startsWith('data:') ? data.indexOf(',') : -1; + return estimateDocumentBlockTokens( + { + type: 'file', + source_type: 'base64', + mime_type: mime, + data: comma >= 0 ? data.slice(comma + 1) : data, + }, + isClaude, + countTokens, + ); + } + return Math.max(URL_DOCUMENT_FALLBACK_TOKENS, data.length); +} + /** * Single-pass token counter for formatted messages (plain objects with role/content/name). * Handles text, tool_call, image, and document content types in one iteration, @@ -349,6 +390,11 @@ export function countFormattedMessageTokens( continue; } + if (type === 'media') { + numTokens += estimateMediaBlockTokens(block, isClaude, countTokens); + continue; + } + if (type === ContentTypes.TOOL_CALL && block.tool_call != null) { const { name, args, output } = block.tool_call; if (typeof name === 'string' && name) {