mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-18 22:14:27 +00:00
🧾 fix: Count Google Native Media Blocks in Precomputed Token Budgets (#17890)
* fix: Count Google Native Media Blocks in Precomputed Token Budgets * test: Cover Gemini PDF context accounting in the mock harness
This commit is contained in:
parent
721aba4bb2
commit
ea584cd519
5 changed files with 264 additions and 0 deletions
|
|
@ -218,6 +218,9 @@ const baseEnv = {
|
|||
ASSISTANTS_API_KEY: 'e2e-mock-assistants-key',
|
||||
ASSISTANTS_BASE_URL: `http://127.0.0.1:${ASSISTANTS_PORT}/v1`,
|
||||
ASSISTANTS_MODELS: 'gpt-4o-mini',
|
||||
/** Enables the native Google provider for agents. The run hook replaces the model, so the
|
||||
* key never reaches Google; it only lets attachments take the Google encoding path. */
|
||||
GOOGLE_KEY: 'e2e-mock-google-key',
|
||||
...(process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN
|
||||
? { E2E_CODE_BRIDGE_ADMIN_TOKEN: process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN }
|
||||
: {}),
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ const INVOKE_SKILL_MARKER = 'E2E_INVOKE_SKILL:';
|
|||
const ASSERT_PROVIDER_FILE_MARKER = 'E2E_ASSERT_PROVIDER_FILE:';
|
||||
const ASSERT_AGENT_CONTEXT_MARKER = 'E2E_ASSERT_AGENT_CONTEXT:';
|
||||
const REPORT_LLM_CONFIG_MARKER = 'E2E_REPORT_LLM_CONFIG';
|
||||
const REPORT_CONTEXT_TOKENS_MARKER = 'E2E_REPORT_CONTEXT_TOKENS';
|
||||
const ASSERT_HISTORY_MARKER = 'E2E_ASSERT_HISTORY:';
|
||||
const ASSERT_PROJECT_CONTEXT_MARKER = 'E2E_ASSERT_PROJECT_CONTEXT:';
|
||||
const ASSERT_QUOTE_MARKER = 'E2E_ASSERT_QUOTE:';
|
||||
|
|
@ -534,6 +535,25 @@ function llmConfigReportResponses(graph) {
|
|||
return { responses: [`E2E llm config: ${JSON.stringify(report)}`] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports the precomputed per-message token counts the primary agent prunes against, read
|
||||
* before the fake model runs: the latest message's count and the total for the history.
|
||||
*/
|
||||
function contextTokensReportResponses(graph) {
|
||||
const agent = graph?.agentContexts?.values().next().value;
|
||||
const tokenMap = agent?.baseIndexTokenCountMap ?? agent?.indexTokenCountMap ?? {};
|
||||
const indices = Object.keys(tokenMap)
|
||||
.map(Number)
|
||||
.filter((index) => Number.isInteger(index));
|
||||
const latestIndex = indices.length > 0 ? Math.max(...indices) : null;
|
||||
const report = {
|
||||
provider: agent?.provider ?? null,
|
||||
latest: latestIndex == null ? null : (tokenMap[latestIndex] ?? null),
|
||||
total: indices.reduce((sum, index) => sum + (tokenMap[index] ?? 0), 0),
|
||||
};
|
||||
return { responses: [`E2E context tokens: ${JSON.stringify(report)}`] };
|
||||
}
|
||||
|
||||
function collectSystemPromptText(messages) {
|
||||
return (messages ?? [])
|
||||
.filter((message) => messageType(message) === 'system')
|
||||
|
|
@ -3787,6 +3807,10 @@ function resolveResponses({ graph, messages, text, toolNames }) {
|
|||
return llmConfigReportResponses(graph);
|
||||
}
|
||||
|
||||
if (text.includes(REPORT_CONTEXT_TOKENS_MARKER)) {
|
||||
return contextTokensReportResponses(graph);
|
||||
}
|
||||
|
||||
if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) {
|
||||
return {
|
||||
responses: [MOCK_REPLY],
|
||||
|
|
|
|||
120
e2e/specs/mock/scenarios/google-media-tokens.spec.ts
Normal file
120
e2e/specs/mock/scenarios/google-media-tokens.spec.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import type { AgentDetail } from '../agents.helpers';
|
||||
import type { UploadedFile } from '../helpers';
|
||||
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from '../agents.helpers';
|
||||
import {
|
||||
NEW_CHAT_PATH,
|
||||
getAccessToken,
|
||||
messagesView,
|
||||
requestJson,
|
||||
sendMessageAndWaitForCompletion,
|
||||
uniqueName,
|
||||
uploadViaUnifiedButton,
|
||||
} from '../helpers';
|
||||
|
||||
const REPORT_MARKER = 'E2E_REPORT_CONTEXT_TOKENS';
|
||||
/** About 300k base64 characters once encoded, which the counter estimates as four pages. */
|
||||
const PDF_BODY_BYTES = 225_000;
|
||||
/** One estimated PDF page; a text-only probe message counts a few dozen tokens at most. */
|
||||
const ONE_PAGE_TOKENS = 1500;
|
||||
|
||||
type ContextTokensReport = { provider: string | null; latest: number | null; total: number };
|
||||
|
||||
function syntheticPdf(): string {
|
||||
return `%PDF-1.4\n%${'A'.repeat(PDF_BODY_BYTES)}\n%%EOF\n`;
|
||||
}
|
||||
|
||||
async function createAndSelectGeminiAgent(page: Page): Promise<string> {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
const token = await getAccessToken(page);
|
||||
const agentName = uniqueAgentName('E2E Gemini Media Agent');
|
||||
const agent = await requestJson<AgentDetail>(page, {
|
||||
path: '/api/agents',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: {
|
||||
name: agentName,
|
||||
description: 'Counts native Google attachments toward the context budget.',
|
||||
instructions: 'Read the attachment.',
|
||||
provider: 'google',
|
||||
model: 'gemini-2.5-flash',
|
||||
model_parameters: { resendFiles: true },
|
||||
},
|
||||
});
|
||||
|
||||
const form = await openAgentBuilder(page);
|
||||
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
|
||||
await page.getByRole('option', { name: agentName }).click();
|
||||
await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
return agent.id;
|
||||
}
|
||||
|
||||
async function reportContextTokens(page: Page): Promise<ContextTokensReport> {
|
||||
const before = await messagesView(page)
|
||||
.getByText(/E2E context tokens: /)
|
||||
.count();
|
||||
await sendMessageAndWaitForCompletion(page, `${REPORT_MARKER} ${Date.now()}`);
|
||||
const reply = messagesView(page).getByText(/E2E context tokens: /);
|
||||
await expect(reply).toHaveCount(before + 1, { timeout: 15000 });
|
||||
const text = (await reply.last().textContent()) ?? '';
|
||||
return JSON.parse(text.slice(text.indexOf('{'))) as ContextTokensReport;
|
||||
}
|
||||
|
||||
/** Set up through the agent builder, which the mobile layout keeps in its drawer. */
|
||||
test.use({ viewport: { width: 1280, height: 860 }, hasTouch: false, isMobile: false });
|
||||
|
||||
test.describe('Google native media in the context budget', () => {
|
||||
let agentId: string | undefined;
|
||||
|
||||
test.afterEach(async ({ page }) => {
|
||||
await cleanupAgent(page, agentId);
|
||||
agentId = undefined;
|
||||
});
|
||||
|
||||
test('a PDF sent to a Gemini agent counts toward the context budget @scenario:gemini-pdf-counts-toward-context-budget', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120000);
|
||||
agentId = await createAndSelectGeminiAgent(page);
|
||||
|
||||
const fileName = `${uniqueName('report')}.pdf`;
|
||||
const upload = await uploadViaUnifiedButton(page, {
|
||||
name: fileName,
|
||||
mimeType: 'application/pdf',
|
||||
content: syntheticPdf(),
|
||||
});
|
||||
expect(upload.ok()).toBeTruthy();
|
||||
expect(((await upload.json()) as UploadedFile).llmDeliveryPath).toBe('provider');
|
||||
await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 });
|
||||
|
||||
const report = await reportContextTokens(page);
|
||||
expect(report.provider).toBe('google');
|
||||
expect(report.latest).toBeGreaterThan(ONE_PAGE_TOKENS);
|
||||
});
|
||||
|
||||
test('a PDF resent from an earlier Gemini turn still counts on the next turn @scenario:gemini-resent-pdf-counts-on-later-turns', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120000);
|
||||
agentId = await createAndSelectGeminiAgent(page);
|
||||
|
||||
const fileName = `${uniqueName('history')}.pdf`;
|
||||
const upload = await uploadViaUnifiedButton(page, {
|
||||
name: fileName,
|
||||
mimeType: 'application/pdf',
|
||||
content: syntheticPdf(),
|
||||
});
|
||||
expect(upload.ok()).toBeTruthy();
|
||||
await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 });
|
||||
|
||||
const first = await reportContextTokens(page);
|
||||
expect(first.latest).toBeGreaterThan(ONE_PAGE_TOKENS);
|
||||
|
||||
const second = await reportContextTokens(page);
|
||||
expect(second.provider).toBe('google');
|
||||
expect(second.latest).toBeLessThan(ONE_PAGE_TOKENS);
|
||||
expect(second.total).toBeGreaterThan(ONE_PAGE_TOKENS);
|
||||
});
|
||||
});
|
||||
|
|
@ -11,6 +11,7 @@ import {
|
|||
collectToolCallIds,
|
||||
countRetainedToolTokens,
|
||||
createCachedTokenCounter,
|
||||
countFormattedMessageTokens,
|
||||
payloadParser,
|
||||
prependQuotes,
|
||||
prependFileContext,
|
||||
|
|
@ -19,6 +20,76 @@ import {
|
|||
import { ATTACHMENT_ONLY_TEXT } from '~/files/context';
|
||||
import Tokenizer from '~/utils/tokenizer';
|
||||
|
||||
describe('countFormattedMessageTokens media blocks', () => {
|
||||
const text = { type: 'text', text: 'Read the attachment.' };
|
||||
const added = (block: Record<string, unknown>, encoding: EncodingName = 'o200k_base') =>
|
||||
countFormattedMessageTokens({ role: 'user', content: [text, block] }, encoding) -
|
||||
countFormattedMessageTokens({ role: 'user', content: [text] }, encoding);
|
||||
|
||||
it('counts an inline Google PDF by its estimated page count', () => {
|
||||
const small = added({ type: 'media', mimeType: 'application/pdf', data: 'A'.repeat(75_000) });
|
||||
const large = added({
|
||||
type: 'media',
|
||||
mimeType: 'application/pdf',
|
||||
data: 'A'.repeat(10_000_000),
|
||||
});
|
||||
expect(small).toBeGreaterThan(0);
|
||||
expect(large).toBeGreaterThan(190_000);
|
||||
expect(
|
||||
added({ type: 'media', mimeType: 'application/pdf', data: 'A'.repeat(75_000) }, 'claude'),
|
||||
).toBeGreaterThan(small);
|
||||
});
|
||||
|
||||
it('counts PDF, audio and video references that carry no inline data', () => {
|
||||
for (const mimeType of ['application/pdf', 'audio/mpeg', 'video/mp4']) {
|
||||
expect(added({ type: 'media', mimeType, fileUri: 'gs://bucket/file' })).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('counts inline audio and video by size', () => {
|
||||
const audio = added({ type: 'media', mimeType: 'audio/wav', data: 'A'.repeat(1_000_000) });
|
||||
const video = added({ type: 'media', mimeType: 'video/mp4', data: 'A'.repeat(10_000_000) });
|
||||
expect(audio).toBeGreaterThan(added({ type: 'media', mimeType: 'audio/wav', data: 'AAAA' }));
|
||||
expect(video).toBeGreaterThan(added({ type: 'media', mimeType: 'video/mp4', data: 'AAAA' }));
|
||||
});
|
||||
|
||||
it('counts inline images and unknown media types', () => {
|
||||
expect(added({ type: 'media', mimeType: 'image/png', data: 'AAAA' })).toBeGreaterThan(0);
|
||||
expect(added({ type: 'media', mimeType: 'text/csv', data: 'AAAA' })).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('rejects an oversized Google PDF before invoking the model', async () => {
|
||||
const content = [
|
||||
text,
|
||||
{ type: 'media', mimeType: 'application/pdf', data: 'A'.repeat(10_000_000) },
|
||||
];
|
||||
const graph = new StandardGraph({
|
||||
runId: 'oversized-native-pdf',
|
||||
agents: [
|
||||
{
|
||||
agentId: 'primary',
|
||||
provider: Providers.GOOGLE,
|
||||
maxContextTokens: 190_000,
|
||||
summarizationEnabled: false,
|
||||
},
|
||||
],
|
||||
tokenCounter: await createCachedTokenCounter('o200k_base'),
|
||||
indexTokenCountMap: {
|
||||
0: countFormattedMessageTokens({ role: 'user', content }, 'o200k_base'),
|
||||
},
|
||||
});
|
||||
graph.overrideTestModel(['should not be called']);
|
||||
await expect(
|
||||
graph
|
||||
.createAgentNode('primary')
|
||||
.invoke(
|
||||
{ messages: [new HumanMessage({ content })] },
|
||||
{ configurable: { thread_id: graph.runId }, recursionLimit: 12 },
|
||||
),
|
||||
).rejects.toThrow('empty_messages');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createCachedTokenCounter', () => {
|
||||
const encodings: EncodingName[] = ['o200k_base', 'claude'];
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
getTokenCountForMessage,
|
||||
estimateOpenAIImageTokens,
|
||||
estimateAnthropicImageTokens,
|
||||
estimateTimedMediaBlockTokens,
|
||||
markTokenCounterCacheCompatible,
|
||||
} from '@librechat/agents';
|
||||
import type { MessageContentComplex, TokenCounter } from '@librechat/agents';
|
||||
|
|
@ -80,6 +81,7 @@ type ContentBlock = {
|
|||
source?: { type?: string; data?: string; media_type?: string; content?: unknown[] };
|
||||
source_type?: string;
|
||||
mime_type?: string;
|
||||
mimeType?: string;
|
||||
data?: string;
|
||||
text?: string;
|
||||
tool_call?: { id?: string; name?: string; args?: string; output?: string };
|
||||
|
|
@ -300,6 +302,45 @@ function estimateDocumentBlockTokens(
|
|||
return URL_DOCUMENT_FALLBACK_TOKENS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Google/Vertex native media blocks (`{ type: 'media', mimeType, data | fileUri }`), as built by
|
||||
* the file encoders and YouTube links. Mirrors the agents SDK's media accounting so these blocks
|
||||
* are not skipped by the precomputed context budget.
|
||||
*/
|
||||
function estimateMediaBlockTokens(
|
||||
block: ContentBlock,
|
||||
isClaude: boolean,
|
||||
countTokens: (text: string) => number,
|
||||
): number {
|
||||
const mime = (block.mimeType ?? block.mime_type ?? '').split(';')[0].trim().toLowerCase();
|
||||
const data = typeof block.data === 'string' && block.data.length > 0 ? block.data : undefined;
|
||||
|
||||
if (mime.startsWith('audio/') || mime.startsWith('video/')) {
|
||||
const timed = estimateTimedMediaBlockTokens({ ...block, mimeType: mime });
|
||||
return Math.ceil(timed * IMAGE_TOKEN_SAFETY_MARGIN);
|
||||
}
|
||||
if (data == null) {
|
||||
return mime.startsWith('image/') ? 1024 : URL_DOCUMENT_FALLBACK_TOKENS;
|
||||
}
|
||||
if (mime.startsWith('image/')) {
|
||||
return estimateImageDataTokens(data, isClaude);
|
||||
}
|
||||
if (mime === 'application/pdf') {
|
||||
const comma = data.startsWith('data:') ? data.indexOf(',') : -1;
|
||||
return estimateDocumentBlockTokens(
|
||||
{
|
||||
type: 'file',
|
||||
source_type: 'base64',
|
||||
mime_type: mime,
|
||||
data: comma >= 0 ? data.slice(comma + 1) : data,
|
||||
},
|
||||
isClaude,
|
||||
countTokens,
|
||||
);
|
||||
}
|
||||
return Math.max(URL_DOCUMENT_FALLBACK_TOKENS, data.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Single-pass token counter for formatted messages (plain objects with role/content/name).
|
||||
* Handles text, tool_call, image, and document content types in one iteration,
|
||||
|
|
@ -349,6 +390,11 @@ export function countFormattedMessageTokens(
|
|||
continue;
|
||||
}
|
||||
|
||||
if (type === 'media') {
|
||||
numTokens += estimateMediaBlockTokens(block, isClaude, countTokens);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (type === ContentTypes.TOOL_CALL && block.tool_call != null) {
|
||||
const { name, args, output } = block.tool_call;
|
||||
if (typeof name === 'string' && name) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue