mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
🎞️ fix: Surface Clear Error for Unprocessable Gemini YouTube Videos (#14396)
Google rejects a YouTube video it cannot ingest with a generic `400 INVALID_ARGUMENT` that names no cause, which LibreChat relayed verbatim. Attribute the failure using request context instead: when a Google/Vertex turn carried an injected YouTube video part and the provider returns that generic rejection, map it to a typed error the client localizes. Verified against the live API: a public 9h15m video is refused this way on gemini-2.5-flash, 3.5-flash, 3.5-flash-lite and 3.6-flash, including at MEDIA_RESOLUTION_LOW, while a short video with an identical payload succeeds. Duration is the dominant trigger; region and access restrictions return the same response, so the copy leads with length without overclaiming. A duration preflight was evaluated and skipped: oEmbed does not expose duration, leaving only watch-page scraping — a blocking call against undocumented markup from rate-limited datacenter IPs that would fail open and still need this mapping underneath.
This commit is contained in:
parent
337facb4f0
commit
ad5bb477af
10 changed files with 356 additions and 2 deletions
|
|
@ -70,7 +70,9 @@ const {
|
|||
buildSkillPrimeContentParts,
|
||||
buildInitialToolSessions,
|
||||
hasUrlContextTool,
|
||||
hasYouTubeVideoParts,
|
||||
appendYouTubeVideoParts,
|
||||
resolveGoogleVideoError,
|
||||
resolveYouTubeInjectionConfig,
|
||||
decrementPendingRequest,
|
||||
maybePrewarmCodeSandbox,
|
||||
|
|
@ -600,6 +602,9 @@ class AgentClient extends BaseClient {
|
|||
max,
|
||||
mimeType,
|
||||
});
|
||||
/** Google rejects an unusable video with a generic `INVALID_ARGUMENT` that names no cause,
|
||||
* so `#sendCompletion` can only attribute one by knowing this turn carried a video. */
|
||||
this.injectedYouTubeVideo = hasYouTubeVideoParts(latestFormatted.content);
|
||||
}
|
||||
|
||||
payload = formattedMessages;
|
||||
|
|
@ -1904,9 +1909,16 @@ class AgentClient extends BaseClient {
|
|||
'[api/server/controllers/agents/client.js #sendCompletion] Unhandled error type',
|
||||
err,
|
||||
);
|
||||
const videoError = resolveGoogleVideoError({
|
||||
error: err,
|
||||
provider: this.options.agent?.provider,
|
||||
hasYouTubeVideo: this.injectedYouTubeVideo,
|
||||
});
|
||||
this.contentParts.push({
|
||||
type: ContentTypes.ERROR,
|
||||
[ContentTypes.ERROR]: `An error occurred while processing the request${err?.message ? `: ${err.message}` : ''}`,
|
||||
[ContentTypes.ERROR]:
|
||||
videoError ??
|
||||
`An error occurred while processing the request${err?.message ? `: ${err.message}` : ''}`,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ const errorMessages = {
|
|||
return info;
|
||||
},
|
||||
[ErrorTypes.GOOGLE_TOOL_CONFLICT]: 'com_error_google_tool_conflict',
|
||||
[ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE]: 'com_error_google_video_unprocessable',
|
||||
[ErrorTypes.STREAM_EXPIRED]: 'com_error_stream_expired',
|
||||
[ViolationTypes.BAN]:
|
||||
'Your account has been temporarily banned due to violations of our service.',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
import React from 'react';
|
||||
import { ErrorTypes } from 'librechat-data-provider';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import translation from '~/locales/en/translation.json';
|
||||
import Error from '../Error';
|
||||
|
||||
/**
|
||||
* Resolves keys against the real English catalog rather than a stub, so a typed error whose
|
||||
* localization key is missing or misspelled fails here instead of reaching users as a raw key.
|
||||
*/
|
||||
jest.mock('~/hooks', () => ({
|
||||
useLocalize:
|
||||
() =>
|
||||
(key: string): string =>
|
||||
(jest.requireActual('~/locales/en/translation.json') as Record<string, string>)[key] ?? key,
|
||||
}));
|
||||
|
||||
const catalog = translation as Record<string, string>;
|
||||
|
||||
describe('Error — typed provider errors', () => {
|
||||
it('renders the localized copy for a rejected Google video', () => {
|
||||
/** The exact payload `resolveGoogleVideoError` emits from the server. */
|
||||
const payload = JSON.stringify({ type: ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE });
|
||||
render(<Error text={payload} />);
|
||||
|
||||
expect(screen.getByText(catalog.com_error_google_video_unprocessable)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('names video length, the dominant cause, in the copy', () => {
|
||||
expect(catalog.com_error_google_video_unprocessable).toMatch(/too long/i);
|
||||
});
|
||||
|
||||
it('falls back to the raw provider text for an unmapped error', () => {
|
||||
const raw =
|
||||
'[GoogleGenerativeAI Error]: [400 Bad Request] Request contains an invalid argument';
|
||||
render(<Error text={raw} />);
|
||||
|
||||
expect(
|
||||
screen.getByText(new RegExp(raw.slice(0, 30).replace(/[[\]]/g, '\\$&'))),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -384,6 +384,7 @@
|
|||
"com_error_files_upload_too_large": "The file is too large. Please upload a file smaller than {{0}} MB",
|
||||
"com_error_files_validation": "An error occurred while validating the file.",
|
||||
"com_error_google_tool_conflict": "Usage of built-in Google tools are not supported with external tools. Please disable either the built-in tools or the external tools.",
|
||||
"com_error_google_video_unprocessable": "The linked video could not be processed. It is most likely too long for this model, but it may also be unavailable in this region or restricted. Try a shorter video, or describe the relevant parts in your message instead.",
|
||||
"com_error_heic_conversion": "Failed to convert HEIC image to JPEG. Please try converting the image manually or use a different format.",
|
||||
"com_error_illegal_model_request": "The model \"{{0}}\" is not available for {{1}}. Please select a different model.",
|
||||
"com_error_input_length": "The latest message token count is too long, exceeding the token limit, or your token limit parameters are misconfigured, adversely affecting the context window. More info: {{0}}. Please shorten your message, adjust the max context size from the conversation parameters, or fork the conversation to continue.",
|
||||
|
|
|
|||
152
packages/api/src/endpoints/google/errors.spec.ts
Normal file
152
packages/api/src/endpoints/google/errors.spec.ts
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import { Providers } from '@librechat/agents';
|
||||
import { ErrorTypes } from 'librechat-data-provider';
|
||||
import { isGoogleInvalidArgumentError, resolveGoogleVideoError } from './errors';
|
||||
|
||||
/**
|
||||
* Verbatim error the `@google/generative-ai` SDK raises for a rejected video, captured from a live
|
||||
* request whose message carried a 9h15m YouTube link.
|
||||
*/
|
||||
const GENERIC_400_MESSAGE =
|
||||
'[GoogleGenerativeAI Error]: Error fetching from https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:streamGenerateContent?alt=sse: [400 Bad Request] Request contains an invalid argument.';
|
||||
|
||||
function googleError(message: string, status?: number): Error & { status?: number } {
|
||||
const error: Error & { status?: number } = new Error(message);
|
||||
if (status != null) {
|
||||
error.status = status;
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
const VIDEO_ERROR = JSON.stringify({ type: ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE });
|
||||
|
||||
describe('isGoogleInvalidArgumentError', () => {
|
||||
it('matches the SDK error for a rejected video', () => {
|
||||
expect(isGoogleInvalidArgumentError(googleError(GENERIC_400_MESSAGE, 400))).toBe(true);
|
||||
});
|
||||
|
||||
it('matches when the status is only present in the message text', () => {
|
||||
expect(isGoogleInvalidArgumentError(googleError(GENERIC_400_MESSAGE))).toBe(true);
|
||||
});
|
||||
|
||||
it('matches a plain string error', () => {
|
||||
expect(isGoogleInvalidArgumentError(GENERIC_400_MESSAGE)).toBe(true);
|
||||
});
|
||||
|
||||
it('matches the Vertex wording carrying the status as a property', () => {
|
||||
expect(
|
||||
isGoogleInvalidArgumentError(googleError('Request contains an invalid argument.', 400)),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects the same wording without any 400 signal', () => {
|
||||
expect(isGoogleInvalidArgumentError(googleError('Request contains an invalid argument.'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a different Google 400 that names its own cause', () => {
|
||||
expect(
|
||||
isGoogleInvalidArgumentError(
|
||||
googleError(
|
||||
'[GoogleGenerativeAI Error]: [400 Bad Request] Please enable tool_config.include_server_side_tool_invocations to use Built-in tools with Function calling.',
|
||||
400,
|
||||
),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects rate limit and server errors', () => {
|
||||
expect(isGoogleInvalidArgumentError(googleError('[429] Resource exhausted', 429))).toBe(false);
|
||||
expect(
|
||||
isGoogleInvalidArgumentError(googleError('[503] Model is overloaded, try again', 503)),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects non-error values', () => {
|
||||
expect(isGoogleInvalidArgumentError(undefined)).toBe(false);
|
||||
expect(isGoogleInvalidArgumentError(null)).toBe(false);
|
||||
expect(isGoogleInvalidArgumentError({})).toBe(false);
|
||||
expect(isGoogleInvalidArgumentError({ message: 42 })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveGoogleVideoError', () => {
|
||||
it('returns the typed payload when a video turn hits the generic rejection', () => {
|
||||
expect(
|
||||
resolveGoogleVideoError({
|
||||
error: googleError(GENERIC_400_MESSAGE, 400),
|
||||
provider: Providers.GOOGLE,
|
||||
hasYouTubeVideo: true,
|
||||
}),
|
||||
).toBe(VIDEO_ERROR);
|
||||
});
|
||||
|
||||
it('resolves for Vertex as well as the Gemini Developer API', () => {
|
||||
expect(
|
||||
resolveGoogleVideoError({
|
||||
error: googleError(GENERIC_400_MESSAGE, 400),
|
||||
provider: Providers.VERTEXAI,
|
||||
hasYouTubeVideo: true,
|
||||
}),
|
||||
).toBe(VIDEO_ERROR);
|
||||
});
|
||||
|
||||
it('emits a payload the client error map can parse back to the typed key', () => {
|
||||
const resolved = resolveGoogleVideoError({
|
||||
error: googleError(GENERIC_400_MESSAGE, 400),
|
||||
provider: Providers.GOOGLE,
|
||||
hasYouTubeVideo: true,
|
||||
});
|
||||
expect(JSON.parse(resolved as string)).toEqual({
|
||||
type: ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE,
|
||||
});
|
||||
});
|
||||
|
||||
it('defers when the turn carried no video, so unrelated 400s keep their own message', () => {
|
||||
expect(
|
||||
resolveGoogleVideoError({
|
||||
error: googleError(GENERIC_400_MESSAGE, 400),
|
||||
provider: Providers.GOOGLE,
|
||||
hasYouTubeVideo: false,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('defers when the injection flag was never set', () => {
|
||||
expect(
|
||||
resolveGoogleVideoError({
|
||||
error: googleError(GENERIC_400_MESSAGE, 400),
|
||||
provider: Providers.GOOGLE,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('defers for non-Google providers', () => {
|
||||
expect(
|
||||
resolveGoogleVideoError({
|
||||
error: googleError(GENERIC_400_MESSAGE, 400),
|
||||
provider: Providers.OPENAI,
|
||||
hasYouTubeVideo: true,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('defers when the provider is unknown', () => {
|
||||
expect(
|
||||
resolveGoogleVideoError({
|
||||
error: googleError(GENERIC_400_MESSAGE, 400),
|
||||
hasYouTubeVideo: true,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('defers on a video turn that fails for an unrelated reason', () => {
|
||||
expect(
|
||||
resolveGoogleVideoError({
|
||||
error: googleError('[GoogleGenerativeAI Error]: [401] API key not valid', 401),
|
||||
provider: Providers.GOOGLE,
|
||||
hasYouTubeVideo: true,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
63
packages/api/src/endpoints/google/errors.ts
Normal file
63
packages/api/src/endpoints/google/errors.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import { Providers } from '@librechat/agents';
|
||||
import { ErrorTypes } from 'librechat-data-provider';
|
||||
|
||||
/**
|
||||
* Google's opaque rejection for content it will not accept. The Gemini API answers an over-length,
|
||||
* region-locked, or otherwise unreadable video with this single generic sentence and no field-level
|
||||
* detail, so the phrase alone cannot identify the cause — the caller supplies that context.
|
||||
*/
|
||||
const INVALID_ARGUMENT_REGEX = /request contains an invalid argument/i;
|
||||
|
||||
function toErrorMessage(error: unknown): string | undefined {
|
||||
if (typeof error === 'string') {
|
||||
return error;
|
||||
}
|
||||
if (error == null || typeof error !== 'object') {
|
||||
return undefined;
|
||||
}
|
||||
const { message } = error as { message?: unknown };
|
||||
return typeof message === 'string' ? message : undefined;
|
||||
}
|
||||
|
||||
function isGoogleProvider(provider?: string): boolean {
|
||||
return provider === Providers.GOOGLE || provider === Providers.VERTEXAI;
|
||||
}
|
||||
|
||||
/**
|
||||
* True for Google's generic `400 INVALID_ARGUMENT`. Both the Gemini Developer API and Vertex use
|
||||
* the same wording, so the HTTP status is accepted from either the error object or the message text
|
||||
* that the `@google/generative-ai` SDK bakes its status into.
|
||||
*/
|
||||
export function isGoogleInvalidArgumentError(error: unknown): boolean {
|
||||
const message = toErrorMessage(error);
|
||||
if (message == null || !INVALID_ARGUMENT_REGEX.test(message)) {
|
||||
return false;
|
||||
}
|
||||
const status = (error as { status?: unknown } | null)?.status;
|
||||
return status === 400 || message.includes('400');
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a failed Google request back to the YouTube video that most likely caused it, returning the
|
||||
* typed error payload the client localizes (or `undefined` to leave the original error alone).
|
||||
*
|
||||
* Attribution rests on context rather than the response body: when the turn carried an injected
|
||||
* YouTube video and Google answers with its generic `INVALID_ARGUMENT`, the video is the cause we
|
||||
* can act on. Verified against the live API — a public 9h15m video is refused this way on every
|
||||
* Gemini model tested, including at `MEDIA_RESOLUTION_LOW`, while a short video with an otherwise
|
||||
* identical payload succeeds. Duration is the common trigger; region and access restrictions
|
||||
* produce the same response, so the localized copy names length first without claiming certainty.
|
||||
*/
|
||||
export function resolveGoogleVideoError(params: {
|
||||
error: unknown;
|
||||
provider?: string;
|
||||
hasYouTubeVideo?: boolean;
|
||||
}): string | undefined {
|
||||
if (params.hasYouTubeVideo !== true || !isGoogleProvider(params.provider)) {
|
||||
return undefined;
|
||||
}
|
||||
if (!isGoogleInvalidArgumentError(params.error)) {
|
||||
return undefined;
|
||||
}
|
||||
return JSON.stringify({ type: ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE });
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
export * from './llm';
|
||||
export * from './errors';
|
||||
export * from './initialize';
|
||||
export * from './youtube';
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import type { MessageContentComplex } from '@librechat/agents';
|
|||
import {
|
||||
hasUrlContextTool,
|
||||
extractYouTubeUrls,
|
||||
hasYouTubeVideoParts,
|
||||
appendYouTubeVideoParts,
|
||||
DEFAULT_MAX_YOUTUBE_PARTS,
|
||||
resolveYouTubeInjectionConfig,
|
||||
|
|
@ -391,6 +392,61 @@ describe('appendYouTubeVideoParts', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('hasYouTubeVideoParts', () => {
|
||||
const youtubeText = 'Summarize https://www.youtube.com/watch?v=dQw4w9WgXcQ for me';
|
||||
|
||||
it('detects the parts appendYouTubeVideoParts actually produces', () => {
|
||||
const content = appendYouTubeVideoParts({
|
||||
enabled: true,
|
||||
text: youtubeText,
|
||||
content: youtubeText,
|
||||
});
|
||||
expect(hasYouTubeVideoParts(content)).toBe(true);
|
||||
});
|
||||
|
||||
it('detects Vertex-shaped parts carrying an explicit mimeType', () => {
|
||||
const content = appendYouTubeVideoParts({
|
||||
enabled: true,
|
||||
text: youtubeText,
|
||||
content: youtubeText,
|
||||
max: 1,
|
||||
mimeType: 'video/mp4',
|
||||
});
|
||||
expect(hasYouTubeVideoParts(content)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for the untouched content of a no-op injection', () => {
|
||||
const text = 'Read https://example.com/article';
|
||||
const content = appendYouTubeVideoParts({ enabled: true, text, content: text });
|
||||
expect(hasYouTubeVideoParts(content)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for string content', () => {
|
||||
expect(hasYouTubeVideoParts(youtubeText)).toBe(false);
|
||||
});
|
||||
|
||||
it('ignores non-YouTube media parts', () => {
|
||||
const content: MessageContentComplex[] = [
|
||||
{ type: ContentTypes.TEXT, text: 'look at this' } as MessageContentComplex,
|
||||
{
|
||||
type: 'media',
|
||||
fileUri: 'https://example.com/clip.mp4',
|
||||
} as unknown as MessageContentComplex,
|
||||
];
|
||||
expect(hasYouTubeVideoParts(content)).toBe(false);
|
||||
});
|
||||
|
||||
it('tolerates malformed parts', () => {
|
||||
const content = [
|
||||
null,
|
||||
undefined,
|
||||
'text',
|
||||
{ type: 'media' },
|
||||
] as unknown as MessageContentComplex[];
|
||||
expect(hasYouTubeVideoParts(content)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveYouTubeInjectionConfig', () => {
|
||||
it('caps Vertex at one video and sets a video/mp4 mimeType', () => {
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -16,6 +16,9 @@ import type { MessageContentComplex } from '@librechat/agents';
|
|||
/** Per-message cap on auto-injected YouTube video parts for Gemini 2.5+ (the API allows up to 10). */
|
||||
export const DEFAULT_MAX_YOUTUBE_PARTS = 5;
|
||||
|
||||
/** Canonical form every extracted YouTube link is normalized to before injection. */
|
||||
const YOUTUBE_WATCH_PREFIX = 'https://www.youtube.com/watch?v=';
|
||||
|
||||
/** A Gemini video-understanding content block (becomes a `fileData` part downstream). */
|
||||
export interface YouTubeVideoPart {
|
||||
type: 'media';
|
||||
|
|
@ -197,7 +200,7 @@ export function extractYouTubeUrls(text?: string | null, max?: number): string[]
|
|||
continue;
|
||||
}
|
||||
seen.add(videoId);
|
||||
urls.push(`https://www.youtube.com/watch?v=${videoId}`);
|
||||
urls.push(`${YOUTUBE_WATCH_PREFIX}${videoId}`);
|
||||
if (urls.length >= limit) {
|
||||
break;
|
||||
}
|
||||
|
|
@ -213,6 +216,25 @@ export function hasUrlContextTool(tools: unknown): boolean {
|
|||
return tools.some((tool) => tool != null && typeof tool === 'object' && 'urlContext' in tool);
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a formatted message carries a YouTube video part produced by
|
||||
* `appendYouTubeVideoParts`. Read after injection so a provider rejection can be attributed to the
|
||||
* video rather than reported as a generic failure — Google returns the same opaque
|
||||
* `INVALID_ARGUMENT` for an over-length or otherwise unreadable video as for unrelated bad input.
|
||||
*/
|
||||
export function hasYouTubeVideoParts(content: string | MessageContentComplex[]): boolean {
|
||||
if (!Array.isArray(content)) {
|
||||
return false;
|
||||
}
|
||||
return content.some((part) => {
|
||||
if (part == null || typeof part !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const { fileUri } = part as { fileUri?: unknown };
|
||||
return typeof fileUri === 'string' && fileUri.startsWith(YOUTUBE_WATCH_PREFIX);
|
||||
});
|
||||
}
|
||||
|
||||
function toBaseParts(content: string | MessageContentComplex[]): MessageContentComplex[] {
|
||||
if (Array.isArray(content)) {
|
||||
return content;
|
||||
|
|
|
|||
|
|
@ -2522,6 +2522,10 @@ export enum ErrorTypes {
|
|||
* Google provider does not allow custom tools with built-in tools
|
||||
*/
|
||||
GOOGLE_TOOL_CONFLICT = 'google_tool_conflict',
|
||||
/**
|
||||
* Google provider could not process a linked video (most often longer than the model accepts)
|
||||
*/
|
||||
GOOGLE_VIDEO_UNPROCESSABLE = 'google_video_unprocessable',
|
||||
/**
|
||||
* Invalid Agent Provider (excluded by Admin)
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue