🧯 fix: Return Approved Tool and Generation Error Text Instead of Raw Exceptions (#17508)

* fix: Return approved tool and generation error text instead of raw exceptions

* fix: Redact generation error text without the data-schemas helper

* fix: Redact credentials and emails from shown generation errors

* refactor: Move the Wolfram status policy into packages/api

* test: Fix the Wolfram probe constructor and jest summary match

* test: Match the jest pass count through color codes

* fix: Redact credentials a tool input error echoes from its arguments

* fix: Drop free-text fields from typed generation errors under a content policy

* fix: Keep only the code and numeric fields of typed errors under a content policy
This commit is contained in:
Marco Beretta 2026-10-28 00:39:01 +01:00 • committed by GitHub
parent a0f063b1c7
commit 65d0ba8563
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 804 additions and 71 deletions

View file

@ -1,5 +1,5 @@
const fetch = require('node-fetch');
const { formatToolError } = require('@librechat/api');
const { ToolInputError, formatToolError, formatToolHttpError } = require('@librechat/api');
const { Tool } = require('@librechat/agents/langchain/tools');
const { getEnvironmentVariable } = require('@librechat/agents/langchain/utils/env');
@ -139,7 +139,7 @@ class OpenWeather extends Tool {
const res = await fetch(geocodeUrl);
const data = await res.json();
if (!res.ok || !Array.isArray(data) || data.length === 0) {
throw new Error(`Could not find coordinates for city: ${city}`);
throw new ToolInputError(`Could not find coordinates for city: ${city}`);
}
return { lat: data[0].lat, lon: data[0].lon };
}
@ -147,18 +147,18 @@ class OpenWeather extends Tool {
convertDateToUnix(dateStr) {
const parts = dateStr.split('-');
if (parts.length !== 3) {
throw new Error('Invalid date format. Expected YYYY-MM-DD.');
throw new ToolInputError('Invalid date format. Expected YYYY-MM-DD.');
}
const year = parseInt(parts[0], 10);
const month = parseInt(parts[1], 10);
const day = parseInt(parts[2], 10);
if (isNaN(year) || isNaN(month) || isNaN(day)) {
throw new Error('Invalid date format. Expected YYYY-MM-DD with valid numbers.');
throw new ToolInputError('Invalid date format. Expected YYYY-MM-DD with valid numbers.');
}
const dateObj = new Date(Date.UTC(year, month - 1, day, 0, 0, 0));
if (isNaN(dateObj.getTime())) {
throw new Error('Invalid date provided. Cannot parse into a valid date.');
throw new ToolInputError('Invalid date provided. Cannot parse into a valid date.');
}
return Math.floor(dateObj.getTime() / 1000);
@ -338,17 +338,15 @@ class OpenWeather extends Tool {
const url = `${baseUrl}${endpoint}?${params.toString()}`;
const response = await fetch(url);
const json = await response.json();
if (!response.ok) {
return `Error: OpenWeather API request failed with status ${response.status}: ${
json.message || JSON.stringify(json)
}`;
return formatToolHttpError('OpenWeather', response.status);
}
const json = await response.json();
const roundedJson = roundTemperatures(json);
return JSON.stringify(roundedJson);
} catch (err) {
return formatToolError(err);
return formatToolError(err, 'OpenWeather');
}
}
}

View file

@ -1,6 +1,6 @@
/* eslint-disable no-useless-escape */
const axios = require('axios');
const { logger } = require('@librechat/data-schemas');
const { formatToolError, WOLFRAM_STATUS_TEXT } = require('@librechat/api');
const { Tool } = require('@librechat/agents/langchain/tools');
const wolframJsonSchema = {
@ -59,13 +59,8 @@ class WolframAlphaAPI extends Tool {
}
async fetchRawText(url) {
try {
const response = await axios.get(url, { responseType: 'text' });
return response.data;
} catch (error) {
logger.error('[WolframAlphaAPI] Error fetching raw text:', error);
throw error;
}
const response = await axios.get(url, { responseType: 'text' });
return response.data;
}
getAppId() {
@ -93,13 +88,7 @@ class WolframAlphaAPI extends Tool {
const response = await this.fetchRawText(url);
return response;
} catch (error) {
if (error.response && error.response.data) {
logger.error('[WolframAlphaAPI] Error data:', error);
return error.response.data;
} else {
logger.error('[WolframAlphaAPI] Error querying Wolfram Alpha', error);
return 'There was an error querying Wolfram Alpha.';
}
return formatToolError(error, 'Wolfram|Alpha', WOLFRAM_STATUS_TEXT);
}
}
}

View file

@ -317,7 +317,7 @@ describe('OpenWeather Tool', () => {
Promise.resolve({
ok: false,
status: 404,
json: async () => ({ message: 'Not found' }),
json: async () => ({ message: 'Not found for PRIVATE-PROVIDER-DETAIL' }),
}),
);
@ -325,7 +325,8 @@ describe('OpenWeather Tool', () => {
action: 'current_forecast',
city: 'Knoxville, Tennessee',
});
expect(result).toMatch(/Error: OpenWeather API request failed with status 404: Not found/);
expect(result).toBe('Error: OpenWeather found no result for this request (status 404).');
expect(result).not.toContain('PRIVATE-PROVIDER-DETAIL');
});
test('invalid date format returns an error', async () => {
@ -340,30 +341,23 @@ describe('OpenWeather Tool', () => {
return Promise.reject('Unexpected fetch call for geocoding');
});
// Mock timestamp API response
fetch.mockImplementationOnce((url) => {
if (url.includes('onecall/timemachine')) {
throw new Error('Invalid date format. Expected YYYY-MM-DD.');
}
return Promise.reject('Unexpected fetch call');
});
const result = await tool.call({
action: 'timestamp',
city: 'Knoxville, Tennessee',
date: '03-04-2020', // Wrong format
date: '2020/03/04', // Wrong format
});
expect(result).toMatch(/Error: Invalid date format. Expected YYYY-MM-DD./);
});
test('a failed request does not echo the API key to the model', async () => {
test('a failed request returns approved text, not the exception message', async () => {
fetch.mockImplementationOnce((url) =>
Promise.reject(new Error(`request to ${url} failed, reason: getaddrinfo ENOTFOUND`)),
);
const result = await tool.call({ action: 'current_forecast', lat: 35.96, lon: -83.92 });
expect(result).toMatch(/^Error: request to .*appid=\[REDACTED\]/);
expect(result).toBe('Error: The tool request failed.');
expect(result).not.toContain('test-api-key');
expect(result).not.toContain('ENOTFOUND');
});
});

View file

@ -41,6 +41,7 @@ const mockGenerationJobManager = {
jest.mock('@librechat/data-schemas', () => ({
logger: mockLogger,
redactMessage: jest.requireActual('@librechat/data-schemas').redactMessage,
tenantStorage: {
run: (...args) => mockTenantStorageRun(...args),
},
@ -50,6 +51,8 @@ jest.mock('@librechat/api', () => ({
savePrivateTextMessage: (save, _req, ...args) => save(...args),
savePrivateTextErrorTurn: (...args) =>
jest.requireActual('@librechat/api').savePrivateTextErrorTurn(...args),
getUserFacingGenerationError: (...args) =>
jest.requireActual('@librechat/api').getUserFacingGenerationError(...args),
stampPreliminaryPrivateTextMessage: (_req, message) => message,
getAgentErrorMetadata: (...args) =>
jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),

View file

@ -253,6 +253,7 @@ const mockCleanupMCPRequestContextForReq = jest.fn(async (req) => {
jest.mock('@librechat/data-schemas', () => ({
logger: mockLogger,
createChatExpirationDate: jest.requireActual('@librechat/data-schemas').createChatExpirationDate,
redactMessage: jest.requireActual('@librechat/data-schemas').redactMessage,
}));
jest.mock('@librechat/api', () => ({
@ -262,6 +263,8 @@ jest.mock('@librechat/api', () => ({
savePrivateTextMessage: (save, _req, ...args) => save(...args),
savePrivateTextErrorTurn: (...args) =>
jest.requireActual('@librechat/api').savePrivateTextErrorTurn(...args),
getUserFacingGenerationError: (...args) =>
jest.requireActual('@librechat/api').getUserFacingGenerationError(...args),
stampPreliminaryPrivateTextMessage: (_req, message) => message,
getSteerRecoveryFailure: jest.requireActual(
'../../../../../packages/api/src/stream/SteerRecovery',
@ -2801,6 +2804,137 @@ describe('ResumableAgentController resume metadata', () => {
expect(JSON.stringify(mockLogger.error.mock.calls)).not.toContain(signedUrl);
});
it('publishes and persists the fallback for a raw generation error under a content policy', async () => {
const privateValue = 'PRIVATE-PROMPT-ECHO';
const req = {
user: { id: 'user-123' },
body: {
text: 'Fail during generation.',
messageId: 'user-msg',
conversationId: 'conversation-123',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {
filters: {
messages: {
pii: {
starterPatterns: [],
customPatterns: [{ id: 'secret', label: 'secret', regex: 'NEVER-MATCHES-XYZ' }],
},
},
},
},
};
const client = {
options: {},
sendMessage: jest.fn().mockRejectedValue(new Error(`Provider echoed ${privateValue}`)),
};
const initializeClient = jest.fn().mockResolvedValue({ client });
await AgentController(req, createResumableResponse(), jest.fn(), initializeClient, null);
await nextTick();
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
'conversation-123',
'Generation failed',
1000,
expect.objectContaining({ beforeErrorPublication: expect.any(Function) }),
);
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.objectContaining({ userId: 'user-123' }),
expect.objectContaining({ error: true, text: 'Generation failed' }),
expect.any(Object),
);
expect(JSON.stringify(mockGenerationJobManager.completeJob.mock.calls)).not.toContain(
privateValue,
);
expect(JSON.stringify(mockSaveMessage.mock.calls)).not.toContain(privateValue);
});
it('redacts credentials and emails from a generation error published and persisted without a content policy', async () => {
const secrets = [
'sk-proj-PRIVATEKEY123',
'PRIVATEHEADER123',
'PRIVATESIG123',
'PRIVATEAUTH123',
'private.user@example.com',
];
const rawMessage = [
'Incorrect API key provided: sk-proj-PRIVATEKEY123;',
'x-api-key: PRIVATEHEADER123;',
'token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJQUklWQVRFIn0.PRIVATESIG123;',
'owner private.user@example.com;',
'Authorization: Basic PRIVATEAUTH123',
].join(' ');
const req = {
user: { id: 'user-123' },
body: {
text: 'Fail during generation.',
messageId: 'user-msg',
conversationId: 'conversation-123',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
const client = {
options: {},
sendMessage: jest.fn().mockRejectedValue(new Error(rawMessage)),
};
const initializeClient = jest.fn().mockResolvedValue({ client });
await AgentController(req, createResumableResponse(), jest.fn(), initializeClient, null);
await nextTick();
const [, published] = mockGenerationJobManager.completeJob.mock.calls[0];
expect(published).toContain('Incorrect API key provided');
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.objectContaining({ userId: 'user-123' }),
expect.objectContaining({ error: true, text: published }),
expect.any(Object),
);
for (const secret of secrets) {
expect(JSON.stringify(mockGenerationJobManager.completeJob.mock.calls)).not.toContain(secret);
expect(JSON.stringify(mockSaveMessage.mock.calls)).not.toContain(secret);
}
});
it('keeps a typed generation error under a content policy so the client can localize it', async () => {
const typedError = JSON.stringify({ type: 'no_user_key' });
const req = {
user: { id: 'user-123' },
body: {
text: 'Fail during generation.',
messageId: 'user-msg',
conversationId: 'conversation-123',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {
filters: {
messages: {
pii: {
starterPatterns: [],
customPatterns: [{ id: 'secret', label: 'secret', regex: 'NEVER-MATCHES-XYZ' }],
},
},
},
},
};
const client = {
options: {},
sendMessage: jest.fn().mockRejectedValue(new Error(typedError)),
};
const initializeClient = jest.fn().mockResolvedValue({ client });
await AgentController(req, createResumableResponse(), jest.fn(), initializeClient, null);
await nextTick();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.objectContaining({ userId: 'user-123' }),
expect.objectContaining({ error: true, text: typedError }),
expect.any(Object),
);
});
it('publishes a safe image storage failure during generation and releases the request', async () => {
const signedUrl =
'https://minio.example.com/bucket/image.png?X-Amz-Credential=secret&X-Amz-Signature=signed';

View file

@ -60,6 +60,7 @@ const {
resolveFailedTurnContent,
savePrivateTextMessage,
savePrivateTextErrorTurn,
getUserFacingGenerationError,
stampPreliminaryPrivateTextMessage,
announceReply,
announceErrorTurn,
@ -3323,7 +3324,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
sendGenerationJson(
res,
500,
{ error: error.message || 'Failed to start generation' },
{
error: getUserFacingGenerationError(error, req.config, 'Failed to start generation'),
},
generationProtocolVersion,
);
}
@ -3381,7 +3384,11 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// abortJob already handled emitDone and completeJob
} else {
logger.error(`[ResumableAgentController] Generation error for ${streamId}:`, error);
const generationError = error.message || 'Generation failed';
const generationError = getUserFacingGenerationError(
error,
req.config,
'Generation failed',
);
try {
// completeJob first wins running -> error and atomically parks
// steers, then publishes. A competing abort/pause emits nothing.
@ -3454,22 +3461,24 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
sendGenerationJson(
res,
500,
{ error: err.message || 'Failed to start generation' },
{ error: getUserFacingGenerationError(err, req.config, 'Failed to start generation') },
generationProtocolVersion,
);
}
let errorFinalized = false;
if (!pausePersistenceFailed) {
errorFinalized =
(await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(
(completeErr) => {
logger.warn(
'[ResumableAgentController] completeJob failed during background-error cleanup',
completeErr,
);
return false;
},
)) === true;
(await GenerationJobManager.completeJob(
streamId,
getUserFacingGenerationError(err, req.config, 'Generation failed'),
jobCreatedAt,
).catch((completeErr) => {
logger.warn(
'[ResumableAgentController] completeJob failed during background-error cleanup',
completeErr,
);
return false;
})) === true;
}
if (
(errorFinalized || (pausePersistenceFailed && pausePersistenceFailureFinalized)) &&
@ -3570,7 +3579,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
sendGenerationJson(
res,
500,
{ error: error.message || 'Failed to start generation' },
{
error: getUserFacingGenerationError(error, req.config, 'Failed to start generation'),
},
generationProtocolVersion,
);
}
@ -3597,7 +3608,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
if (jobCreatedAt != null) {
const initializationError = initializationFailure
? JSON.stringify(initializationFailure)
: error.message || 'Failed to start generation';
: getUserFacingGenerationError(error, req.config, 'Failed to start generation');
const persistInitializationError = shouldPersistCodeWorkspaceInitializationError({
streamStarted,
isNewConversation: isNewConvo,

View file

@ -1,4 +1,4 @@
const { logger, redactMessage, getTenantId } = require('@librechat/data-schemas');
const { logger, getTenantId } = require('@librechat/data-schemas');
const { tool: toolFn, DynamicStructuredTool } = require('@librechat/agents/langchain/tools');
const {
sleep,
@ -77,6 +77,7 @@ const {
appendAssistantMCPAppArtifact,
resolveMCPClientCapabilityProfile,
getOutboundTimeoutMs,
getToolErrorText,
} = require('@librechat/api');
const {
Time,
@ -732,7 +733,7 @@ async function processRequiredActions(client, requiredActions) {
const output = getSafeRequiredActionOutput(
client,
currentAction,
`Error processing tool ${currentAction.tool}: ${redactMessage(error.message, 256)}`,
`Error processing tool ${currentAction.tool}: ${getToolErrorText(error)}`,
);
return {
tool_call_id: currentAction.toolCallId,

View file

@ -193,7 +193,7 @@ const {
} = require('../ToolService');
const { createOnSearchResults } = require('~/server/services/Tools/search');
const { reinitMCPServer } = require('~/server/services/Tools/mcp');
const { ContentFilterError, PENDING_STALE_MS } = require('@librechat/api');
const { ContentFilterError, PENDING_STALE_MS, ToolInputError } = require('@librechat/api');
/** Role document shape `checkAccess` reads; all three role-gated tools granted. */
function buildRole(overrides = {}) {
@ -689,6 +689,46 @@ describe('ToolService - Action Capability Gating', () => {
expect(output).not.toContain('bearer_header');
});
it('returns approved tool error text instead of a raw exception message', async () => {
const rawMessage =
'connect ECONNREFUSED 10.0.0.5:443 for https://api.example.test/?q=PRIVATE';
mockLoadToolsUtil.mockResolvedValue({
loadedTools: [
{ name: 'safe_tool', _call: jest.fn().mockRejectedValue(new Error(rawMessage)) },
],
toolContextMap: {},
});
const result = await processRequiredActions(buildClient(undefined), [buildAction()]);
expect(result.tool_outputs).toEqual([
{
tool_call_id: 'call_1',
output: 'Error processing tool safe_tool: The tool request failed.',
},
]);
});
it('keeps the message of a tool input error the tool wrote for the model', async () => {
mockLoadToolsUtil.mockResolvedValue({
loadedTools: [
{
name: 'safe_tool',
_call: jest
.fn()
.mockRejectedValue(new ToolInputError('Could not find coordinates for city: Rome')),
},
],
toolContextMap: {},
});
const result = await processRequiredActions(buildClient(undefined), [buildAction()]);
expect(result.tool_outputs[0].output).toBe(
'Error processing tool safe_tool: Could not find coordinates for city: Rome',
);
});
it('blocks persisted Assistant action metadata before required-action execution', async () => {
const actionToolName = `get_weather${actionDelimiter}api_example_com`;
const filters = {

View file

@ -0,0 +1,52 @@
import { resolve } from 'node:path';
import { expect, test } from '@playwright/test';
import { run, repoRoot, inOneProject } from './lint.helpers';
/**
* A generation that fails outside the model run publishes its error and persists it on the failed
* turn's error row. These run the resumable controller's own suite for those paths: the real
* controller and `saveErrorTurn`, with the job store and message writes captured.
*/
const apiRoot = resolve(repoRoot, 'api');
function runControllerTests(pattern: string): void {
const result = run(
'npx',
[
'jest',
'server/controllers/agents/__tests__/request.resumeMetadata.spec.js',
'-t',
pattern,
'--maxWorkers=1',
],
{ cwd: apiRoot },
);
expect(result.status, result.output).toBe(0);
/** Jest may color the summary, so match the count alone. */
expect(result.output).toMatch(/(?<!\d)1 passed/);
}
test.describe('generation error text', () => {
test.beforeEach(() => inOneProject());
test('under a content policy a raw generation error is published and persisted as the fallback @scenario:generation-error-withheld-under-policy', () => {
test.setTimeout(300_000);
runControllerTests('publishes and persists the fallback for a raw generation error');
});
test('under a content policy a typed generation error keeps its code for the client @scenario:typed-generation-error-kept', () => {
test.setTimeout(300_000);
runControllerTests('keeps a typed generation error under a content policy');
});
test('without a content policy a host-authored generation error keeps its wording @scenario:generation-error-kept-without-policy', () => {
test.setTimeout(300_000);
runControllerTests('publishes a safe image storage failure during generation');
});
test('without a content policy credentials and emails never reach the published or persisted error @scenario:generation-error-secrets-redacted', () => {
test.setTimeout(300_000);
runControllerTests('redacts credentials and emails from a generation error');
});
});

View file

@ -0,0 +1,124 @@
import { resolve } from 'node:path';
import { expect, test } from '@playwright/test';
import { run, repoRoot, inOneProject } from './lint.helpers';
/**
* A tool result lands in the conversation, so a failing tool may only hand the model text the
* tool wrote for it. The probes load the real tools from `api/` and replace only the transport
* (node-fetch, axios) so the upstream failure is deterministic and never leaves the machine.
*/
const apiRoot = resolve(repoRoot, 'api');
const SECRET = 'PRIVATE-PROVIDER-DETAIL';
function probe(source: string): Record<string, string> {
const result = run('node', ['-e', source], { cwd: apiRoot });
expect(result.status, result.output).toBe(0);
const marker = result.stdout.lastIndexOf('RESULT');
expect(marker, result.output).toBeGreaterThanOrEqual(0);
return JSON.parse(result.stdout.slice(marker + 'RESULT'.length));
}
const OPEN_WEATHER_PROBE = `
process.env.OPENWEATHER_API_KEY = 'probe-api-key';
const fetchPath = require.resolve('node-fetch');
let respond;
require.cache[fetchPath] = {
id: fetchPath,
filename: fetchPath,
loaded: true,
exports: (url) => respond(url),
};
const OpenWeather = require('./app/clients/tools/structured/OpenWeather');
(async () => {
const tool = new OpenWeather();
const out = {};
respond = (url) => Promise.reject(new Error('request to ' + url + ' failed, reason: ${SECRET}'));
out.network = await tool._call({ action: 'current_forecast', lat: 1, lon: 2 });
respond = () => Promise.resolve({ ok: false, status: 401, json: async () => ({ message: '${SECRET}' }) });
out.unauthorized = await tool._call({ action: 'current_forecast', lat: 1, lon: 2 });
respond = () => Promise.resolve({ ok: true, json: async () => [] });
out.city = await tool._call({ action: 'current_forecast', city: 'Atlantis' });
out.secretCity = await tool._call({ action: 'current_forecast', city: 'sk-proj-PRIVATEKEY123 owner@example.com' });
out.date = await tool._call({ action: 'timestamp', lat: 1, lon: 2, date: '2020/03/04' });
process.stdout.write('RESULT' + JSON.stringify(out));
process.exit(0);
})().catch((error) => { console.error(error); process.exit(1); });
`;
const WOLFRAM_PROBE = `
process.env.WOLFRAM_APP_ID = 'probe-app-id';
const axios = require('axios');
const WolframAlphaAPI = require('./app/clients/tools/structured/Wolfram');
const httpError = (status) =>
Object.assign(new Error('Request failed with status code ' + status), {
response: { status, data: 'Wolfram|Alpha did not understand ${SECRET}' },
});
(async () => {
const tool = new WolframAlphaAPI({});
const out = {};
axios.get = () => Promise.reject(httpError(501));
out.uninterpretable = await tool._call({ input: '${SECRET}' });
axios.get = () => Promise.reject(httpError(503));
out.unavailable = await tool._call({ input: 'integrate x' });
axios.get = () => Promise.reject(new Error('connect ECONNREFUSED ${SECRET}'));
out.network = await tool._call({ input: 'integrate x' });
axios.get = () => Promise.resolve({ data: 'x^2/2' });
out.ok = await tool._call({ input: 'integrate x' });
process.stdout.write('RESULT' + JSON.stringify(out));
process.exit(0);
})().catch((error) => { console.error(error); process.exit(1); });
`;
test.describe('approved tool error text', () => {
test.beforeEach(() => inOneProject());
test('a failing OpenWeather call returns approved text and keeps its own input guidance @scenario:openweather-approved-error-text', () => {
test.setTimeout(120_000);
const out = probe(OPEN_WEATHER_PROBE);
expect(out.network).toBe('Error: The tool request failed.');
expect(out.unauthorized).toBe(
'Error: OpenWeather rejected the configured credentials (status 401).',
);
expect(out.city).toBe('Error: Could not find coordinates for city: Atlantis');
expect(out.date).toBe('Error: Invalid date format. Expected YYYY-MM-DD.');
expect(out.secretCity).toMatch(/^Error: Could not find coordinates for city: /);
expect(out.secretCity).not.toContain('PRIVATEKEY123');
expect(out.secretCity).not.toContain('owner@example.com');
expect(JSON.stringify(out)).not.toContain(SECRET);
expect(JSON.stringify(out)).not.toContain('probe-api-key');
});
test('a failing Wolfram|Alpha call returns approved text, not the provider body @scenario:wolfram-approved-error-text', () => {
test.setTimeout(120_000);
const out = probe(WOLFRAM_PROBE);
expect(out.uninterpretable).toMatch(/^Error: Wolfram\|Alpha could not interpret the input\./);
expect(out.unavailable).toBe(
'Error: Wolfram|Alpha is unavailable; try again later (status 503).',
);
expect(out.network).toBe('Error: The tool request failed.');
expect(out.ok).toBe('x^2/2');
expect(JSON.stringify(out)).not.toContain(SECRET);
expect(JSON.stringify(out)).not.toContain('probe-app-id');
});
test('an Assistants required action whose tool throws submits approved text @scenario:assistant-tool-error-approved-text', () => {
test.setTimeout(300_000);
const result = run(
'npx',
[
'jest',
'server/services/__tests__/ToolService.spec.js',
'-t',
'approved tool error text|tool input error the tool wrote',
'--maxWorkers=1',
],
{ cwd: apiRoot },
);
expect(result.status, result.output).toBe(0);
/** Jest may color the summary, so match the count alone. */
expect(result.output).toMatch(/(?<!\d)2 passed/);
});
});

View file

@ -1,5 +1,11 @@
import { errors } from 'undici';
import { createTerminalRunErrorObserver, getUpstreamModelErrorMetadata } from './terminal';
import { ErrorTypes, ViolationTypes } from 'librechat-data-provider';
import {
getUpstreamModelErrorMetadata,
getUserFacingGenerationError,
createTerminalRunErrorObserver,
} from './terminal';
import { ContentFilterError } from '../../middleware/contentFilter';
describe('terminal agent-run error logging', () => {
it('logs stable upstream metadata and deterministic trace correlation', () => {
@ -373,3 +379,123 @@ describe('terminal agent-run error logging', () => {
});
});
});
describe('getUserFacingGenerationError', () => {
const protectedConfig = {
filters: {
messages: {
pii: {
starterPatterns: [],
customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE' }],
},
},
},
};
it('replaces a raw exception message with the fallback while a content policy is active', () => {
const error = new Error('400 Bad Request: prompt "PRIVATE-PROMPT" was rejected');
expect(getUserFacingGenerationError(error, protectedConfig, 'Generation failed')).toBe(
'Generation failed',
);
});
it('keeps the message with URLs redacted when no content policy is active', () => {
const error = new Error('request to https://api.example.test/?api_key=secret-key failed');
expect(getUserFacingGenerationError(error, {}, 'Generation failed')).toBe(
'request to https://api.example.test/[redacted] failed',
);
expect(getUserFacingGenerationError(new Error(' '), undefined, 'Generation failed')).toBe(
'Generation failed',
);
expect(getUserFacingGenerationError('PRIVATE-STRING', undefined, 'Generation failed')).toBe(
'Generation failed',
);
});
it.each([
['an sk- key', 'Incorrect API key provided: sk-proj-PRIVATEKEY123', 'PRIVATEKEY123'],
['an x-api-key header', 'upstream rejected x-api-key: PRIVATEHEADER123', 'PRIVATEHEADER123'],
[
'a JWT',
'token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJQUklWQVRFIn0.PRIVATESIG123 expired',
'PRIVATESIG123',
],
['an Authorization header', 'sent Authorization: Basic PRIVATEAUTH123', 'PRIVATEAUTH123'],
['an email address', 'no account for private.user@example.com', 'private.user@example.com'],
])('never returns %s without a content policy', (_label, message, secret) => {
const text = getUserFacingGenerationError(new Error(message), {}, 'Generation failed');
expect(text).not.toContain(secret);
expect(text).not.toBe('Generation failed');
});
it('keeps a typed payload the client maps to localized copy under protection', () => {
const typed = JSON.stringify({ type: ErrorTypes.NO_USER_KEY });
const violation = JSON.stringify({ type: ViolationTypes.TOKEN_BALANCE, balance: 0 });
expect(getUserFacingGenerationError(new Error(typed), protectedConfig, 'x')).toBe(typed);
expect(getUserFacingGenerationError(new Error(violation), protectedConfig, 'x')).toBe(
violation,
);
});
it('keeps only the code and numeric fields of a typed payload under a content policy', () => {
const typed = JSON.stringify({
type: ViolationTypes.TOKEN_BALANCE,
balance: 0,
tokenCost: 120,
message: 'PRIVATE-PROMPT',
prompt: 'PRIVATE-FIELD',
nested: { value: 'PRIVATE-NESTED' },
});
const text = getUserFacingGenerationError(new Error(typed), protectedConfig, 'x');
expect(JSON.parse(text)).toEqual({
type: ViolationTypes.TOKEN_BALANCE,
balance: 0,
tokenCost: 120,
});
});
it('redacts credentials in a typed payload without a content policy', () => {
const typed = JSON.stringify({ type: ErrorTypes.INVALID_REQUEST, info: 'key sk-PRIVATEKEY99' });
const text = getUserFacingGenerationError(new Error(typed), {}, 'x');
expect(JSON.parse(text).type).toBe(ErrorTypes.INVALID_REQUEST);
expect(text).not.toContain('PRIVATEKEY99');
});
it('treats JSON with an unknown type as an ordinary message', () => {
const untyped = JSON.stringify({ type: 'PRIVATE-TYPE', prompt: 'PRIVATE-PROMPT' });
expect(getUserFacingGenerationError(new Error(untyped), protectedConfig, 'fallback')).toBe(
'fallback',
);
expect(getUserFacingGenerationError(new Error('[1,2]'), protectedConfig, 'fallback')).toBe(
'fallback',
);
expect(getUserFacingGenerationError(new Error('{not json'), protectedConfig, 'fallback')).toBe(
'fallback',
);
});
it('classifies a LangChain-coded provider failure under protection', () => {
const error = Object.assign(new Error('rate limited PRIVATE-DETAIL'), {
lc_error_code: 'MODEL_RATE_LIMIT',
});
expect(getUserFacingGenerationError(error, protectedConfig, 'Generation failed')).toBe(
JSON.stringify({ type: ErrorTypes.MODEL_RATE_LIMIT }),
);
});
it('keeps the host-authored message of a content policy block under protection', () => {
const error = new ContentFilterError({
detectorId: 'pii-pattern',
ruleId: 'bearer_header',
label: 'Bearer token',
source: 'file',
field: 'content',
provenance: 'tool',
fragmentId: 'generated-file',
fragmentPath: '/content',
});
expect(
getUserFacingGenerationError(new Error('wrapped', { cause: error }), protectedConfig, 'x'),
).toBe('Submitted content contains a Bearer token. Remove it and try again.');
});
});

View file

@ -1,4 +1,6 @@
import { ErrorTypes } from 'librechat-data-provider';
import { redactMessage } from '@librechat/data-schemas';
import { ErrorTypes, ViolationTypes } from 'librechat-data-provider';
import type { FiltersConfig, MessageFilterPiiConfig } from 'librechat-data-provider';
import type { SafeErrorMetadata } from '../../utils/errors';
import type { ModelErrorTrackerCallback } from './tracker';
import {
@ -7,7 +9,9 @@ import {
resolveModelStreamError,
getProviderErrorMessage,
} from '../errors';
import { getSafeErrorMetadata, isOwnedAbortError } from '../../utils/errors';
import { getSafeErrorText, getSafeErrorMetadata, isOwnedAbortError } from '../../utils/errors';
import { hasModelBoundContentProtection } from '../../middleware/modelBoundContent';
import { getContentFilterError } from '../../middleware/contentFilter';
import { traceIdForMessage } from '../../langfuse/trace';
import { createModelErrorTracker } from './tracker';
@ -148,3 +152,95 @@ export function createTerminalRunErrorObserver({
},
});
}
const TYPED_ERROR_CODES: ReadonlySet<string> = new Set<string>([
...Object.values(ErrorTypes),
...Object.values(ViolationTypes),
]);
type TypedErrorPayload = Record<string, unknown> & { type: string };
/** A host-thrown `{"type": ...}` payload whose type the client maps to localized copy. */
function parseTypedErrorMessage(message: string): TypedErrorPayload | undefined {
if (!message.trimStart().startsWith('{')) {
return undefined;
}
try {
const parsed: unknown = JSON.parse(message);
if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {
return undefined;
}
const type = (parsed as { type?: unknown }).type;
return typeof type === 'string' && TYPED_ERROR_CODES.has(type)
? (parsed as TypedErrorPayload)
: undefined;
} catch {
return undefined;
}
}
/**
* The typed payload as a reader may see it: credentials and emails are always redacted, and while
* a content policy is active only the code and its numeric and boolean fields (a balance, a limit)
* remain, since any string field may carry provider or submitted content.
*/
function getSafeTypedErrorText(
message: string,
payload: TypedErrorPayload,
protectionEnabled: boolean,
): string {
if (!protectionEnabled) {
return redactMessage(message);
}
const safe: Record<string, unknown> = { type: payload.type };
for (const [key, value] of Object.entries(payload)) {
if ((typeof value === 'number' && Number.isFinite(value)) || typeof value === 'boolean') {
safe[key] = value;
}
}
return JSON.stringify(safe);
}
/** The deployment config that decides whether a failure may show its own text. */
export interface GenerationErrorConfig {
readonly filters?: FiltersConfig | null;
readonly messageFilter?: { readonly pii?: MessageFilterPiiConfig | null } | null;
}
/**
* The error text a failed generation publishes and persists on its error row, under the same rule
* the run's content error parts follow. Host-authored outcomes keep their wording: a typed
* `{"type": ...}` payload (redacted, and reduced to its code and numeric fields under a content policy), a LangChain-classified provider failure, or a content policy block.
* Any other exception may echo submitted content, so while a content policy is active the turn
* carries `fallback`; otherwise it carries the message with URLs, credentials (API keys, tokens, auth headers) and email
* addresses redacted. The exception
* itself reaches the logs only through `getSafeErrorMetadata`.
*/
export function getUserFacingGenerationError(
error: unknown,
appConfig: GenerationErrorConfig | null | undefined,
fallback: string,
): string {
const policyError = getContentFilterError(error);
if (policyError != null) {
return policyError.message;
}
const message = error instanceof Error ? error.message : undefined;
const protectionEnabled = hasModelBoundContentProtection(
appConfig?.filters,
appConfig?.messageFilter?.pii,
);
const typed = message != null ? parseTypedErrorMessage(message) : undefined;
if (message != null && typed != null) {
return getSafeTypedErrorText(message, typed, protectionEnabled);
}
const classified = safelyResolveLangChainError(error);
if (classified != null) {
return classified;
}
if (message == null || protectionEnabled) {
return fallback;
}
const trimmed = message.trim();
return trimmed === '' ? fallback : redactMessage(getSafeErrorText(trimmed));
}

View file

@ -1,6 +1,17 @@
import { logger } from '@librechat/data-schemas';
import { AuthType, EToolResources } from 'librechat-data-provider';
import type { TPlugin } from 'librechat-data-provider';
import { filterUniquePlugins, checkPluginAuth, getToolkitKey, formatToolError } from './format';
import {
ToolInputError,
getToolkitKey,
checkPluginAuth,
formatToolError,
getToolErrorText,
filterUniquePlugins,
formatToolHttpError,
TOOL_ERROR_FALLBACK,
WOLFRAM_STATUS_TEXT,
} from './format';
describe('format.ts helper functions', () => {
describe('filterUniquePlugins', () => {
@ -349,20 +360,91 @@ describe('format.ts helper functions', () => {
});
describe('formatToolError', () => {
it('redacts a key carried in the failed request URL', () => {
it('returns the fallback instead of a raw exception message', () => {
const error = new Error(
'request to https://api.example.test/data?lat=1&appid=secret-key failed, reason: ENOTFOUND',
);
expect(formatToolError(error)).toBe(
'Error: request to https://api.example.test/data?lat=1&appid=[REDACTED] failed, reason: ENOTFOUND',
expect(formatToolError(error, 'Example')).toBe(`Error: ${TOOL_ERROR_FALLBACK}`);
expect(formatToolError('timeout')).toBe(`Error: ${TOOL_ERROR_FALLBACK}`);
});
it('keeps the message of a tool input error', () => {
expect(formatToolError(new ToolInputError('Could not find coordinates for city: Rome'))).toBe(
'Error: Could not find coordinates for city: Rome',
);
});
it('keeps a plain message and stringifies non-errors', () => {
expect(formatToolError(new Error('Could not find coordinates for city: Rome'))).toBe(
'Error: Could not find coordinates for city: Rome',
it('redacts credentials and emails a tool input error echoes from its arguments', () => {
const text = formatToolError(
new ToolInputError(
'Could not find coordinates for city: sk-proj-PRIVATEKEY123 private.user@example.com',
),
);
expect(text).toContain('Could not find coordinates for city:');
expect(text).not.toContain('PRIVATEKEY123');
expect(text).not.toContain('private.user@example.com');
expect(getToolErrorText(new ToolInputError('key=PRIVATEQUERY123 rejected'))).not.toContain(
'PRIVATEQUERY123',
);
});
it('logs only safe metadata for an unapproved failure', () => {
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
const error = Object.assign(new Error('upstream said PRIVATE-DETAIL'), {
code: 'ECONNRESET',
});
formatToolError(error, 'Example');
expect(warn).toHaveBeenCalledWith('[Example] Tool request failed', { type: 'Error' });
expect(JSON.stringify(warn.mock.calls)).not.toContain('PRIVATE-DETAIL');
warn.mockRestore();
});
it('words an HTTP failure from its status, never its response body', () => {
const httpError = (status: number) =>
Object.assign(new Error('Request failed'), {
response: { status, data: 'PRIVATE-PROVIDER-BODY' },
});
expect(formatToolError(httpError(503), 'Wolfram|Alpha', WOLFRAM_STATUS_TEXT)).toBe(
'Error: Wolfram|Alpha is unavailable; try again later (status 503).',
);
expect(formatToolError(httpError(501), 'Wolfram|Alpha', WOLFRAM_STATUS_TEXT)).toBe(
`Error: ${WOLFRAM_STATUS_TEXT[501]}`,
);
expect(formatToolError(httpError(404), 'Example')).not.toContain('PRIVATE-PROVIDER-BODY');
});
});
describe('getToolErrorText', () => {
it('approves only tool input errors', () => {
expect(getToolErrorText(new ToolInputError('Invalid date format.'))).toBe(
'Invalid date format.',
);
expect(getToolErrorText(new Error('socket hang up'))).toBe(TOOL_ERROR_FALLBACK);
expect(getToolErrorText({ message: 'not an error' })).toBe(TOOL_ERROR_FALLBACK);
});
});
describe('formatToolHttpError', () => {
it.each([
[401, 'Error: Service rejected the configured credentials (status 401).'],
[404, 'Error: Service found no result for this request (status 404).'],
[429, 'Error: Service is rate limiting requests; try again later (status 429).'],
[
400,
'Error: Service rejected the request parameters; check them and try again (status 400).',
],
[503, 'Error: Service is unavailable; try again later (status 503).'],
])('words status %i without provider text', (status, expected) => {
expect(formatToolHttpError('Service', status)).toBe(expected);
});
it('prefers a tool-specific wording for a documented status', () => {
expect(formatToolHttpError('Service', 501, { 501: 'Rephrase the input.' })).toBe(
'Error: Rephrase the input.',
);
expect(formatToolError('timeout')).toBe('Error: timeout');
});
});
});

View file

@ -1,6 +1,7 @@
import { redactMessage } from '@librechat/data-schemas';
import { logger, redactMessage } from '@librechat/data-schemas';
import { AuthType, EToolResources } from 'librechat-data-provider';
import type { TPlugin } from 'librechat-data-provider';
import { getSafeErrorMetadata } from '~/utils/errors';
/**
* Filters out duplicate plugins from the list of plugins.
@ -84,8 +85,90 @@ export function getToolkitKey({
return toolkitKey;
}
/** Error text a tool returns to the model, with credentials in request URLs and headers redacted. */
export function formatToolError(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return `Error: ${redactMessage(message)}`;
/** Tool error text for any failure the tool did not word for the model itself. */
export const TOOL_ERROR_FALLBACK = 'The tool request failed.';
/**
* A tool failure whose message the tool wrote for the model: input guidance such as an unknown
* city or a malformed date. It is the only exception text a tool result carries; provider,
* network and runtime messages stay in the logs.
*/
export class ToolInputError extends Error {
constructor(message: string) {
super(message);
this.name = 'ToolInputError';
Object.setPrototypeOf(this, ToolInputError.prototype);
}
}
/**
* The approved text for a tool failure: a `ToolInputError` message with credentials and email
* addresses redacted (it may echo a tool argument), otherwise the fallback.
*/
export function getToolErrorText(error: unknown): string {
return error instanceof ToolInputError ? redactMessage(error.message) : TOOL_ERROR_FALLBACK;
}
/** Approved wording for an upstream HTTP status, keyed by the status class a model can act on. */
function describeToolHttpStatus(status: number): string {
if (status === 401 || status === 403) {
return 'rejected the configured credentials';
}
if (status === 404) {
return 'found no result for this request';
}
if (status === 429) {
return 'is rate limiting requests; try again later';
}
if (status >= 500) {
return 'is unavailable; try again later';
}
if (status >= 400) {
return 'rejected the request parameters; check them and try again';
}
return `failed with status ${status}`;
}
/**
* Error text for a tool whose upstream service answered with a failing HTTP status. The response
* body is provider text, so only the status reaches the model; `overrides` lets a tool word a
* status its service documents specifically.
*/
export function formatToolHttpError(
service: string,
status: number,
overrides?: Readonly<Partial<Record<number, string>>>,
): string {
logger.warn(`[${service}] Tool request failed`, { status });
const override = overrides?.[status];
if (override != null) {
return `Error: ${override}`;
}
return `Error: ${service} ${describeToolHttpStatus(status)} (status ${status}).`;
}
/**
* Error text a tool returns to the model. A `ToolInputError` keeps its message; an upstream HTTP
* failure is worded from its status alone (see `formatToolHttpError`); anything else becomes the
* stable fallback, and only its sanitized metadata is logged under `service`.
*/
export function formatToolError(
error: unknown,
service = 'tool',
statusOverrides?: Readonly<Partial<Record<number, string>>>,
): string {
if (error instanceof ToolInputError) {
return `Error: ${getToolErrorText(error)}`;
}
const metadata = getSafeErrorMetadata(error);
if (metadata.status != null && metadata.status >= 400) {
return formatToolHttpError(service, metadata.status, statusOverrides);
}
logger.warn(`[${service}] Tool request failed`, metadata);
return `Error: ${TOOL_ERROR_FALLBACK}`;
}
/** The Wolfram|Alpha LLM API answers 501 when it cannot interpret the input; its body echoes the query. */
export const WOLFRAM_STATUS_TEXT: Readonly<Partial<Record<number, string>>> = {
501: 'Wolfram|Alpha could not interpret the input. Rephrase it as a simpler query, such as single keywords or standard math notation, and try again.',
};