diff --git a/api/app/clients/tools/structured/OpenWeather.js b/api/app/clients/tools/structured/OpenWeather.js index 10629c104e..eec6beecdc 100644 --- a/api/app/clients/tools/structured/OpenWeather.js +++ b/api/app/clients/tools/structured/OpenWeather.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch'); -const { formatToolError } = require('@librechat/api'); +const { ToolInputError, formatToolError, formatToolHttpError } = require('@librechat/api'); const { Tool } = require('@librechat/agents/langchain/tools'); const { getEnvironmentVariable } = require('@librechat/agents/langchain/utils/env'); @@ -139,7 +139,7 @@ class OpenWeather extends Tool { const res = await fetch(geocodeUrl); const data = await res.json(); if (!res.ok || !Array.isArray(data) || data.length === 0) { - throw new Error(`Could not find coordinates for city: ${city}`); + throw new ToolInputError(`Could not find coordinates for city: ${city}`); } return { lat: data[0].lat, lon: data[0].lon }; } @@ -147,18 +147,18 @@ class OpenWeather extends Tool { convertDateToUnix(dateStr) { const parts = dateStr.split('-'); if (parts.length !== 3) { - throw new Error('Invalid date format. Expected YYYY-MM-DD.'); + throw new ToolInputError('Invalid date format. Expected YYYY-MM-DD.'); } const year = parseInt(parts[0], 10); const month = parseInt(parts[1], 10); const day = parseInt(parts[2], 10); if (isNaN(year) || isNaN(month) || isNaN(day)) { - throw new Error('Invalid date format. Expected YYYY-MM-DD with valid numbers.'); + throw new ToolInputError('Invalid date format. Expected YYYY-MM-DD with valid numbers.'); } const dateObj = new Date(Date.UTC(year, month - 1, day, 0, 0, 0)); if (isNaN(dateObj.getTime())) { - throw new Error('Invalid date provided. Cannot parse into a valid date.'); + throw new ToolInputError('Invalid date provided. Cannot parse into a valid date.'); } return Math.floor(dateObj.getTime() / 1000); @@ -338,17 +338,15 @@ class OpenWeather extends Tool { const url = `${baseUrl}${endpoint}?${params.toString()}`; const response = await fetch(url); - const json = await response.json(); if (!response.ok) { - return `Error: OpenWeather API request failed with status ${response.status}: ${ - json.message || JSON.stringify(json) - }`; + return formatToolHttpError('OpenWeather', response.status); } + const json = await response.json(); const roundedJson = roundTemperatures(json); return JSON.stringify(roundedJson); } catch (err) { - return formatToolError(err); + return formatToolError(err, 'OpenWeather'); } } } diff --git a/api/app/clients/tools/structured/Wolfram.js b/api/app/clients/tools/structured/Wolfram.js index 1163c0fa83..3285d08663 100644 --- a/api/app/clients/tools/structured/Wolfram.js +++ b/api/app/clients/tools/structured/Wolfram.js @@ -1,6 +1,6 @@ /* eslint-disable no-useless-escape */ const axios = require('axios'); -const { logger } = require('@librechat/data-schemas'); +const { formatToolError, WOLFRAM_STATUS_TEXT } = require('@librechat/api'); const { Tool } = require('@librechat/agents/langchain/tools'); const wolframJsonSchema = { @@ -59,13 +59,8 @@ class WolframAlphaAPI extends Tool { } async fetchRawText(url) { - try { - const response = await axios.get(url, { responseType: 'text' }); - return response.data; - } catch (error) { - logger.error('[WolframAlphaAPI] Error fetching raw text:', error); - throw error; - } + const response = await axios.get(url, { responseType: 'text' }); + return response.data; } getAppId() { @@ -93,13 +88,7 @@ class WolframAlphaAPI extends Tool { const response = await this.fetchRawText(url); return response; } catch (error) { - if (error.response && error.response.data) { - logger.error('[WolframAlphaAPI] Error data:', error); - return error.response.data; - } else { - logger.error('[WolframAlphaAPI] Error querying Wolfram Alpha', error); - return 'There was an error querying Wolfram Alpha.'; - } + return formatToolError(error, 'Wolfram|Alpha', WOLFRAM_STATUS_TEXT); } } } diff --git a/api/app/clients/tools/structured/specs/openweather.test.js b/api/app/clients/tools/structured/specs/openweather.test.js index e3fc2c3ff1..3a24757466 100644 --- a/api/app/clients/tools/structured/specs/openweather.test.js +++ b/api/app/clients/tools/structured/specs/openweather.test.js @@ -317,7 +317,7 @@ describe('OpenWeather Tool', () => { Promise.resolve({ ok: false, status: 404, - json: async () => ({ message: 'Not found' }), + json: async () => ({ message: 'Not found for PRIVATE-PROVIDER-DETAIL' }), }), ); @@ -325,7 +325,8 @@ describe('OpenWeather Tool', () => { action: 'current_forecast', city: 'Knoxville, Tennessee', }); - expect(result).toMatch(/Error: OpenWeather API request failed with status 404: Not found/); + expect(result).toBe('Error: OpenWeather found no result for this request (status 404).'); + expect(result).not.toContain('PRIVATE-PROVIDER-DETAIL'); }); test('invalid date format returns an error', async () => { @@ -340,30 +341,23 @@ describe('OpenWeather Tool', () => { return Promise.reject('Unexpected fetch call for geocoding'); }); - // Mock timestamp API response - fetch.mockImplementationOnce((url) => { - if (url.includes('onecall/timemachine')) { - throw new Error('Invalid date format. Expected YYYY-MM-DD.'); - } - return Promise.reject('Unexpected fetch call'); - }); - const result = await tool.call({ action: 'timestamp', city: 'Knoxville, Tennessee', - date: '03-04-2020', // Wrong format + date: '2020/03/04', // Wrong format }); expect(result).toMatch(/Error: Invalid date format. Expected YYYY-MM-DD./); }); - test('a failed request does not echo the API key to the model', async () => { + test('a failed request returns approved text, not the exception message', async () => { fetch.mockImplementationOnce((url) => Promise.reject(new Error(`request to ${url} failed, reason: getaddrinfo ENOTFOUND`)), ); const result = await tool.call({ action: 'current_forecast', lat: 35.96, lon: -83.92 }); - expect(result).toMatch(/^Error: request to .*appid=\[REDACTED\]/); + expect(result).toBe('Error: The tool request failed.'); expect(result).not.toContain('test-api-key'); + expect(result).not.toContain('ENOTFOUND'); }); }); diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js index 69492ce29a..6411c5c831 100644 --- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js +++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js @@ -41,6 +41,7 @@ const mockGenerationJobManager = { jest.mock('@librechat/data-schemas', () => ({ logger: mockLogger, + redactMessage: jest.requireActual('@librechat/data-schemas').redactMessage, tenantStorage: { run: (...args) => mockTenantStorageRun(...args), }, @@ -50,6 +51,8 @@ jest.mock('@librechat/api', () => ({ savePrivateTextMessage: (save, _req, ...args) => save(...args), savePrivateTextErrorTurn: (...args) => jest.requireActual('@librechat/api').savePrivateTextErrorTurn(...args), + getUserFacingGenerationError: (...args) => + jest.requireActual('@librechat/api').getUserFacingGenerationError(...args), stampPreliminaryPrivateTextMessage: (_req, message) => message, getAgentErrorMetadata: (...args) => jest.requireActual('@librechat/api').getAgentErrorMetadata(...args), diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index 4e797ab4c2..0cd2f11aee 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -253,6 +253,7 @@ const mockCleanupMCPRequestContextForReq = jest.fn(async (req) => { jest.mock('@librechat/data-schemas', () => ({ logger: mockLogger, createChatExpirationDate: jest.requireActual('@librechat/data-schemas').createChatExpirationDate, + redactMessage: jest.requireActual('@librechat/data-schemas').redactMessage, })); jest.mock('@librechat/api', () => ({ @@ -262,6 +263,8 @@ jest.mock('@librechat/api', () => ({ savePrivateTextMessage: (save, _req, ...args) => save(...args), savePrivateTextErrorTurn: (...args) => jest.requireActual('@librechat/api').savePrivateTextErrorTurn(...args), + getUserFacingGenerationError: (...args) => + jest.requireActual('@librechat/api').getUserFacingGenerationError(...args), stampPreliminaryPrivateTextMessage: (_req, message) => message, getSteerRecoveryFailure: jest.requireActual( '../../../../../packages/api/src/stream/SteerRecovery', @@ -2801,6 +2804,137 @@ describe('ResumableAgentController resume metadata', () => { expect(JSON.stringify(mockLogger.error.mock.calls)).not.toContain(signedUrl); }); + it('publishes and persists the fallback for a raw generation error under a content policy', async () => { + const privateValue = 'PRIVATE-PROMPT-ECHO'; + const req = { + user: { id: 'user-123' }, + body: { + text: 'Fail during generation.', + messageId: 'user-msg', + conversationId: 'conversation-123', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: { + filters: { + messages: { + pii: { + starterPatterns: [], + customPatterns: [{ id: 'secret', label: 'secret', regex: 'NEVER-MATCHES-XYZ' }], + }, + }, + }, + }, + }; + const client = { + options: {}, + sendMessage: jest.fn().mockRejectedValue(new Error(`Provider echoed ${privateValue}`)), + }; + const initializeClient = jest.fn().mockResolvedValue({ client }); + + await AgentController(req, createResumableResponse(), jest.fn(), initializeClient, null); + await nextTick(); + + expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith( + 'conversation-123', + 'Generation failed', + 1000, + expect.objectContaining({ beforeErrorPublication: expect.any(Function) }), + ); + expect(mockSaveMessage).toHaveBeenCalledWith( + expect.objectContaining({ userId: 'user-123' }), + expect.objectContaining({ error: true, text: 'Generation failed' }), + expect.any(Object), + ); + expect(JSON.stringify(mockGenerationJobManager.completeJob.mock.calls)).not.toContain( + privateValue, + ); + expect(JSON.stringify(mockSaveMessage.mock.calls)).not.toContain(privateValue); + }); + + it('redacts credentials and emails from a generation error published and persisted without a content policy', async () => { + const secrets = [ + 'sk-proj-PRIVATEKEY123', + 'PRIVATEHEADER123', + 'PRIVATESIG123', + 'PRIVATEAUTH123', + 'private.user@example.com', + ]; + const rawMessage = [ + 'Incorrect API key provided: sk-proj-PRIVATEKEY123;', + 'x-api-key: PRIVATEHEADER123;', + 'token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJQUklWQVRFIn0.PRIVATESIG123;', + 'owner private.user@example.com;', + 'Authorization: Basic PRIVATEAUTH123', + ].join(' '); + const req = { + user: { id: 'user-123' }, + body: { + text: 'Fail during generation.', + messageId: 'user-msg', + conversationId: 'conversation-123', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: {}, + }; + const client = { + options: {}, + sendMessage: jest.fn().mockRejectedValue(new Error(rawMessage)), + }; + const initializeClient = jest.fn().mockResolvedValue({ client }); + + await AgentController(req, createResumableResponse(), jest.fn(), initializeClient, null); + await nextTick(); + + const [, published] = mockGenerationJobManager.completeJob.mock.calls[0]; + expect(published).toContain('Incorrect API key provided'); + expect(mockSaveMessage).toHaveBeenCalledWith( + expect.objectContaining({ userId: 'user-123' }), + expect.objectContaining({ error: true, text: published }), + expect.any(Object), + ); + for (const secret of secrets) { + expect(JSON.stringify(mockGenerationJobManager.completeJob.mock.calls)).not.toContain(secret); + expect(JSON.stringify(mockSaveMessage.mock.calls)).not.toContain(secret); + } + }); + + it('keeps a typed generation error under a content policy so the client can localize it', async () => { + const typedError = JSON.stringify({ type: 'no_user_key' }); + const req = { + user: { id: 'user-123' }, + body: { + text: 'Fail during generation.', + messageId: 'user-msg', + conversationId: 'conversation-123', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: { + filters: { + messages: { + pii: { + starterPatterns: [], + customPatterns: [{ id: 'secret', label: 'secret', regex: 'NEVER-MATCHES-XYZ' }], + }, + }, + }, + }, + }; + const client = { + options: {}, + sendMessage: jest.fn().mockRejectedValue(new Error(typedError)), + }; + const initializeClient = jest.fn().mockResolvedValue({ client }); + + await AgentController(req, createResumableResponse(), jest.fn(), initializeClient, null); + await nextTick(); + + expect(mockSaveMessage).toHaveBeenCalledWith( + expect.objectContaining({ userId: 'user-123' }), + expect.objectContaining({ error: true, text: typedError }), + expect.any(Object), + ); + }); + it('publishes a safe image storage failure during generation and releases the request', async () => { const signedUrl = 'https://minio.example.com/bucket/image.png?X-Amz-Credential=secret&X-Amz-Signature=signed'; diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index 9cdba77cb2..d43a1f9849 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -60,6 +60,7 @@ const { resolveFailedTurnContent, savePrivateTextMessage, savePrivateTextErrorTurn, + getUserFacingGenerationError, stampPreliminaryPrivateTextMessage, announceReply, announceErrorTurn, @@ -3323,7 +3324,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit sendGenerationJson( res, 500, - { error: error.message || 'Failed to start generation' }, + { + error: getUserFacingGenerationError(error, req.config, 'Failed to start generation'), + }, generationProtocolVersion, ); } @@ -3381,7 +3384,11 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit // abortJob already handled emitDone and completeJob } else { logger.error(`[ResumableAgentController] Generation error for ${streamId}:`, error); - const generationError = error.message || 'Generation failed'; + const generationError = getUserFacingGenerationError( + error, + req.config, + 'Generation failed', + ); try { // completeJob first wins running -> error and atomically parks // steers, then publishes. A competing abort/pause emits nothing. @@ -3454,22 +3461,24 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit sendGenerationJson( res, 500, - { error: err.message || 'Failed to start generation' }, + { error: getUserFacingGenerationError(err, req.config, 'Failed to start generation') }, generationProtocolVersion, ); } let errorFinalized = false; if (!pausePersistenceFailed) { errorFinalized = - (await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch( - (completeErr) => { - logger.warn( - '[ResumableAgentController] completeJob failed during background-error cleanup', - completeErr, - ); - return false; - }, - )) === true; + (await GenerationJobManager.completeJob( + streamId, + getUserFacingGenerationError(err, req.config, 'Generation failed'), + jobCreatedAt, + ).catch((completeErr) => { + logger.warn( + '[ResumableAgentController] completeJob failed during background-error cleanup', + completeErr, + ); + return false; + })) === true; } if ( (errorFinalized || (pausePersistenceFailed && pausePersistenceFailureFinalized)) && @@ -3570,7 +3579,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit sendGenerationJson( res, 500, - { error: error.message || 'Failed to start generation' }, + { + error: getUserFacingGenerationError(error, req.config, 'Failed to start generation'), + }, generationProtocolVersion, ); } @@ -3597,7 +3608,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit if (jobCreatedAt != null) { const initializationError = initializationFailure ? JSON.stringify(initializationFailure) - : error.message || 'Failed to start generation'; + : getUserFacingGenerationError(error, req.config, 'Failed to start generation'); const persistInitializationError = shouldPersistCodeWorkspaceInitializationError({ streamStarted, isNewConversation: isNewConvo, diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index ef1cfecd5b..eb2a1d2b74 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -1,4 +1,4 @@ -const { logger, redactMessage, getTenantId } = require('@librechat/data-schemas'); +const { logger, getTenantId } = require('@librechat/data-schemas'); const { tool: toolFn, DynamicStructuredTool } = require('@librechat/agents/langchain/tools'); const { sleep, @@ -77,6 +77,7 @@ const { appendAssistantMCPAppArtifact, resolveMCPClientCapabilityProfile, getOutboundTimeoutMs, + getToolErrorText, } = require('@librechat/api'); const { Time, @@ -732,7 +733,7 @@ async function processRequiredActions(client, requiredActions) { const output = getSafeRequiredActionOutput( client, currentAction, - `Error processing tool ${currentAction.tool}: ${redactMessage(error.message, 256)}`, + `Error processing tool ${currentAction.tool}: ${getToolErrorText(error)}`, ); return { tool_call_id: currentAction.toolCallId, diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index 84c63f8ae5..abb25d2443 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -193,7 +193,7 @@ const { } = require('../ToolService'); const { createOnSearchResults } = require('~/server/services/Tools/search'); const { reinitMCPServer } = require('~/server/services/Tools/mcp'); -const { ContentFilterError, PENDING_STALE_MS } = require('@librechat/api'); +const { ContentFilterError, PENDING_STALE_MS, ToolInputError } = require('@librechat/api'); /** Role document shape `checkAccess` reads; all three role-gated tools granted. */ function buildRole(overrides = {}) { @@ -689,6 +689,46 @@ describe('ToolService - Action Capability Gating', () => { expect(output).not.toContain('bearer_header'); }); + it('returns approved tool error text instead of a raw exception message', async () => { + const rawMessage = + 'connect ECONNREFUSED 10.0.0.5:443 for https://api.example.test/?q=PRIVATE'; + mockLoadToolsUtil.mockResolvedValue({ + loadedTools: [ + { name: 'safe_tool', _call: jest.fn().mockRejectedValue(new Error(rawMessage)) }, + ], + toolContextMap: {}, + }); + + const result = await processRequiredActions(buildClient(undefined), [buildAction()]); + + expect(result.tool_outputs).toEqual([ + { + tool_call_id: 'call_1', + output: 'Error processing tool safe_tool: The tool request failed.', + }, + ]); + }); + + it('keeps the message of a tool input error the tool wrote for the model', async () => { + mockLoadToolsUtil.mockResolvedValue({ + loadedTools: [ + { + name: 'safe_tool', + _call: jest + .fn() + .mockRejectedValue(new ToolInputError('Could not find coordinates for city: Rome')), + }, + ], + toolContextMap: {}, + }); + + const result = await processRequiredActions(buildClient(undefined), [buildAction()]); + + expect(result.tool_outputs[0].output).toBe( + 'Error processing tool safe_tool: Could not find coordinates for city: Rome', + ); + }); + it('blocks persisted Assistant action metadata before required-action execution', async () => { const actionToolName = `get_weather${actionDelimiter}api_example_com`; const filters = { diff --git a/e2e/specs/mock/scenarios/generation-error-text.spec.ts b/e2e/specs/mock/scenarios/generation-error-text.spec.ts new file mode 100644 index 0000000000..98217fb77e --- /dev/null +++ b/e2e/specs/mock/scenarios/generation-error-text.spec.ts @@ -0,0 +1,52 @@ +import { resolve } from 'node:path'; +import { expect, test } from '@playwright/test'; +import { run, repoRoot, inOneProject } from './lint.helpers'; + +/** + * A generation that fails outside the model run publishes its error and persists it on the failed + * turn's error row. These run the resumable controller's own suite for those paths: the real + * controller and `saveErrorTurn`, with the job store and message writes captured. + */ + +const apiRoot = resolve(repoRoot, 'api'); + +function runControllerTests(pattern: string): void { + const result = run( + 'npx', + [ + 'jest', + 'server/controllers/agents/__tests__/request.resumeMetadata.spec.js', + '-t', + pattern, + '--maxWorkers=1', + ], + { cwd: apiRoot }, + ); + expect(result.status, result.output).toBe(0); + /** Jest may color the summary, so match the count alone. */ + expect(result.output).toMatch(/(? { + test.beforeEach(() => inOneProject()); + + test('under a content policy a raw generation error is published and persisted as the fallback @scenario:generation-error-withheld-under-policy', () => { + test.setTimeout(300_000); + runControllerTests('publishes and persists the fallback for a raw generation error'); + }); + + test('under a content policy a typed generation error keeps its code for the client @scenario:typed-generation-error-kept', () => { + test.setTimeout(300_000); + runControllerTests('keeps a typed generation error under a content policy'); + }); + + test('without a content policy a host-authored generation error keeps its wording @scenario:generation-error-kept-without-policy', () => { + test.setTimeout(300_000); + runControllerTests('publishes a safe image storage failure during generation'); + }); + + test('without a content policy credentials and emails never reach the published or persisted error @scenario:generation-error-secrets-redacted', () => { + test.setTimeout(300_000); + runControllerTests('redacts credentials and emails from a generation error'); + }); +}); diff --git a/e2e/specs/mock/scenarios/tool-error-text.spec.ts b/e2e/specs/mock/scenarios/tool-error-text.spec.ts new file mode 100644 index 0000000000..cd208d8267 --- /dev/null +++ b/e2e/specs/mock/scenarios/tool-error-text.spec.ts @@ -0,0 +1,124 @@ +import { resolve } from 'node:path'; +import { expect, test } from '@playwright/test'; +import { run, repoRoot, inOneProject } from './lint.helpers'; + +/** + * A tool result lands in the conversation, so a failing tool may only hand the model text the + * tool wrote for it. The probes load the real tools from `api/` and replace only the transport + * (node-fetch, axios) so the upstream failure is deterministic and never leaves the machine. + */ + +const apiRoot = resolve(repoRoot, 'api'); +const SECRET = 'PRIVATE-PROVIDER-DETAIL'; + +function probe(source: string): Record { + const result = run('node', ['-e', source], { cwd: apiRoot }); + expect(result.status, result.output).toBe(0); + const marker = result.stdout.lastIndexOf('RESULT'); + expect(marker, result.output).toBeGreaterThanOrEqual(0); + return JSON.parse(result.stdout.slice(marker + 'RESULT'.length)); +} + +const OPEN_WEATHER_PROBE = ` +process.env.OPENWEATHER_API_KEY = 'probe-api-key'; +const fetchPath = require.resolve('node-fetch'); +let respond; +require.cache[fetchPath] = { + id: fetchPath, + filename: fetchPath, + loaded: true, + exports: (url) => respond(url), +}; +const OpenWeather = require('./app/clients/tools/structured/OpenWeather'); +(async () => { + const tool = new OpenWeather(); + const out = {}; + respond = (url) => Promise.reject(new Error('request to ' + url + ' failed, reason: ${SECRET}')); + out.network = await tool._call({ action: 'current_forecast', lat: 1, lon: 2 }); + respond = () => Promise.resolve({ ok: false, status: 401, json: async () => ({ message: '${SECRET}' }) }); + out.unauthorized = await tool._call({ action: 'current_forecast', lat: 1, lon: 2 }); + respond = () => Promise.resolve({ ok: true, json: async () => [] }); + out.city = await tool._call({ action: 'current_forecast', city: 'Atlantis' }); + out.secretCity = await tool._call({ action: 'current_forecast', city: 'sk-proj-PRIVATEKEY123 owner@example.com' }); + out.date = await tool._call({ action: 'timestamp', lat: 1, lon: 2, date: '2020/03/04' }); + process.stdout.write('RESULT' + JSON.stringify(out)); + process.exit(0); +})().catch((error) => { console.error(error); process.exit(1); }); +`; + +const WOLFRAM_PROBE = ` +process.env.WOLFRAM_APP_ID = 'probe-app-id'; +const axios = require('axios'); +const WolframAlphaAPI = require('./app/clients/tools/structured/Wolfram'); +const httpError = (status) => + Object.assign(new Error('Request failed with status code ' + status), { + response: { status, data: 'Wolfram|Alpha did not understand ${SECRET}' }, + }); +(async () => { + const tool = new WolframAlphaAPI({}); + const out = {}; + axios.get = () => Promise.reject(httpError(501)); + out.uninterpretable = await tool._call({ input: '${SECRET}' }); + axios.get = () => Promise.reject(httpError(503)); + out.unavailable = await tool._call({ input: 'integrate x' }); + axios.get = () => Promise.reject(new Error('connect ECONNREFUSED ${SECRET}')); + out.network = await tool._call({ input: 'integrate x' }); + axios.get = () => Promise.resolve({ data: 'x^2/2' }); + out.ok = await tool._call({ input: 'integrate x' }); + process.stdout.write('RESULT' + JSON.stringify(out)); + process.exit(0); +})().catch((error) => { console.error(error); process.exit(1); }); +`; + +test.describe('approved tool error text', () => { + test.beforeEach(() => inOneProject()); + + test('a failing OpenWeather call returns approved text and keeps its own input guidance @scenario:openweather-approved-error-text', () => { + test.setTimeout(120_000); + const out = probe(OPEN_WEATHER_PROBE); + + expect(out.network).toBe('Error: The tool request failed.'); + expect(out.unauthorized).toBe( + 'Error: OpenWeather rejected the configured credentials (status 401).', + ); + expect(out.city).toBe('Error: Could not find coordinates for city: Atlantis'); + expect(out.date).toBe('Error: Invalid date format. Expected YYYY-MM-DD.'); + expect(out.secretCity).toMatch(/^Error: Could not find coordinates for city: /); + expect(out.secretCity).not.toContain('PRIVATEKEY123'); + expect(out.secretCity).not.toContain('owner@example.com'); + expect(JSON.stringify(out)).not.toContain(SECRET); + expect(JSON.stringify(out)).not.toContain('probe-api-key'); + }); + + test('a failing Wolfram|Alpha call returns approved text, not the provider body @scenario:wolfram-approved-error-text', () => { + test.setTimeout(120_000); + const out = probe(WOLFRAM_PROBE); + + expect(out.uninterpretable).toMatch(/^Error: Wolfram\|Alpha could not interpret the input\./); + expect(out.unavailable).toBe( + 'Error: Wolfram|Alpha is unavailable; try again later (status 503).', + ); + expect(out.network).toBe('Error: The tool request failed.'); + expect(out.ok).toBe('x^2/2'); + expect(JSON.stringify(out)).not.toContain(SECRET); + expect(JSON.stringify(out)).not.toContain('probe-app-id'); + }); + + test('an Assistants required action whose tool throws submits approved text @scenario:assistant-tool-error-approved-text', () => { + test.setTimeout(300_000); + const result = run( + 'npx', + [ + 'jest', + 'server/services/__tests__/ToolService.spec.js', + '-t', + 'approved tool error text|tool input error the tool wrote', + '--maxWorkers=1', + ], + { cwd: apiRoot }, + ); + expect(result.status, result.output).toBe(0); + /** Jest may color the summary, so match the count alone. */ + expect(result.output).toMatch(/(? { it('logs stable upstream metadata and deterministic trace correlation', () => { @@ -373,3 +379,123 @@ describe('terminal agent-run error logging', () => { }); }); }); + +describe('getUserFacingGenerationError', () => { + const protectedConfig = { + filters: { + messages: { + pii: { + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE' }], + }, + }, + }, + }; + + it('replaces a raw exception message with the fallback while a content policy is active', () => { + const error = new Error('400 Bad Request: prompt "PRIVATE-PROMPT" was rejected'); + expect(getUserFacingGenerationError(error, protectedConfig, 'Generation failed')).toBe( + 'Generation failed', + ); + }); + + it('keeps the message with URLs redacted when no content policy is active', () => { + const error = new Error('request to https://api.example.test/?api_key=secret-key failed'); + expect(getUserFacingGenerationError(error, {}, 'Generation failed')).toBe( + 'request to https://api.example.test/[redacted] failed', + ); + expect(getUserFacingGenerationError(new Error(' '), undefined, 'Generation failed')).toBe( + 'Generation failed', + ); + expect(getUserFacingGenerationError('PRIVATE-STRING', undefined, 'Generation failed')).toBe( + 'Generation failed', + ); + }); + + it.each([ + ['an sk- key', 'Incorrect API key provided: sk-proj-PRIVATEKEY123', 'PRIVATEKEY123'], + ['an x-api-key header', 'upstream rejected x-api-key: PRIVATEHEADER123', 'PRIVATEHEADER123'], + [ + 'a JWT', + 'token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJQUklWQVRFIn0.PRIVATESIG123 expired', + 'PRIVATESIG123', + ], + ['an Authorization header', 'sent Authorization: Basic PRIVATEAUTH123', 'PRIVATEAUTH123'], + ['an email address', 'no account for private.user@example.com', 'private.user@example.com'], + ])('never returns %s without a content policy', (_label, message, secret) => { + const text = getUserFacingGenerationError(new Error(message), {}, 'Generation failed'); + expect(text).not.toContain(secret); + expect(text).not.toBe('Generation failed'); + }); + + it('keeps a typed payload the client maps to localized copy under protection', () => { + const typed = JSON.stringify({ type: ErrorTypes.NO_USER_KEY }); + const violation = JSON.stringify({ type: ViolationTypes.TOKEN_BALANCE, balance: 0 }); + expect(getUserFacingGenerationError(new Error(typed), protectedConfig, 'x')).toBe(typed); + expect(getUserFacingGenerationError(new Error(violation), protectedConfig, 'x')).toBe( + violation, + ); + }); + + it('keeps only the code and numeric fields of a typed payload under a content policy', () => { + const typed = JSON.stringify({ + type: ViolationTypes.TOKEN_BALANCE, + balance: 0, + tokenCost: 120, + message: 'PRIVATE-PROMPT', + prompt: 'PRIVATE-FIELD', + nested: { value: 'PRIVATE-NESTED' }, + }); + const text = getUserFacingGenerationError(new Error(typed), protectedConfig, 'x'); + expect(JSON.parse(text)).toEqual({ + type: ViolationTypes.TOKEN_BALANCE, + balance: 0, + tokenCost: 120, + }); + }); + + it('redacts credentials in a typed payload without a content policy', () => { + const typed = JSON.stringify({ type: ErrorTypes.INVALID_REQUEST, info: 'key sk-PRIVATEKEY99' }); + const text = getUserFacingGenerationError(new Error(typed), {}, 'x'); + expect(JSON.parse(text).type).toBe(ErrorTypes.INVALID_REQUEST); + expect(text).not.toContain('PRIVATEKEY99'); + }); + + it('treats JSON with an unknown type as an ordinary message', () => { + const untyped = JSON.stringify({ type: 'PRIVATE-TYPE', prompt: 'PRIVATE-PROMPT' }); + expect(getUserFacingGenerationError(new Error(untyped), protectedConfig, 'fallback')).toBe( + 'fallback', + ); + expect(getUserFacingGenerationError(new Error('[1,2]'), protectedConfig, 'fallback')).toBe( + 'fallback', + ); + expect(getUserFacingGenerationError(new Error('{not json'), protectedConfig, 'fallback')).toBe( + 'fallback', + ); + }); + + it('classifies a LangChain-coded provider failure under protection', () => { + const error = Object.assign(new Error('rate limited PRIVATE-DETAIL'), { + lc_error_code: 'MODEL_RATE_LIMIT', + }); + expect(getUserFacingGenerationError(error, protectedConfig, 'Generation failed')).toBe( + JSON.stringify({ type: ErrorTypes.MODEL_RATE_LIMIT }), + ); + }); + + it('keeps the host-authored message of a content policy block under protection', () => { + const error = new ContentFilterError({ + detectorId: 'pii-pattern', + ruleId: 'bearer_header', + label: 'Bearer token', + source: 'file', + field: 'content', + provenance: 'tool', + fragmentId: 'generated-file', + fragmentPath: '/content', + }); + expect( + getUserFacingGenerationError(new Error('wrapped', { cause: error }), protectedConfig, 'x'), + ).toBe('Submitted content contains a Bearer token. Remove it and try again.'); + }); +}); diff --git a/packages/api/src/agents/failures/terminal.ts b/packages/api/src/agents/failures/terminal.ts index 79e054ad8d..346dbae16d 100644 --- a/packages/api/src/agents/failures/terminal.ts +++ b/packages/api/src/agents/failures/terminal.ts @@ -1,4 +1,6 @@ -import { ErrorTypes } from 'librechat-data-provider'; +import { redactMessage } from '@librechat/data-schemas'; +import { ErrorTypes, ViolationTypes } from 'librechat-data-provider'; +import type { FiltersConfig, MessageFilterPiiConfig } from 'librechat-data-provider'; import type { SafeErrorMetadata } from '../../utils/errors'; import type { ModelErrorTrackerCallback } from './tracker'; import { @@ -7,7 +9,9 @@ import { resolveModelStreamError, getProviderErrorMessage, } from '../errors'; -import { getSafeErrorMetadata, isOwnedAbortError } from '../../utils/errors'; +import { getSafeErrorText, getSafeErrorMetadata, isOwnedAbortError } from '../../utils/errors'; +import { hasModelBoundContentProtection } from '../../middleware/modelBoundContent'; +import { getContentFilterError } from '../../middleware/contentFilter'; import { traceIdForMessage } from '../../langfuse/trace'; import { createModelErrorTracker } from './tracker'; @@ -148,3 +152,95 @@ export function createTerminalRunErrorObserver({ }, }); } + +const TYPED_ERROR_CODES: ReadonlySet = new Set([ + ...Object.values(ErrorTypes), + ...Object.values(ViolationTypes), +]); + +type TypedErrorPayload = Record & { type: string }; + +/** A host-thrown `{"type": ...}` payload whose type the client maps to localized copy. */ +function parseTypedErrorMessage(message: string): TypedErrorPayload | undefined { + if (!message.trimStart().startsWith('{')) { + return undefined; + } + try { + const parsed: unknown = JSON.parse(message); + if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) { + return undefined; + } + const type = (parsed as { type?: unknown }).type; + return typeof type === 'string' && TYPED_ERROR_CODES.has(type) + ? (parsed as TypedErrorPayload) + : undefined; + } catch { + return undefined; + } +} + +/** + * The typed payload as a reader may see it: credentials and emails are always redacted, and while + * a content policy is active only the code and its numeric and boolean fields (a balance, a limit) + * remain, since any string field may carry provider or submitted content. + */ +function getSafeTypedErrorText( + message: string, + payload: TypedErrorPayload, + protectionEnabled: boolean, +): string { + if (!protectionEnabled) { + return redactMessage(message); + } + const safe: Record = { type: payload.type }; + for (const [key, value] of Object.entries(payload)) { + if ((typeof value === 'number' && Number.isFinite(value)) || typeof value === 'boolean') { + safe[key] = value; + } + } + return JSON.stringify(safe); +} + +/** The deployment config that decides whether a failure may show its own text. */ +export interface GenerationErrorConfig { + readonly filters?: FiltersConfig | null; + readonly messageFilter?: { readonly pii?: MessageFilterPiiConfig | null } | null; +} + +/** + * The error text a failed generation publishes and persists on its error row, under the same rule + * the run's content error parts follow. Host-authored outcomes keep their wording: a typed + * `{"type": ...}` payload (redacted, and reduced to its code and numeric fields under a content policy), a LangChain-classified provider failure, or a content policy block. + * Any other exception may echo submitted content, so while a content policy is active the turn + * carries `fallback`; otherwise it carries the message with URLs, credentials (API keys, tokens, auth headers) and email + * addresses redacted. The exception + * itself reaches the logs only through `getSafeErrorMetadata`. + */ +export function getUserFacingGenerationError( + error: unknown, + appConfig: GenerationErrorConfig | null | undefined, + fallback: string, +): string { + const policyError = getContentFilterError(error); + if (policyError != null) { + return policyError.message; + } + const message = error instanceof Error ? error.message : undefined; + const protectionEnabled = hasModelBoundContentProtection( + appConfig?.filters, + appConfig?.messageFilter?.pii, + ); + const typed = message != null ? parseTypedErrorMessage(message) : undefined; + if (message != null && typed != null) { + return getSafeTypedErrorText(message, typed, protectionEnabled); + } + const classified = safelyResolveLangChainError(error); + if (classified != null) { + return classified; + } + if (message == null || protectionEnabled) { + return fallback; + } + const trimmed = message.trim(); + return trimmed === '' ? fallback : redactMessage(getSafeErrorText(trimmed)); +} diff --git a/packages/api/src/tools/format.spec.ts b/packages/api/src/tools/format.spec.ts index a04fbb4b5d..cdb3e3962b 100644 --- a/packages/api/src/tools/format.spec.ts +++ b/packages/api/src/tools/format.spec.ts @@ -1,6 +1,17 @@ +import { logger } from '@librechat/data-schemas'; import { AuthType, EToolResources } from 'librechat-data-provider'; import type { TPlugin } from 'librechat-data-provider'; -import { filterUniquePlugins, checkPluginAuth, getToolkitKey, formatToolError } from './format'; +import { + ToolInputError, + getToolkitKey, + checkPluginAuth, + formatToolError, + getToolErrorText, + filterUniquePlugins, + formatToolHttpError, + TOOL_ERROR_FALLBACK, + WOLFRAM_STATUS_TEXT, +} from './format'; describe('format.ts helper functions', () => { describe('filterUniquePlugins', () => { @@ -349,20 +360,91 @@ describe('format.ts helper functions', () => { }); describe('formatToolError', () => { - it('redacts a key carried in the failed request URL', () => { + it('returns the fallback instead of a raw exception message', () => { const error = new Error( 'request to https://api.example.test/data?lat=1&appid=secret-key failed, reason: ENOTFOUND', ); - expect(formatToolError(error)).toBe( - 'Error: request to https://api.example.test/data?lat=1&appid=[REDACTED] failed, reason: ENOTFOUND', + expect(formatToolError(error, 'Example')).toBe(`Error: ${TOOL_ERROR_FALLBACK}`); + expect(formatToolError('timeout')).toBe(`Error: ${TOOL_ERROR_FALLBACK}`); + }); + + it('keeps the message of a tool input error', () => { + expect(formatToolError(new ToolInputError('Could not find coordinates for city: Rome'))).toBe( + 'Error: Could not find coordinates for city: Rome', ); }); - it('keeps a plain message and stringifies non-errors', () => { - expect(formatToolError(new Error('Could not find coordinates for city: Rome'))).toBe( - 'Error: Could not find coordinates for city: Rome', + it('redacts credentials and emails a tool input error echoes from its arguments', () => { + const text = formatToolError( + new ToolInputError( + 'Could not find coordinates for city: sk-proj-PRIVATEKEY123 private.user@example.com', + ), + ); + expect(text).toContain('Could not find coordinates for city:'); + expect(text).not.toContain('PRIVATEKEY123'); + expect(text).not.toContain('private.user@example.com'); + expect(getToolErrorText(new ToolInputError('key=PRIVATEQUERY123 rejected'))).not.toContain( + 'PRIVATEQUERY123', + ); + }); + + it('logs only safe metadata for an unapproved failure', () => { + const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger); + const error = Object.assign(new Error('upstream said PRIVATE-DETAIL'), { + code: 'ECONNRESET', + }); + + formatToolError(error, 'Example'); + + expect(warn).toHaveBeenCalledWith('[Example] Tool request failed', { type: 'Error' }); + expect(JSON.stringify(warn.mock.calls)).not.toContain('PRIVATE-DETAIL'); + warn.mockRestore(); + }); + + it('words an HTTP failure from its status, never its response body', () => { + const httpError = (status: number) => + Object.assign(new Error('Request failed'), { + response: { status, data: 'PRIVATE-PROVIDER-BODY' }, + }); + + expect(formatToolError(httpError(503), 'Wolfram|Alpha', WOLFRAM_STATUS_TEXT)).toBe( + 'Error: Wolfram|Alpha is unavailable; try again later (status 503).', + ); + expect(formatToolError(httpError(501), 'Wolfram|Alpha', WOLFRAM_STATUS_TEXT)).toBe( + `Error: ${WOLFRAM_STATUS_TEXT[501]}`, + ); + expect(formatToolError(httpError(404), 'Example')).not.toContain('PRIVATE-PROVIDER-BODY'); + }); + }); + + describe('getToolErrorText', () => { + it('approves only tool input errors', () => { + expect(getToolErrorText(new ToolInputError('Invalid date format.'))).toBe( + 'Invalid date format.', + ); + expect(getToolErrorText(new Error('socket hang up'))).toBe(TOOL_ERROR_FALLBACK); + expect(getToolErrorText({ message: 'not an error' })).toBe(TOOL_ERROR_FALLBACK); + }); + }); + + describe('formatToolHttpError', () => { + it.each([ + [401, 'Error: Service rejected the configured credentials (status 401).'], + [404, 'Error: Service found no result for this request (status 404).'], + [429, 'Error: Service is rate limiting requests; try again later (status 429).'], + [ + 400, + 'Error: Service rejected the request parameters; check them and try again (status 400).', + ], + [503, 'Error: Service is unavailable; try again later (status 503).'], + ])('words status %i without provider text', (status, expected) => { + expect(formatToolHttpError('Service', status)).toBe(expected); + }); + + it('prefers a tool-specific wording for a documented status', () => { + expect(formatToolHttpError('Service', 501, { 501: 'Rephrase the input.' })).toBe( + 'Error: Rephrase the input.', ); - expect(formatToolError('timeout')).toBe('Error: timeout'); }); }); }); diff --git a/packages/api/src/tools/format.ts b/packages/api/src/tools/format.ts index 3304e56c1b..71f25ce387 100644 --- a/packages/api/src/tools/format.ts +++ b/packages/api/src/tools/format.ts @@ -1,6 +1,7 @@ -import { redactMessage } from '@librechat/data-schemas'; +import { logger, redactMessage } from '@librechat/data-schemas'; import { AuthType, EToolResources } from 'librechat-data-provider'; import type { TPlugin } from 'librechat-data-provider'; +import { getSafeErrorMetadata } from '~/utils/errors'; /** * Filters out duplicate plugins from the list of plugins. @@ -84,8 +85,90 @@ export function getToolkitKey({ return toolkitKey; } -/** Error text a tool returns to the model, with credentials in request URLs and headers redacted. */ -export function formatToolError(error: unknown): string { - const message = error instanceof Error ? error.message : String(error); - return `Error: ${redactMessage(message)}`; +/** Tool error text for any failure the tool did not word for the model itself. */ +export const TOOL_ERROR_FALLBACK = 'The tool request failed.'; + +/** + * A tool failure whose message the tool wrote for the model: input guidance such as an unknown + * city or a malformed date. It is the only exception text a tool result carries; provider, + * network and runtime messages stay in the logs. + */ +export class ToolInputError extends Error { + constructor(message: string) { + super(message); + this.name = 'ToolInputError'; + Object.setPrototypeOf(this, ToolInputError.prototype); + } } + +/** + * The approved text for a tool failure: a `ToolInputError` message with credentials and email + * addresses redacted (it may echo a tool argument), otherwise the fallback. + */ +export function getToolErrorText(error: unknown): string { + return error instanceof ToolInputError ? redactMessage(error.message) : TOOL_ERROR_FALLBACK; +} + +/** Approved wording for an upstream HTTP status, keyed by the status class a model can act on. */ +function describeToolHttpStatus(status: number): string { + if (status === 401 || status === 403) { + return 'rejected the configured credentials'; + } + if (status === 404) { + return 'found no result for this request'; + } + if (status === 429) { + return 'is rate limiting requests; try again later'; + } + if (status >= 500) { + return 'is unavailable; try again later'; + } + if (status >= 400) { + return 'rejected the request parameters; check them and try again'; + } + return `failed with status ${status}`; +} + +/** + * Error text for a tool whose upstream service answered with a failing HTTP status. The response + * body is provider text, so only the status reaches the model; `overrides` lets a tool word a + * status its service documents specifically. + */ +export function formatToolHttpError( + service: string, + status: number, + overrides?: Readonly>>, +): string { + logger.warn(`[${service}] Tool request failed`, { status }); + const override = overrides?.[status]; + if (override != null) { + return `Error: ${override}`; + } + return `Error: ${service} ${describeToolHttpStatus(status)} (status ${status}).`; +} + +/** + * Error text a tool returns to the model. A `ToolInputError` keeps its message; an upstream HTTP + * failure is worded from its status alone (see `formatToolHttpError`); anything else becomes the + * stable fallback, and only its sanitized metadata is logged under `service`. + */ +export function formatToolError( + error: unknown, + service = 'tool', + statusOverrides?: Readonly>>, +): string { + if (error instanceof ToolInputError) { + return `Error: ${getToolErrorText(error)}`; + } + const metadata = getSafeErrorMetadata(error); + if (metadata.status != null && metadata.status >= 400) { + return formatToolHttpError(service, metadata.status, statusOverrides); + } + logger.warn(`[${service}] Tool request failed`, metadata); + return `Error: ${TOOL_ERROR_FALLBACK}`; +} + +/** The Wolfram|Alpha LLM API answers 501 when it cannot interpret the input; its body echoes the query. */ +export const WOLFRAM_STATUS_TEXT: Readonly>> = { + 501: 'Wolfram|Alpha could not interpret the input. Rephrase it as a simpler query, such as single keywords or standard math notation, and try again.', +};